diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/actions')
82 files changed, 9549 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java b/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java new file mode 100644 index 0000000000..87105d549c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/AbstractAction.java @@ -0,0 +1,420 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.util.Collection; + +/** + * Abstract implementation of Action which implements basic functionality: the + * inputs, outputs, and toString method. Both input and output sets are + * immutable. + */ +@Immutable @ThreadSafe +public abstract class AbstractAction implements Action { + + /** + * An arbitrary default resource set. Currently 250MB of memory, 50% CPU and 0% of total I/O. + */ + public static final ResourceSet DEFAULT_RESOURCE_SET = new ResourceSet(250, 0.5, 0); + + // owner/inputs/outputs attributes below should never be directly accessed even + // within AbstractAction itself. The appropriate getter methods should be used + // instead. This has to be done due to the fact that the getter methods can be + // overridden in subclasses. + private final ActionOwner owner; + // The variable inputs is non-final only so that actions that discover their inputs can modify it. + private Iterable<Artifact> inputs; + private final ImmutableSet<Artifact> outputs; + + private int cachedInputCount = -1; + private String cachedKey; + + /** + * Construct an abstract action with the specified inputs and outputs; + */ + protected AbstractAction(ActionOwner owner, + Iterable<Artifact> inputs, + Iterable<Artifact> outputs) { + Preconditions.checkNotNull(owner); + // TODO(bazel-team): Use RuleContext.actionOwner here instead + this.owner = new ActionOwnerDescription(owner); + this.inputs = CollectionUtils.makeImmutable(inputs); + this.outputs = ImmutableSet.copyOf(outputs); + Preconditions.checkArgument(!this.outputs.isEmpty(), owner); + } + + @Override + public final ActionOwner getOwner() { + return owner; + } + + @Override + public boolean inputsKnown() { + return true; + } + + @Override + public boolean discoversInputs() { + return false; + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + throw new IllegalStateException("discoverInputs cannot be called for " + this.prettyPrint() + + " since it does not discover inputs"); + } + + @Override + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths) { + throw new IllegalStateException( + "Method must be overridden for actions that may have unknown inputs."); + } + + /** + * Should only be overridden by actions that need to optionally insert inputs. Actions that + * discover their inputs should use {@link #setInputs} to set the new iterable of inputs when they + * know it. + */ + @Override + public Iterable<Artifact> getInputs() { + return inputs; + } + + /** + * Set the inputs of the action. May only be used by an action that {@link #discoversInputs()}. + * The iterable passed in is automatically made immutable. + */ + public void setInputs(Iterable<Artifact> inputs) { + Preconditions.checkState(discoversInputs()); + this.inputs = CollectionUtils.makeImmutable(inputs); + cachedInputCount = -1; + } + + /* + * Get count of inputs. + * + * <p>Computes the count on first invocation, returns cached value for further invocations. + */ + @Override + @ThreadSafe + public synchronized int getInputCount() { + if (cachedInputCount == -1) { + cachedInputCount = Iterables.size(getInputs()); + } + return cachedInputCount; + } + + @Override + public ImmutableSet<Artifact> getOutputs() { + return outputs; + } + + @Override + public Artifact getPrimaryInput() { + // The default behavior is to return the first input artifact. + // Call through the method, not the field, because it may be overridden. + return Iterables.getFirst(getInputs(), null); + } + + @Override + public Artifact getPrimaryOutput() { + // Default behavior is to return the first output artifact. + // Use the method rather than field in case of overriding in subclasses. + return Iterables.getFirst(getOutputs(), null); + } + + @Override + public Iterable<Artifact> getMandatoryInputs() { + return getInputs(); + } + + @Override + public String toString() { + return prettyPrint() + " (" + getMnemonic() + "[" + ImmutableList.copyOf(getInputs()) + + (inputsKnown() ? " -> " : ", unknown inputs -> ") + + getOutputs() + "]" + ")"; + } + + @Override + public abstract String getMnemonic(); + protected abstract String computeKey(); + + @Override + public synchronized final String getKey() { + if (cachedKey == null) { + cachedKey = computeKey(); + } + return cachedKey; + } + + @Override + public String describeKey() { + return null; + } + + @Override + public boolean executeUnconditionally() { + return false; + } + + @Override + public boolean isVolatile() { + return false; + } + + @Override + public boolean showsOutputUnconditionally() { + return false; + } + + @Override + public final String getProgressMessage() { + String message = getRawProgressMessage(); + if (message == null) { + return null; + } + String additionalInfo = getOwner().getAdditionalProgressInfo(); + return additionalInfo == null ? message : message + " [" + additionalInfo + "]"; + } + + /** + * Returns a progress message string that is specific for this action. This is + * then annotated with additional information, currently the string '[for host]' + * for actions in the host configurations. + * + * <p>A return value of null indicates no message should be reported. + */ + protected String getRawProgressMessage() { + // A cheesy default implementation. Subclasses are invited to do something + // more meaningful. + return defaultProgressMessage(); + } + + private String defaultProgressMessage() { + return getMnemonic() + " " + getPrimaryOutput().prettyPrint(); + } + + @Override + public String prettyPrint() { + return "action '" + describe() + "'"; + } + + /** + * Deletes all of the action's output files, if they exist. If any of the + * Artifacts refers to a directory recursively removes the contents of the + * directory. + * + * @param execRoot the exec root in which this action is executed + */ + protected void deleteOutputs(Path execRoot) throws IOException { + for (Artifact output : getOutputs()) { + deleteOutput(output); + } + } + + /** + * Helper method to remove an Artifact. If the Artifact refers to a directory + * recursively removes the contents of the directory. + */ + protected void deleteOutput(Artifact output) throws IOException { + Path path = output.getPath(); + try { + // Optimize for the common case: output artifacts are files. + path.delete(); + } catch (IOException e) { + // Only try to recursively delete a directory if the output root is known. This is just a + // sanity check so that we do not start deleting random files on disk. + // TODO(bazel-team): Strengthen this test by making sure that the output is part of the + // output tree. + if (path.isDirectory(Symlinks.NOFOLLOW) && output.getRoot() != null) { + FileSystemUtils.deleteTree(path); + } else { + throw e; + } + } + } + + /** + * If the action might read directories as inputs in a way that is unsound wrt dependency + * checking, this method must be called. + */ + protected void checkInputsForDirectories(EventHandler eventHandler, + MetadataHandler metadataHandler) { + // Report "directory dependency checking" warning only for non-generated directories (generated + // ones will be reported earlier). + for (Artifact input : getMandatoryInputs()) { + // Assume that if the file did not exist, we would not have gotten here. + if (input.isSourceArtifact() && !metadataHandler.isRegularFile(input)) { + eventHandler.handle(Event.warn(getOwner().getLocation(), "input '" + + input.prettyPrint() + "' to " + getOwner().getLabel() + + " is a directory; dependency checking of directories is unsound")); + } + } + } + + @Override + public MiddlemanType getActionType() { + return MiddlemanType.NORMAL; + } + + /** + * If the action might create directories as outputs this method must be called. + */ + protected void checkOutputsForDirectories(EventHandler eventHandler) { + for (Artifact output : getOutputs()) { + Path path = output.getPath(); + String ownerString = Label.print(getOwner().getLabel()); + if (path.isDirectory()) { + eventHandler.handle(new Event(EventKind.WARNING, getOwner().getLocation(), + "output '" + output.prettyPrint() + "' of " + ownerString + + " is a directory; dependency checking of directories is unsound", + ownerString)); + } + } + } + + @Override + public void prepare(Path execRoot) throws IOException { + deleteOutputs(execRoot); + } + + @Override + public String describe() { + String progressMessage = getProgressMessage(); + return progressMessage != null ? progressMessage : defaultProgressMessage(); + } + + @Override + public abstract ResourceSet estimateResourceConsumption(Executor executor); + + @Override + public boolean shouldReportPathPrefixConflict(Action action) { + return this != action; + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + return ExtraActionInfo.newBuilder() + .setOwner(getOwner().getLabel().toString()) + .setId(getKey()) + .setMnemonic(getMnemonic()); + } + + /** + * Returns input files that need to be present to allow extra_action rules to shadow this action + * correctly when run remotely. This is at least the normal inputs of the action, but may include + * other files as well. For example C(++) compilation may perform include file header scanning. + * This needs to be mirrored by the extra_action rule. Called by + * {@link com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time. + * + * <p>As this method is called from the ExtraAction, make sure it is ok to call + * this method from a different thread than the one this action is executed on. + * + * @param actionExecutionContext Services in the scope of the action, like the Out/Err streams. + * @throws ActionExecutionException only when code called from this method + * throws that exception. + * @throws InterruptedException if interrupted + */ + public Iterable<Artifact> getInputFilesForExtraAction( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + return getInputs(); + } + + /** + * A copying implementation of {@link ActionOwner}. + * + * <p>ConfiguredTargets implement ActionOwner themselves, but we do not want actions + * to keep direct references to configured targets just for a label and a few strings. + */ + @Immutable + private static class ActionOwnerDescription implements ActionOwner { + + private final Location location; + private final Label label; + private final String configurationName; + private final String configurationMnemonic; + private final String configurationKey; + private final String targetKind; + private final String additionalProgressInfo; + + private ActionOwnerDescription(ActionOwner originalOwner) { + this.location = originalOwner.getLocation(); + this.label = originalOwner.getLabel(); + this.configurationName = originalOwner.getConfigurationName(); + this.configurationMnemonic = originalOwner.getConfigurationMnemonic(); + this.configurationKey = originalOwner.getConfigurationShortCacheKey(); + this.targetKind = originalOwner.getTargetKind(); + this.additionalProgressInfo = originalOwner.getAdditionalProgressInfo(); + } + + @Override + public Location getLocation() { + return location; + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public String getConfigurationName() { + return configurationName; + } + + @Override + public String getConfigurationMnemonic() { + return configurationMnemonic; + } + + @Override + public String getConfigurationShortCacheKey() { + return configurationKey; + } + + @Override + public String getTargetKind() { + return targetKind; + } + + @Override + public String getAdditionalProgressInfo() { + return additionalProgressInfo; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java new file mode 100644 index 0000000000..0272160b96 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/AbstractActionOwner.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * An action owner base class that provides default implementations for some of + * the {@link ActionOwner} methods. + */ +public abstract class AbstractActionOwner implements ActionOwner { + + @Override + public String getAdditionalProgressInfo() { + return null; + } + + @Override + public Location getLocation() { + return null; + } + + @Override + public Label getLabel() { + return null; + } + + @Override + public String getTargetKind() { + return "empty target kind"; + } + + @Override + public String getConfigurationName() { + return "empty configuration"; + } + + /** + * An action owner for special cases. Usage is strongly discouraged. + */ + public static final ActionOwner SYSTEM_ACTION_OWNER = new AbstractActionOwner() { + @Override + public final String getConfigurationName() { + return "system"; + } + + @Override + public String getConfigurationMnemonic() { + return "system"; + } + + @Override + public final String getConfigurationShortCacheKey() { + return "system"; + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/Action.java b/src/main/java/com/google/devtools/build/lib/actions/Action.java new file mode 100644 index 0000000000..4970203ece --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Action.java @@ -0,0 +1,188 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadCompatible; +import com.google.devtools.build.lib.profiler.Describable; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * An Action represents a function from Artifacts to Artifacts executed as an + * atomic build step. Examples include compilation of a single C++ source + * file, or linking a single library. + */ +public interface Action extends ActionMetadata, Describable { + + /** + * Prepares for executing this action; called by the Builder prior to + * executing the Action itself. This method should prepare the file system, so + * that the execution of the Action can write the output files. At a minimum + * any pre-existing and write protected output files should be removed or the + * permissions should be changed, so that they can be safely overwritten by + * the action. + * + * @throws IOException if there is an error deleting the outputs. + */ + void prepare(Path execRoot) throws IOException; + + /** + * Executes this action; called by the Builder when all of this Action's + * inputs have been successfully created. (Behaviour is undefined if the + * prerequisites are not up to date.) This method <i>actually does the work + * of the Action, unconditionally</i>; in other words, it is invoked by the + * Builder only when dependency analysis has deemed it necessary.</p> + * + * <p>The framework guarantees that the output directory for each file in + * <code>getOutputs()</code> has already been created, and will check to + * ensure that each of those files is indeed created.</p> + * + * <p>Implementations of this method should try to honour the {@link + * java.lang.Thread#interrupted} contract: if an interrupt is delivered to + * the thread in which execution occurs, the action should detect this on a + * best-effort basis and terminate as quickly as possible by throwing an + * ActionExecutionException. + * + * <p>Action execution must be ThreadCompatible in order to be safely used + * with a concurrent Builder implementation such as ParallelBuilder. + * + * @param actionExecutionContext Services in the scope of the action, like the output and error + * streams to use for messages arising during action execution. + * @throws ActionExecutionException if execution fails for any reason. + * @throws InterruptedException + */ + @ConditionallyThreadCompatible + void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; + + /** + * Returns true iff action must be executed regardless of its current state. + * Default implementation can be overridden by some actions that might be + * executed unconditionally under certain circumstances - e.g., if caching of + * test results is not requested, this method could be used to force test + * execution even if all dependencies are up-to-date. + * + * <p>Note, it is <b>very</b> important not to abuse this method, since it + * completely overrides dependency checking. Any use of this method must + * be carefully reviewed and proved to be necessary. + * + * <p>Note that the definition of {@link #isVolatile} depends on the + * definition of this method, so be sure to consider both methods together + * when making changes. + */ + boolean executeUnconditionally(); + + /** + * Returns true if it's ever possible that {@link #executeUnconditionally} + * could evaluate to true during the lifetime of this instance, false + * otherwise. + */ + boolean isVolatile(); + + /** + * Method used to find inputs before execution for an action that + * {@link ActionMetadata#discoversInputs}. + */ + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; + + /** + * Method used to update action inputs based on the information contained in + * the action cache. It will be called iff inputsKnown() is false for the + * given action instance and there is a related cache entry in the action + * cache. + * + * Method must be redefined for any action that may return + * inputsKnown() == false. It also expects that implementation will ensure + * that inputsKnown() returns true after call to this method. + * + * @param artifactResolver the artifact factory that can be used to manufacture artifacts + * @param inputPaths List of relative (to the execution root) input paths + */ + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection<PathFragment> inputPaths); + + /** + * Return a best-guess estimate of the operation's resource consumption on the + * local host itself for use in scheduling. + * + * @param executor the application-specific value passed to the + * executor parameter of the top-level call to + * Builder.buildArtifacts(). + */ + @Nullable ResourceSet estimateResourceConsumption(Executor executor); + + /** + * @return true iff path prefix conflict (conflict where two actions generate + * two output artifacts with one of the artifact's path being the + * prefix for another) between this action and another action should + * be reported. + */ + boolean shouldReportPathPrefixConflict(Action action); + + /** + * Returns true if the output should bypass output filtering. This is used for test actions. + */ + boolean showsOutputUnconditionally(); + + /** + * Called by {@link com.google.devtools.build.lib.rules.extra.ExtraAction} at execution time to + * extract information from this action into a protocol buffer to be used by extra_action rules. + * + * <p>As this method is called from the ExtraAction, make sure it is ok to call this method from + * a different thread than the one this action is executed on. + */ + ExtraActionInfo.Builder getExtraActionInfo(); + + /** + * Returns the action type. Must not be {@code null}. + */ + MiddlemanType getActionType(); + + /** + * The action type. + */ + public enum MiddlemanType { + + /** A normal action. */ + NORMAL, + + /** A normal middleman, which just encapsulates a list of artifacts. */ + AGGREGATING_MIDDLEMAN, + + /** + * A middleman that enforces action ordering, is not validated by the dependency checker, but + * allows errors to be propagated. + */ + ERROR_PROPAGATING_MIDDLEMAN, + + /** + * A runfiles middleman, which is validated by the dependency checker, but is not expanded + * in blaze. Instead, the runfiles manifest is sent to remote execution client, which + * performs the expansion. + */ + RUNFILES_MIDDLEMAN; + + public boolean isMiddleman() { + return this != NORMAL; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java new file mode 100644 index 0000000000..2525c8dea8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java @@ -0,0 +1,341 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.Digest; +import com.google.devtools.build.lib.actions.cache.Metadata; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Checks whether an {@link Action} needs to be executed, or whether it has not changed since it was + * last stored in the action cache. Must be informed of the new Action data after execution as well. + * + * <p>The fingerprint, input files names, and metadata (either mtimes or MD5sums) of each action are + * cached in the action cache to avoid unnecessary rebuilds. Middleman artifacts are handled + * specially, avoiding the need to create actual files corresponding to the middleman artifacts. + * Instead of that, results of MiddlemanAction dependency checks are cached internally and then + * reused whenever an input middleman artifact is encountered. + * + * <p>While instances of this class hold references to action and metadata cache instances, they are + * otherwise lightweight, and should be constructed anew and discarded for each build request. + */ +public class ActionCacheChecker { + private final ActionCache actionCache; + private final Predicate<? super Action> executionFilter; + private final ArtifactResolver artifactResolver; + // True iff --verbose_explanations flag is set. + private final boolean verboseExplanations; + + public ActionCacheChecker(ActionCache actionCache, ArtifactResolver artifactResolver, + Predicate<? super Action> executionFilter, boolean verboseExplanations) { + this.actionCache = actionCache; + this.executionFilter = executionFilter; + this.artifactResolver = artifactResolver; + this.verboseExplanations = verboseExplanations; + } + + public boolean isActionExecutionProhibited(Action action) { + return !executionFilter.apply(action); + } + + /** + * Checks whether one of existing output paths is already used as a key. + * If yes, returns it - otherwise uses first output file as a key + */ + private ActionCache.Entry getCacheEntry(Action action) { + for (Artifact output : action.getOutputs()) { + ActionCache.Entry entry = actionCache.get(output.getExecPathString()); + if (entry != null) { + return entry; + } + } + return null; + } + + /** + * Validate metadata state for action input or output artifacts. + * + * @param entry cached action information. + * @param action action to be validated. + * @param metadataHandler provider of metadata for the artifacts this action interacts with. + * @param checkOutput true to validate output artifacts, Otherwise, just + * validate inputs. + * + * @return true if at least one artifact has changed, false - otherwise. + */ + private boolean validateArtifacts(ActionCache.Entry entry, Action action, + MetadataHandler metadataHandler, boolean checkOutput) { + Iterable<Artifact> artifacts = checkOutput + ? Iterables.concat(action.getOutputs(), action.getInputs()) + : action.getInputs(); + Map<String, Metadata> mdMap = new HashMap<>(); + for (Artifact artifact : artifacts) { + mdMap.put(artifact.getExecPathString(), metadataHandler.getMetadataMaybe(artifact)); + } + return !Digest.fromMetadata(mdMap).equals(entry.getFileDigest()); + } + + private void reportCommand(EventHandler handler, Action action) { + if (handler != null) { + if (verboseExplanations) { + String keyDescription = action.describeKey(); + reportRebuild(handler, action, + keyDescription == null ? "action command has changed" : + "action command has changed.\nNew action: " + keyDescription); + } else { + reportRebuild(handler, action, + "action command has changed (try --verbose_explanations for more info)"); + } + } + } + + protected boolean unconditionalExecution(Action action) { + return !isActionExecutionProhibited(action) && action.executeUnconditionally(); + } + + /** + * Checks whether {@code action} needs to be executed and returns a non-null Token if so. + * + * <p>The method checks if any of the action's inputs or outputs have changed. Returns a non-null + * {@link Token} if the action needs to be executed, and null otherwise. + * + * <p>If this method returns non-null, indicating that the action will be executed, the + * metadataHandler's {@link MetadataHandler#discardMetadata} method must be called, so that it + * does not serve stale metadata for the action's outputs after the action is executed. + */ + // Note: the handler should only be used for DEPCHECKER events; there's no + // guarantee it will be available for other events. + public Token getTokenIfNeedToExecute(Action action, EventHandler handler, + MetadataHandler metadataHandler) { + // TODO(bazel-team): (2010) For RunfilesAction/SymlinkAction and similar actions that + // produce only symlinks we should not check whether inputs are valid at all - all that matters + // that inputs and outputs are still exist (and new inputs have not appeared). All other checks + // are unnecessary. In other words, the only metadata we should check for them is file existence + // itself. + + MiddlemanType middlemanType = action.getActionType(); + if (middlemanType.isMiddleman()) { + // Some types of middlemen are not checked because they should not + // propagate invalidation of their inputs. + if (middlemanType != MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN) { + checkMiddlemanAction(action, handler, metadataHandler); + } + return null; + } + ActionCache.Entry entry = null; // Populated lazily. + + // Update action inputs from cache, if necessary. + boolean inputsKnown = action.inputsKnown(); + if (!inputsKnown) { + Preconditions.checkState(action.discoversInputs()); + entry = getCacheEntry(action); + updateActionInputs(action, entry); + } + if (mustExecute(action, entry, handler, metadataHandler)) { + return new Token(getKeyString(action)); + } + return null; + } + + protected boolean mustExecute(Action action, @Nullable ActionCache.Entry entry, + EventHandler handler, MetadataHandler metadataHandler) { + // Unconditional execution can be applied only for actions that are allowed to be executed. + if (unconditionalExecution(action)) { + Preconditions.checkState(action.isVolatile()); + reportUnconditionalExecution(handler, action); + return true; // must execute - unconditional execution is requested. + } + + if (entry == null) { + entry = getCacheEntry(action); + } + if (entry == null) { + reportNewAction(handler, action); + return true; // must execute -- no cache entry (e.g. first build) + } + + if (entry.isCorrupted()) { + reportCorruptedCacheEntry(handler, action); + return true; // cache entry is corrupted - must execute + } else if (validateArtifacts(entry, action, metadataHandler, true)) { + reportChanged(handler, action); + return true; // files have changed + } else if (!entry.getActionKey().equals(action.getKey())){ + reportCommand(handler, action); + return true; // must execute -- action key is different + } + + entry.getFileDigest(); + return false; // cache hit + } + + public void afterExecution(Action action, Token token, MetadataHandler metadataHandler) + throws IOException { + Preconditions.checkArgument(token != null); + String key = token.cacheKey; + ActionCache.Entry entry = actionCache.createEntry(action.getKey()); + for (Artifact output : action.getOutputs()) { + // Remove old records from the cache if they used different key. + String execPath = output.getExecPathString(); + if (!key.equals(execPath)) { + actionCache.remove(key); + } + // Output files *must* exist and be accessible after successful action execution. + Metadata metadata = metadataHandler.getMetadata(output); + Preconditions.checkState(metadata != null); + entry.addFile(output.getExecPath(), metadata); + } + for (Artifact input : action.getInputs()) { + entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input)); + } + entry.getFileDigest(); + actionCache.put(key, entry); + } + + protected void updateActionInputs(Action action, ActionCache.Entry entry) { + if (entry == null || entry.isCorrupted()) { + return; + } + + List<PathFragment> outputs = new ArrayList<>(); + for (Artifact output : action.getOutputs()) { + outputs.add(output.getExecPath()); + } + List<PathFragment> inputs = new ArrayList<>(); + for (String path : entry.getPaths()) { + PathFragment execPath = new PathFragment(path); + // Code assumes that action has only 1-2 outputs and ArrayList.contains() will be + // most efficient. + if (!outputs.contains(execPath)) { + inputs.add(execPath); + } + } + action.updateInputsFromCache(artifactResolver, inputs); + } + + /** + * Special handling for the MiddlemanAction. Since MiddlemanAction output + * artifacts are purely fictional and used only to stay within dependency + * graph model limitations (action has to depend on artifacts, not on other + * actions), we do not need to validate metadata for the outputs - only for + * inputs. We also do not need to validate MiddlemanAction key, since action + * cache entry key already incorporates that information for the middlemen + * and we will experience a cache miss when it is different. Whenever it + * encounters middleman artifacts as input artifacts for other actions, it + * consults with the aggregated middleman digest computed here. + */ + protected void checkMiddlemanAction(Action action, EventHandler handler, + MetadataHandler metadataHandler) { + Artifact middleman = action.getPrimaryOutput(); + String cacheKey = middleman.getExecPathString(); + ActionCache.Entry entry = actionCache.get(cacheKey); + boolean changed = false; + if (entry != null) { + if (entry.isCorrupted()) { + reportCorruptedCacheEntry(handler, action); + changed = true; + } else if (validateArtifacts(entry, action, metadataHandler, false)) { + reportChanged(handler, action); + changed = true; + } + } else { + reportChangedDeps(handler, action); + changed = true; + } + if (changed) { + // Compute the aggregated middleman digest. + // Since we never validate action key for middlemen, we should not store + // it in the cache entry and just use empty string instead. + entry = actionCache.createEntry(""); + for (Artifact input : action.getInputs()) { + entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input)); + } + } + + metadataHandler.setDigestForVirtualArtifact(middleman, entry.getFileDigest()); + if (changed) { + actionCache.put(cacheKey, entry); + } + } + + /** + * Returns an action key. It is always set to the first output exec path string. + */ + private static String getKeyString(Action action) { + Preconditions.checkState(!action.getOutputs().isEmpty()); + return action.getOutputs().iterator().next().getExecPathString(); + } + + + /** + * In most cases, this method should not be called directly - reportXXX() methods + * should be used instead. This is done to avoid cost associated with building + * the message. + */ + private static void reportRebuild(@Nullable EventHandler handler, Action action, String message) { + // For MiddlemanAction, do not report rebuild. + if (handler != null && !action.getActionType().isMiddleman()) { + handler.handle(new Event( + EventKind.DEPCHECKER, null, "Executing " + action.prettyPrint() + ": " + message + ".")); + } + } + + // Called by IncrementalDependencyChecker. + protected static void reportUnconditionalExecution( + @Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "unconditional execution is requested"); + } + + private static void reportChanged(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "One of the files has changed"); + } + + private static void reportChangedDeps(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "the set of files on which this action depends has changed"); + } + + private static void reportNewAction(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "no entry in the cache (action is new)"); + } + + private static void reportCorruptedCacheEntry(@Nullable EventHandler handler, Action action) { + reportRebuild(handler, action, "cache entry is corrupted"); + } + + /** Wrapper for all context needed by the ActionCacheChecker to handle a single action. */ + public static final class Token { + private final String cacheKey; + + private Token(String cacheKey) { + this.cacheKey = Preconditions.checkNotNull(cacheKey); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java new file mode 100644 index 0000000000..a2cb5776c8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCompletionEvent.java @@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An event that is fired after an action completes (either successfully or not). + */ +public final class ActionCompletionEvent { + + private final ActionMetadata actionMetadata; + + public ActionCompletionEvent(ActionMetadata actionMetadata) { + this.actionMetadata = actionMetadata; + } + + /** + * Returns the action metadata. + */ + public ActionMetadata getActionMetadata() { + return actionMetadata; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java new file mode 100644 index 0000000000..4c0eaa2aa8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextConsumer.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.util.Map; + +/** + * An object describing that actions require a particular implementation of an + * {@link ActionContext}. + * + * <p>This is expected to be implemented by modules that also implement actions which need these + * contexts. Other modules will provide implementations for various action contexts by implementing + * {@link ActionContextProvider}. + * + * <p>Example: a module requires {@code SpawnActionContext} to do its job, and it creates + * actions with the mnemonic <code>C++</code>. Then the {@link #getSpawnActionContexts} method of + * this module would return a map with the key <code>"C++"</code> in it. + * + * <p>The module can either decide for itself which implementation is needed and make the value + * associated with this key a constant or defer that decision to the user, for example, by + * providing a command line option and setting the value in the map based on that. + * + * <p>Other modules are free to provide different implementations of {@code SpawnActionContext}. + * This can be used, for example, to implement sandboxed or distributed execution of + * {@code SpawnAction}s in different ways, while giving the user control over how exactly they + * are executed. + */ +public interface ActionContextConsumer { + /** + * Returns a map from spawn action mnemonics created by this module to the name of the + * implementation of {@code SpawnActionContext} that the module wants to use for executing + * it. + * + * <p>If a spawn action is executed whose mnemonic maps to the empty string or is not + * present in the map at all, the choice of the implementation is left to Blaze. + */ + public Map<String, String> getSpawnActionContexts(); + + /** + * Returns a map from action context class to the implementation required by the module. + * + * <p>If the implementation name is the empty string, the choice is left to Blaze. + */ + public Map<Class<? extends ActionContext>, String> getActionContexts(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java new file mode 100644 index 0000000000..fcb2e3d555 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextMarker.java @@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation for action contexts. Actions contexts should also implement {@link ActionContext}. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface ActionContextMarker { + String name(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java new file mode 100644 index 0000000000..6e52a4a404 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionContextProvider.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +/** + * An object that provides execution strategies to {@link BlazeExecutor}. + * + * <p>For more information, see {@link ActionContextConsumer}. + */ +public interface ActionContextProvider { + /** + * Returns the execution strategies that are provided by this object. + * + * <p>These may or may not actually end up in the executor depending on the command line options + * and other factors influencing how the executor is set up. + */ + Iterable<ActionContext> getActionContexts(); + + /** + * Called when the executor is constructed. The parameter contains all the contexts that were + * selected for this execution phase. + */ + void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException; + + /** + * Called when the execution phase is started. + */ + void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable<Artifact> topLevelArtifacts) + throws ExecutorInitException, InterruptedException; + + /** + * Called when the execution phase is finished. + */ + void executionPhaseEnding(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java new file mode 100644 index 0000000000..00ef9b41b4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutedEvent.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * This event is fired during the build, when an action is executed. It contains information about + * the action: the Action itself, and the output file names its stdout and stderr are recorded in. + */ +public class ActionExecutedEvent { + private final Action action; + private final ActionExecutionException exception; + private final String stdout; + private final String stderr; + + public ActionExecutedEvent(Action action, + ActionExecutionException exception, String stdout, String stderr) { + this.action = action; + this.exception = exception; + this.stdout = stdout; + this.stderr = stderr; + } + + public Action getAction() { + return action; + } + + // null if action succeeded + public ActionExecutionException getException() { + return exception; + } + + public String getStdout() { + return stdout; + } + + public String getStderr() { + return stderr; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java new file mode 100644 index 0000000000..d6d08faa7b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionContext.java @@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.util.io.FileOutErr; + +/** + * A class that groups services in the scope of the action. Like the FileOutErr object. + */ +public class ActionExecutionContext { + + private final Executor executor; + private final ActionInputFileCache actionInputFileCache; + private final MetadataHandler metadataHandler; + private final FileOutErr fileOutErr; + private final MiddlemanExpander middlemanExpander; + + public ActionExecutionContext(Executor executor, ActionInputFileCache actionInputFileCache, + MetadataHandler metadataHandler, FileOutErr fileOutErr, MiddlemanExpander middlemanExpander) { + this.actionInputFileCache = actionInputFileCache; + this.metadataHandler = metadataHandler; + this.fileOutErr = fileOutErr; + this.executor = executor; + this.middlemanExpander = middlemanExpander; + } + + public ActionInputFileCache getActionInputFileCache() { + return actionInputFileCache; + } + + public MetadataHandler getMetadataHandler() { + return metadataHandler; + } + + public Executor getExecutor() { + return executor; + } + + public MiddlemanExpander getMiddlemanExpander() { + return middlemanExpander; + } + + /** + * Provide that {@code FileOutErr} that the action should use for redirecting the output and error + * stream. + */ + public FileOutErr getFileOutErr() { + return fileOutErr; + } + + /** + * Allows us to create a new context that overrides the FileOutErr with another one. This is + * useful for muting the output for example. + */ + public ActionExecutionContext withFileOutErr(FileOutErr fileOutErr) { + return new ActionExecutionContext(executor, actionInputFileCache, metadataHandler, fileOutErr, + middlemanExpander); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java new file mode 100644 index 0000000000..0d0d908a01 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionException.java @@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * This exception gets thrown if {@link Action#execute(ActionExecutionContext)} is unsuccessful. + * Typically these are re-raised ExecException throwables. + */ +@ThreadSafe +public class ActionExecutionException extends Exception { + + private final Action action; + private final NestedSet<Label> rootCauses; + private final boolean catastrophe; + + public ActionExecutionException(Throwable cause, Action action, boolean catastrophe) { + super(cause.getMessage(), cause); + this.action = action; + this.rootCauses = rootCausesFromAction(action); + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, + Throwable cause, Action action, boolean catastrophe) { + super(message + ": " + cause.getMessage(), cause); + this.action = action; + this.rootCauses = rootCausesFromAction(action); + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, Action action, boolean catastrophe) { + super(message); + this.action = action; + this.rootCauses = rootCausesFromAction(action); + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, Action action, + NestedSet<Label> rootCauses, boolean catastrophe) { + super(message); + this.action = action; + this.rootCauses = rootCauses; + this.catastrophe = catastrophe; + } + + public ActionExecutionException(String message, Throwable cause, Action action, + NestedSet<Label> rootCauses, boolean catastrophe) { + super(message, cause); + this.action = action; + this.rootCauses = rootCauses; + this.catastrophe = catastrophe; + } + + static NestedSet<Label> rootCausesFromAction(Action action) { + return action == null || action.getOwner() == null || action.getOwner().getLabel() == null + ? NestedSetBuilder.<Label>emptySet(Order.STABLE_ORDER) + : NestedSetBuilder.create(Order.STABLE_ORDER, action.getOwner().getLabel()); + } + + /** + * Returns the action that failed. + */ + public Action getAction() { + return action; + } + + /** + * Return the root causes that should be reported. Usually the owner of the action, but it can + * be the label of a missing artifact. + */ + public NestedSet<Label> getRootCauses() { + return rootCauses; + } + + /** + * Returns the location of the owner of this action. May be null. + */ + public Location getLocation() { + return action.getOwner().getLocation(); + } + + /** + * Catastrophic exceptions should stop builds, even if --keep_going. + */ + public boolean isCatastrophe() { + return catastrophe; + } + + /** + * Returns true if the error should be shown. + */ + public boolean showError() { + return getMessage() != null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java new file mode 100644 index 0000000000..34aadc4a33 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporter.java @@ -0,0 +1,262 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.Pair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +import javax.annotation.Nullable; + +/** + * Implements "Still waiting..." message functionality, displaying current status for "in-flight" + * actions. Used by the ParallelBuilder. + * + * TODO(bazel-team): (2010) It would be nice if "duplicated" actions (e.g. test shards and multiple + * test runs) were merged into the single line. + */ +@ThreadSafe +public final class ActionExecutionStatusReporter { + // Maximum number of lines to output per each status category before truncation. + private static final int MAX_LINES = 10; + + private final EventHandler eventHandler; + private final Executor executor; + private final EventBus eventBus; + private final Clock clock; + + /** + * The status of each action "in flight", i.e. whose ExecuteBuildAction.call() method is active. + * Used for implementing the "still waiting" message. + */ + private final Map<ActionMetadata, Pair<String, Long>> actionStatus = + new ConcurrentHashMap<>(100); + + public static ActionExecutionStatusReporter create(EventHandler eventHandler) { + return create(eventHandler, null, null); + } + + @VisibleForTesting + static ActionExecutionStatusReporter create(EventHandler eventHandler, Clock clock) { + return create(eventHandler, null, null, clock); + } + + public static ActionExecutionStatusReporter create(EventHandler eventHandler, + @Nullable Executor executor, @Nullable EventBus eventBus) { + return create(eventHandler, executor, eventBus, null); + } + + private static ActionExecutionStatusReporter create(EventHandler eventHandler, + @Nullable Executor executor, @Nullable EventBus eventBus, @Nullable Clock clock) { + ActionExecutionStatusReporter result = new ActionExecutionStatusReporter(eventHandler, executor, + eventBus, clock == null ? BlazeClock.instance() : clock); + if (eventBus != null) { + eventBus.register(result); + } + return result; + } + + private ActionExecutionStatusReporter(EventHandler eventHandler, @Nullable Executor executor, + @Nullable EventBus eventBus, Clock clock) { + this.eventHandler = Preconditions.checkNotNull(eventHandler); + this.executor = executor; + this.eventBus = eventBus; + this.clock = Preconditions.checkNotNull(clock); + } + + public void unregisterFromEventBus() { + if (eventBus != null) { + eventBus.unregister(this); + } + } + + private void setStatus(ActionMetadata action, String message) { + actionStatus.put(action, Pair.of(message, clock.nanoTime())); + } + + /** + * Remove action from the list of active actions. + */ + public void remove(Action action) { + Preconditions.checkNotNull(actionStatus.remove(action), action); + } + + /** + * Set "Preparing" status. + */ + public void setPreparing(Action action) { + updateStatus(ActionStatusMessage.preparingStrategy(action)); + } + + public void setRunningFromBuildData(ActionMetadata action) { + updateStatus(ActionStatusMessage.runningStrategy(action)); + } + + @Subscribe + public void updateStatus(ActionStatusMessage statusMsg) { + String message = statusMsg.getMessage(); + ActionMetadata action = statusMsg.getActionMetadata(); + if (statusMsg.needsStrategy()) { + String strategy = action.describeStrategy(executor); + if (strategy == null) { + return; + } + message = String.format(message, strategy); + } + setStatus(action, message); + } + + public int getCount() { + return actionStatus.size(); + } + + private static void appendGroupStatus(StringBuilder buffer, + Map<ActionMetadata, Pair<String, Long>> statusMap, String status, long currentTime) { + List<Pair<Long, ActionMetadata>> actions = new ArrayList<>(); + for (Map.Entry<ActionMetadata, Pair<String, Long>> entry : statusMap.entrySet()) { + if (entry.getValue().first.equals(status)) { + actions.add(Pair.of(entry.getValue().second, entry.getKey())); + } + } + if (actions.size() == 0) { + return; + } + Collections.sort(actions, Pair.<Long, ActionMetadata>compareByFirst()); + + buffer.append("\n " + status + ":"); + + boolean truncateList = actions.size() > MAX_LINES; + for (Pair<Long, ActionMetadata> entry : actions.subList(0, + truncateList ? MAX_LINES - 1 : actions.size())) { + String message = entry.second.getProgressMessage(); + if (message == null) { + // Actions will a null progress message should run so + // fast we never see them here. In any case... + message = entry.second.prettyPrint(); + } + buffer.append("\n ").append(message); + long runTime = (currentTime - entry.first) / 1000000000L; // Convert to seconds. + buffer.append(", ").append(runTime).append(" s"); + } + if (truncateList) { + buffer.append("\n ... ").append(actions.size() - MAX_LINES + 1).append(" more jobs"); + } + } + + /** + * Get message showing currently executing actions. + */ + private String getExecutionStatusMessage(Map<ActionMetadata, Pair<String, Long>> statusMap) { + int count = statusMap.size(); + StringBuilder s = count != 1 + ? new StringBuilder("Still waiting for ").append(count).append(" jobs to complete:") + : new StringBuilder("Still waiting for 1 job to complete:"); + + long currentTime = clock.nanoTime(); + + // A tree is just as fast as HashSet for small data sets. + Set<String> statuses = new TreeSet<String>(); + for (Map.Entry<ActionMetadata, Pair<String, Long>> entry : statusMap.entrySet()) { + statuses.add(entry.getValue().first); + } + + for (String status : statuses) { + appendGroupStatus(s, statusMap, status, currentTime); + } + return s.toString(); + } + + /** + * Show currently executing actions. + */ + public void showCurrentlyExecutingActions(String progressPercentageMessage) { + // Defensive copy to ensure thread safety. + Map<ActionMetadata, Pair<String, Long>> statusMap = new HashMap<>(actionStatus); + if (statusMap.size() > 0) { + eventHandler.handle( + Event.progress(progressPercentageMessage + getExecutionStatusMessage(statusMap))); + } + } + + /** + * Warn about actions that are still being executed. + * Method is used to produce informative message when build is interrupted. + */ + void warnAboutCurrentlyExecutingActions() { + // Defensive copy to ensure thread safety. + Map<ActionMetadata, Pair<String, Long>> statusMap = new HashMap<>(actionStatus); + if (statusMap.size() == 0) { + // There are no tasks in the queue so there is nothing to report. + eventHandler.handle(Event.warn("There are no active jobs - stopping the build")); + return; + } + Iterator<ActionMetadata> iterator = statusMap.keySet().iterator(); + while (iterator.hasNext()) { + // Filter out actions that are not executed yet. + if (statusMap.get(iterator.next()).first.equals(ActionStatusMessage.PREPARING)) { + iterator.remove(); + } + } + if (statusMap.size() > 0) { + eventHandler.handle(Event.warn(getExecutionStatusMessage(statusMap) + + "\nBuild will be stopped after these tasks terminate")); + } else { + // It is possible that one or more tasks in "Preparing" state just started being executed. + // So warn user just in case. + eventHandler.handle(Event.warn("Still waiting for unfinished jobs")); + } + } + + /** + * Returns the number of seconds to wait before reporting slow progress again. + * + * @param userSpecifiedProgressInterval value of the --progress_report_interval flag; 0 means + * use default 10, then 30, then 60 seconds wait times + * @param previousWaitTime previous value returned by this method + */ + public static int getWaitTime(int userSpecifiedProgressInterval, int previousWaitTime) { + if (userSpecifiedProgressInterval > 0) { + return userSpecifiedProgressInterval; + } + + // Increase waitTime to 10, then to 30 and then to 60 seconds to reduce + // spamming during long wait periods. If the user specified a + // waitTime directly through progressReportInterval, then use + // that value. + if (previousWaitTime == 0) { + return 10; + } else if (previousWaitTime == 10) { + return 30; + } else { + return 60; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java new file mode 100644 index 0000000000..bb2b707989 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionGraph.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import javax.annotation.Nullable; + +/** + * An action graph. + * + * <p>Provides lookups of generating actions for artifacts. + */ +public interface ActionGraph { + + /** + * Returns the Action that, when executed, gives rise to this file. + * + * <p>If this Artifact is a source file, null is returned. (We don't try to return a "no-op + * action" because that would require creating a new no-op Action for every source file, since + * each Action knows its outputs, so sharing all the no-ops is not an option.) + * + * <p>It's also possible for derived Artifacts to have null generating Actions when these actions + * are unknown. + */ + @Nullable + Action getGeneratingAction(Artifact artifact); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java b/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java new file mode 100644 index 0000000000..4ddda4c027 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionGraphVisitor.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An abstract visitor for the action graph. Specializes {@link BipartiteVisitor} for artifacts and + * actions, and takes care of visiting the complete transitive closure. + */ +public abstract class ActionGraphVisitor extends BipartiteVisitor<Action, Artifact> { + + private final ActionGraph actionGraph; + + public ActionGraphVisitor(ActionGraph actionGraph) { + this.actionGraph = actionGraph; + } + + /** + * Called for all artifacts in the visitation. Hook for subclasses. + * + * @param artifact + */ + protected void visitArtifact(Artifact artifact) {} + + /** + * Called for all actions in the visitation. Hook for subclasses. + * + * @param action + */ + protected void visitAction(Action action) {} + + /** + * Whether the given action should be visited. If this returns false, the visitation stops here, + * so the dependencies of this action are also not visited. + * + * @param action + */ + protected boolean shouldVisit(Action action) { + return true; + } + + /** + * Whether the given artifact should be visited. If this returns false, the visitation stops here, + * so dependencies of this artifact (if it is a generated one) are also not visited. + * + * @param artifact + */ + protected boolean shouldVisit(Artifact artifact) { + return true; + } + + @SuppressWarnings("unused") + protected final void visitArtifacts(Iterable<Artifact> artifacts) { + for (Artifact artifact : artifacts) { + visitArtifact(artifact); + } + } + + @Override protected void white(Artifact artifact) { + Action action = actionGraph.getGeneratingAction(artifact); + visitArtifact(artifact); + if (action != null && shouldVisit(action)) { + visitBlackNode(action); + } + } + + @Override protected void black(Action action) { + visitAction(action); + for (Artifact input : action.getInputs()) { + if (shouldVisit(input)) { + visitWhiteNode(input); + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java new file mode 100644 index 0000000000..c3705915ba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInput.java @@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * Represents an input file to a build action, with an appropriate relative path and digest + * value. + * + * <p>Artifact is the only notable implementer of the interface, but the interface remains + * because 1) some Google specific rules ship files that could be Artifacts to remote execution + * by instantiating ad-hoc derived classes of ActionInput. 2) historically, Google C++ rules + * allow underspecified C++ builds. For that case, we have extra logic to guess the undeclared + * header inclusions (eg. computed inclusions). The extra logic lives in a file that is not + * needed for remote execution, but is a dependency, and it is inserted as a non-Artifact + * ActionInput. + * + * <p>ActionInput is used as a cache "key" for ActionInputFileCache: for Artifacts, the + * digest/size is already stored in Artifact, but for non-artifacts, we use getExecPathString + * to find this data in a filesystem related cache. + */ +public interface ActionInput { + + /** + * @return the relative path to the input file. + */ + public String getExecPathString(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java new file mode 100644 index 0000000000..b45e9cd65a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputFileCache.java @@ -0,0 +1,77 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.IOException; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * The interface for Action inputs metadata (Digest and size). + * + * NOTE: Implementations must be thread safe. + */ +@ThreadSafe +public interface ActionInputFileCache { + /** + * Returns digest for the given artifact. This digest is current as of some time t >= the start of + * the present build. If the artifact is an output of an action that already executed at time p, + * then t >= p. Aside from these properties, t can be any value and may vary arbitrarily across + * calls. + * + * @param input the input to retrieve the digest for + * @return the artifact's digest or null if digest cannot be obtained (due to artifact + * non-existence, lookup errors, or any other reason) + * + * @throws DigestOfDirectoryException in case {@code input} is a directory. + * @throws IOException If the file cannot be digested. + * + */ + @Nullable + ByteString getDigest(ActionInput input) throws IOException; + + /** + * Retrieve the size of the file at the given path. Will usually return 0 on failure instead of + * throwing an IOException. Returns 0 for files inaccessible to user, but available to the + * execution environment. + * + * @param input the input. + * @return the file size in bytes. + * @throws IOException on failure. + */ + long getSizeInBytes(ActionInput input) throws IOException; + + /** + * Checks if the file is available locally, based on the assumption that previous operations on + * the ActionInputFileCache would have created a cache entry for it. + * + * @param digest the digest to lookup. + * @return true if the specified digest is backed by a locally-readable file, false otherwise + */ + boolean contentsAvailableLocally(ByteString digest); + + /** + * Concrete subclasses must implement this to provide a mapping from digest to file path, + * based on files previously seen as inputs. + * + * @param digest the digest. + * @return a File path. + */ + @Nullable + File getFileFromDigest(ByteString digest) throws IOException; +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java new file mode 100644 index 0000000000..0fed9283b8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java @@ -0,0 +1,160 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Helper utility to create ActionInput instances. + */ +public final class ActionInputHelper { + private ActionInputHelper() { + } + + @VisibleForTesting + public static Artifact.MiddlemanExpander actionGraphMiddlemanExpander( + final ActionGraph actionGraph) { + return new Artifact.MiddlemanExpander() { + @Override + public void expand(Artifact mm, Collection<? super Artifact> output) { + // Skyframe is stricter in that it checks that "mm" is a input of the action, because + // it cannot expand arbitrary middlemen without access to a global action graph. + // We could check this constraint here too, but it seems unnecessary. This code is + // going away anyway. + Preconditions.checkArgument(mm.isMiddlemanArtifact(), + "%s is not a middleman artifact", mm); + Action middlemanAction = actionGraph.getGeneratingAction(mm); + Preconditions.checkState(middlemanAction != null, mm); + // TODO(bazel-team): Consider expanding recursively or throwing an exception here. + // Most likely, this code will cause silent errors if we ever have a middleman that + // contains a middleman. + if (middlemanAction.getActionType() == Action.MiddlemanType.AGGREGATING_MIDDLEMAN) { + Artifact.addNonMiddlemanArtifacts(middlemanAction.getInputs(), output, + Functions.<Artifact>identity()); + } + + } + }; + } + + /** + * Most ActionInputs are created and never used again. On the off chance that one is, however, we + * implement equality via path comparison. Since file caches are keyed by ActionInput, equality + * checking does come up. + */ + private static class BasicActionInput implements ActionInput { + private final String path; + public BasicActionInput(String path) { + this.path = Preconditions.checkNotNull(path); + } + + @Override + public String getExecPathString() { + return path; + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null) { + return false; + } + if (!this.getClass().equals(other.getClass())) { + return false; + } + return this.path.equals(((BasicActionInput) other).path); + } + + @Override + public String toString() { + return "BasicActionInput: " + path; + } + } + + /** + * Creates an ActionInput with just the given relative path and no digest. + * + * @param path the relative path of the input. + * @return a ActionInput. + */ + public static ActionInput fromPath(String path) { + return new BasicActionInput(path); + } + + private static final Function<String, ActionInput> FROM_PATH = + new Function<String, ActionInput>() { + @Override + public ActionInput apply(String path) { + return fromPath(path); + } + }; + + /** + * Creates a sequence of {@link ActionInput}s from a sequence of string paths. + */ + public static Collection<ActionInput> fromPaths(Collection<String> paths) { + return Collections2.transform(paths, FROM_PATH); + } + + /** + * Expands middleman artifacts in a sequence of {@link ActionInput}s. + * + * <p>Non-middleman artifacts are returned untouched. + */ + public static List<ActionInput> expandMiddlemen(Iterable<? extends ActionInput> inputs, + Artifact.MiddlemanExpander middlemanExpander) { + + List<ActionInput> result = new ArrayList<>(); + List<Artifact> containedArtifacts = new ArrayList<>(); + for (ActionInput input : inputs) { + if (!(input instanceof Artifact)) { + result.add(input); + continue; + } + containedArtifacts.add((Artifact) input); + } + Artifact.addExpandedArtifacts(containedArtifacts, result, middlemanExpander); + return result; + } + + /** Formatter for execPath String output. Public because Artifact uses it directly. */ + public static final Function<ActionInput, String> EXEC_PATH_STRING_FORMATTER = + new Function<ActionInput, String>() { + @Override + public String apply(ActionInput input) { + return input.getExecPathString(); + } + }; + + public static Iterable<String> toExecPaths(Iterable<? extends ActionInput> artifacts) { + return Iterables.transform(artifacts, EXEC_PATH_STRING_FORMATTER); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java b/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java new file mode 100644 index 0000000000..2c80e8a92b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionLogBufferPathGenerator.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * A source for generating unique action log paths. + */ +public final class ActionLogBufferPathGenerator { + + private final AtomicInteger actionCounter = new AtomicInteger(); + + private final Path actionOutputRoot; + + public ActionLogBufferPathGenerator(Path actionOutputRoot) { + this.actionOutputRoot = actionOutputRoot; + } + + /** + * Generates a unique filename for an action to store its output. + */ + public FileOutErr generate() { + int actionId = actionCounter.incrementAndGet(); + return new FileOutErr(actionOutputRoot.getRelative("stdout-" + actionId), + actionOutputRoot.getRelative("stderr-" + actionId)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java b/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java new file mode 100644 index 0000000000..3569f07a8b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionMetadata.java @@ -0,0 +1,217 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import javax.annotation.Nullable; + +/** + * Side-effect free query methods for information about an {@link Action}. + * + * <p>This method is intended for use in situations when the intention is to pass around information + * about an action without allowing actual execution of the action. + * + * <p>The split between {@link Action} and {@link ActionMetadata} is somewhat arbitrary, other than + * that all methods with side effects must belong to the former. + */ +public interface ActionMetadata { + /** + * If this executable can supply verbose information, returns a string that can be used as a + * progress message while this executable is running. A return value of {@code null} indicates no + * message should be reported. + */ + @Nullable + public String getProgressMessage(); + + /** + * Returns the owner of this executable if this executable can supply verbose information. This is + * typically the rule that constructed it; see ActionOwner class comment for details. Returns + * {@code null} if no owner can be determined. + * + * <p>If this executable does not supply verbose information, this function may throw an + * IllegalStateException. + */ + public ActionOwner getOwner(); + + /** + * Returns a mnemonic (string constant) for this kind of action; written into + * the master log so that the appropriate parser can be invoked for the output + * of the action. Effectively a public method as the value is used by the + * extra_action feature to match actions. + */ + String getMnemonic(); + + /** + * Returns a pretty string representation of this action, suitable for use in + * progress messages or error messages. + */ + String prettyPrint(); + + /** + * Returns a string that can be used to describe the execution strategy. + * For example, "local". + * + * May return null if the action chooses to update its strategy + * locality "manually", via ActionLocalityMessage. + * + * @param executor the application-specific value passed to the + * executor parameter of the top-level call to + * Builder.buildArtifacts(). + */ + public String describeStrategy(Executor executor); + + /** + * Returns true iff the getInputs set is known to be complete. + * + * <p>For most Actions, this always returns true, but in some cases (e.g. C++ compilation), inputs + * are dynamically discovered from the previous execution of the Action, and so before the initial + * execution, this method will return false in those cases. + * + * <p>Any builder <em>must</em> unconditionally execute an Action for which inputsKnown() returns + * false, regardless of all other inferences made by its dependency analysis. In addition, all + * prerequisites mentioned in the (possibly incomplete) value returned by getInputs must also be + * built first, as usual. + */ + @ThreadSafe + boolean inputsKnown(); + + /** + * Returns true iff inputsKnown() may ever return false. + */ + @ThreadSafe + boolean discoversInputs(); + + /** + * Returns the input Artifacts that this Action depends upon. May be empty. + * + * <p>For subclasses overriding getInputs(), if getInputs() could return different values in the + * lifetime of an object, {@link #getInputCount()} must also be overridden. + * + * <p>During execution, the {@link Iterable} returned by {@code getInputs} <em>must not</em> be + * concurrently modified before the value is fully read in {@code JavaDistributorDriver#exec} (via + * the {@code Iterable<ActionInput>} argument there). Violating this would require somewhat + * pathological behavior by the {@link Action}, since it would have to modify its inputs, as a + * list, say, without reassigning them. This should never happen with any Action subclassing + * AbstractAction, since AbstractAction's implementation of getInputs() returns an immutable + * iterable. + */ + Iterable<Artifact> getInputs(); + + /** + * Returns the number of input Artifacts that this Action depends upon. + * + * <p>Must be consistent with {@link #getInputs()}. + */ + int getInputCount(); + + /** + * Returns the (unordered, immutable) set of output Artifacts that + * this action generates. (It would not make sense for this to be empty.) + */ + ImmutableSet<Artifact> getOutputs(); + + /** + * Returns the "primary" input of this action, if applicable. + * + * <p>For example, a C++ compile action would return the .cc file which is being compiled, + * irrespective of the other inputs. + * + * <p>May return null. + */ + Artifact getPrimaryInput(); + + /** + * Returns the "primary" output of this action. + * + * <p>For example, the linked library would be the primary output of a LinkAction. + * + * <p>Never returns null. + */ + Artifact getPrimaryOutput(); + + /** + * Returns an iterable of input Artifacts that MUST exist prior to executing an action. In other + * words, in case when action is scheduled for execution, builder will ensure that all artifacts + * returned by this method are present in the filesystem (artifact.getPath().exists() is true) or + * action execution will be aborted with an error that input file does not exist. While in + * majority of cases this method will return all action inputs, for some actions (e.g. + * CppCompileAction) it can return a subset of inputs because that not all action inputs might be + * mandatory for action execution to succeed (e.g. header files retrieved from *.d file from the + * previous build). + */ + Iterable<Artifact> getMandatoryInputs(); + + /** + * <p>Returns a string encoding all of the significant behaviour of this + * Action that might affect the output. The general contract of + * <code>getKey</code> is this: if the work to be performed by the + * execution of this action changes, the key must change. </p> + * + * <p>As a corollary, the build system is free to omit the execution of an + * Action <code>a1</code> if (a) at some time in the past, it has already + * executed an Action <code>a0</code> with the same key as + * <code>a1</code>, and (b) the names and contents of the input files listed + * by <code>a1.getInputs()</code> are identical to the names and contents of + * the files listed by <code>a0.getInputs()</code>. </p> + * + * <p>Examples of changes that should affect the key are: + * <ul> + * <li>Changes to the BUILD file that materially affect the rule which gave + * rise to this Action.</li> + * + * <li>Changes to the command-line options, environment, or other global + * configuration resources which affect the behaviour of this kind of Action + * (other than changes to the names of the input/output files, which are + * handled externally).</li> + * + * <li>An upgrade to the build tools which changes the program logic of this + * kind of Action (typically this is achieved by incorporating a UUID into + * the key, which is changed each time the program logic of this action + * changes).</li> + * + * </ul></p> + */ + String getKey(); + + /** + * Returns a human-readable description of the inputs to {@link #getKey()}. + * Used in the output from '--explain', and in error messages for + * '--check_up_to_date' and '--check_tests_up_to_date'. + * May return null, meaning no extra information is available. + * + * <p>If the return value is non-null, for consistency it should be a multiline message of the + * form: + * <pre> + * <var>Summary</var> + * <var>Fieldname</var>: <var>value</var> + * <var>Fieldname</var>: <var>value</var> + * ... + * </pre> + * where each line after the first one is intended two spaces, and where any fields that might + * contain newlines or other funny characters are escaped using {@link + * com.google.devtools.build.lib.shell.ShellUtils#shellEscape}. + * For example: + * <pre> + * Compiling foo.cc + * Command: /usr/bin/gcc + * Argument: '-c' + * Argument: foo.cc + * Argument: '-o' + * Argument: foo.o + * </pre> + */ + @Nullable String describeKey(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java new file mode 100644 index 0000000000..855d97a932 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionMiddlemanEvent.java @@ -0,0 +1,54 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; + +/** + * This event is fired during the build, when a middleman action is executed. Middleman actions + * don't usually do any computation but we need them in the critical path because they depend on + * other actions. + */ +public class ActionMiddlemanEvent { + + private final Action action; + private final long nanoTimeStart; + + /** + * Create an event for action that has been started. + * + * @param action the middleman action. + * @param nanoTimeStart the time when the action was started. This allow us to record more + * accurately the time spent by the middleman action, since even for middleman actions we execute + * some. + */ + public ActionMiddlemanEvent(Action action, long nanoTimeStart) { + Preconditions.checkArgument(action.getActionType().isMiddleman(), + "Only middleman actions should be passed: %s", action); + this.action = action; + this.nanoTimeStart = nanoTimeStart; + } + + /** + * Returns the associated action. + */ + public Action getAction() { + return action; + } + + public long getNanoTimeStart() { + return nanoTimeStart; + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java new file mode 100644 index 0000000000..ab59f635bf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionOwner.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.syntax.Label; + +/** + * The owner of an action is responsible for reporting conflicts in the action + * graph (two actions attempting to generate the same artifact). + * + * Typically an action's owner is the RuleConfiguredTarget instance responsible + * for creating it, but to avoid coupling between the view and actions + * packages, the RuleConfiguredTarget is hidden behind this interface, which + * exposes only the error reporting functionality. + */ +public interface ActionOwner { + + /** + * Returns the location of this ActionOwner, if any; null otherwise. + */ + Location getLocation(); + + /** + * Returns the label for this ActionOwner, if any; null otherwise. + */ + Label getLabel(); + + /** + * Returns the name of the configuration of the action owner. + */ + String getConfigurationName(); + + /** + * Returns the configuration's mnemonic. + */ + String getConfigurationMnemonic(); + + /** + * Returns the short cache key for the configuration of the action owner. + * + * <p>Special action owners that are not targets can return any string here as long as it is + * constant. If the configuration is null, this should return "null". + * + * <p>These requirements exist so that {@link ActionOwner} instances are consistent with + * {@code BuildView.ActionOwnerIdentity(ConfiguredTargetValue)}. + */ + String getConfigurationShortCacheKey(); + + /** + * Returns the target kind (rule class name) for this ActionOwner, if any; null otherwise. + */ + String getTargetKind(); + + /** + * Returns additional information that should be displayed in progress messages, or {@code null} + * if nothing should be added. + */ + String getAdditionalProgressInfo(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java b/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java new file mode 100644 index 0000000000..db7e35044e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionRegistry.java @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; + +/** + * An interface for registering actions. + */ +public interface ActionRegistry { + /** + * This method notifies the registry new actions. + */ + void registerAction(Action... actions); + + /** + * Get the (Label and BuildConfiguration) of the ConfiguredTarget ultimately responsible for all + * these actions. + */ + ArtifactOwner getOwner(); + + /** + * An action registry that does exactly nothing. + */ + @VisibleForTesting + public static final ActionRegistry NOP = new ActionRegistry() { + @Override + public void registerAction(Action... actions) {} + + @Override + public ArtifactOwner getOwner() { + return ArtifactOwner.NULL_OWNER; + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java new file mode 100644 index 0000000000..a2a978f2c5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionStartedEvent.java @@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * This event is fired during the build, when an action is started. + */ +public class ActionStartedEvent { + private final Action action; + private final long nanoTimeStart; + + /** + * Create an event for action that has been started. + * + * @param action the started action. + * @param nanoTimeStart the time when the action was started. This allow us to + * record more accurately the time spend by the action, since we execute some code before + * deciding if we execute the action or not. + */ + public ActionStartedEvent(Action action, long nanoTimeStart) { + this.action = action; + this.nanoTimeStart = nanoTimeStart; + } + + /** + * Returns the associated action. + */ + public Action getAction() { + return action; + } + + public long getNanoTimeStart() { + return nanoTimeStart; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java new file mode 100644 index 0000000000..c932a9e809 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionStatusMessage.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * A message used to update in-flight action status. An action's status may change low down in the + * execution stack (for instance, from running remotely to running locally), so this message can be + * used to notify any interested parties. + */ +public class ActionStatusMessage { + private final ActionMetadata action; + private final String message; + public static final String PREPARING = "Preparing"; + + public ActionStatusMessage(ActionMetadata action, String message) { + this.action = action; + this.message = message; + } + + public ActionMetadata getActionMetadata() { + return action; + } + + public String getMessage() { + return message; + } + + /** Returns whether the message needs further interpolation of a 'strategy' when printed. */ + public boolean needsStrategy() { + return false; + } + + /** Creates "Analyzing" status message. */ + public static ActionStatusMessage analysisStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, "Analyzing"); + } + + /** Creates "Preparing" status message. */ + public static ActionStatusMessage preparingStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, PREPARING); + } + + /** Creates "Scheduling" status message. */ + public static ActionStatusMessage schedulingStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, "Scheduling"); + } + + /** Creates "Running (%s)" status message (needs strategy interpolated). */ + public static ActionStatusMessage runningStrategy(ActionMetadata action) { + return new ActionStatusMessage(action, "Running (%s)") { + @Override + public boolean needsStrategy() { + return true; + } + }; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/Actions.java b/src/main/java/com/google/devtools/build/lib/actions/Actions.java new file mode 100644 index 0000000000..fb2b834785 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Actions.java @@ -0,0 +1,79 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.Iterables; +import com.google.common.escape.Escaper; +import com.google.common.escape.Escapers; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Helper class for actions. + */ +@ThreadSafe +public final class Actions { + private static final Escaper PATH_ESCAPER = Escapers.builder() + .addEscape('_', "_U") + .addEscape('/', "_S") + .addEscape('\\', "_B") + .addEscape(':', "_C") + .build(); + + /** + * Checks if the two actions are equivalent. This method exists to support sharing actions between + * configured targets for cases where there is no canonical target that could own the action. In + * the action graph construction this case shows up as two actions generating the same output + * file. + * + * <p>This method implements an equivalence relationship across actions, based on the action + * class, the key, and the list of inputs and outputs. + */ + public static boolean canBeShared(Action a, Action b) { + if (!a.getMnemonic().equals(b.getMnemonic())) { + return false; + } + if (!a.getKey().equals(b.getKey())) { + return false; + } + // Don't bother to check input and output counts first; the expected result for these tests is + // to always be true (i.e., that this method returns true). + if (!Iterables.elementsEqual(a.getMandatoryInputs(), b.getMandatoryInputs())) { + return false; + } + if (!Iterables.elementsEqual(a.getOutputs(), b.getOutputs())) { + return false; + } + return true; + } + + /** + * Returns the escaped name for a given relative path as a string. This takes + * a short relative path and turns it into a string suitable for use as a + * filename. Invalid filename characters are escaped with an '_' + a single + * character token. + */ + public static String escapedPath(String path) { + return PATH_ESCAPER.escape(path); + } + + /** + * Returns a string that is usable as a unique path component for a label. It is guaranteed + * that no other label maps to this string. + */ + public static String escapeLabel(Label label) { + return PATH_ESCAPER.escape(label.getPackageName() + ":" + label.getName()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java b/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java new file mode 100644 index 0000000000..4ce258b5b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/AlreadyReportedActionExecutionException.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * This wrapper exception is used as a marker class to already reported errors. Errors are reported + * at {@code AbstractBuilder.executeActionTask()} method in case of the builder not aborting in case + * of exceptions (For example keepgoing). + * + * <p>Then In upper levels we wrap catch the exception and throw a BuildFailedException + * unconditionally, that is caught and shown as error in AbstractBuildCommand (because the message + * of the exception is !=null). + * + * With this exception we detect that the error was already shown and we wrap it in a + * BuildFailedException without message. + */ +public class AlreadyReportedActionExecutionException extends ActionExecutionException { + + public AlreadyReportedActionExecutionException(ActionExecutionException cause) { + super(cause.getMessage(), cause.getCause(), cause.getAction(), cause.getRootCauses(), + cause.isCatastrophe()); + } + + @Override + public boolean showError() { + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java new file mode 100644 index 0000000000..2f2272b378 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java @@ -0,0 +1,654 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * An Artifact represents a file used by the build system, whether it's a source + * file or a derived (output) file. Not all Artifacts have a corresponding + * FileTarget object in the <code>build.packages</code> API: for example, + * low-level intermediaries internal to a given rule, such as a Java class files + * or C++ object files. However all FileTargets have a corresponding Artifact. + * + * <p>In any given call to Builder#buildArtifacts(), no two Artifacts in the + * action graph may refer to the same path. + * + * <p>Artifacts generally fall into two classifications, source and derived, but + * there exist a few other cases that are fuzzy and difficult to classify. The + * following cases exist: + * <ul> + * <li>Well-formed source Artifacts will have null generating Actions and a root + * that is orthogonal to execRoot. (With the root coming from the package path.) + * <li>Well-formed derived Artifacts will have non-null generating Actions, and + * a root that is below execRoot. + * <li>Symlinked include source Artifacts under the output/include tree will + * appear to be derived artifacts with null generating Actions. + * <li>Some derived Artifacts, mostly in the genfiles tree and mostly discovered + * during include validation, will also have null generating Actions. + * </ul> + * + * <p>This class is "theoretically" final; it should not be subclassed except by + * {@link SpecialArtifact}. + */ +@Immutable +@SkylarkModule(name = "File", + doc = "This type represents a file used by the build system. It can be " + + "either a source file or a derived file produced by a rule.") +public class Artifact implements FileType.HasFilename, Comparable<Artifact>, ActionInput { + + /** An object that can expand middleman artifacts. */ + public interface MiddlemanExpander { + + /** + * Expands the middleman artifact "mm", and populates "output" with the result. + * + * <p>{@code mm.isMiddlemanArtifact()} must be true. Only aggregating middlemen are expanded. + */ + void expand(Artifact mm, Collection<? super Artifact> output); + } + + public static final ImmutableList<Artifact> NO_ARTIFACTS = ImmutableList.of(); + + /** + * A Predicate that evaluates to true if the Artifact is not a middleman artifact. + */ + public static final Predicate<Artifact> MIDDLEMAN_FILTER = new Predicate<Artifact>() { + @Override + public boolean apply(Artifact input) { + return !input.isMiddlemanArtifact(); + } + }; + + private final Path path; + private final Root root; + private final PathFragment execPath; + private final PathFragment rootRelativePath; + // Non-final only for use when dealing with deserialized artifacts. + private ArtifactOwner owner; + + /** + * Constructs an artifact for the specified path, root and execPath. The root must be an ancestor + * of path, and execPath must be a non-absolute tail of path. Outside of testing, this method + * should only be called by ArtifactFactory. The ArtifactOwner may be null. + * + * <p>In a source Artifact, the path tail after the root will be identical to the execPath, but + * the root will be orthogonal to execRoot. + * <pre> + * [path] == [/root/][execPath] + * </pre> + * + * <p>In a derived Artifact, the execPath will overlap with part of the root, which in turn will + * be below of the execRoot. + * <pre> + * [path] == [/root][pathTail] == [/execRoot][execPath] == [/execRoot][rootPrefix][pathTail] + * <pre> + */ + @VisibleForTesting + public Artifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner) { + if (root == null || !path.startsWith(root.getPath())) { + throw new IllegalArgumentException(root + ": illegal root for " + path); + } + if (execPath == null || execPath.isAbsolute() || !path.asFragment().endsWith(execPath)) { + throw new IllegalArgumentException(execPath + ": illegal execPath for " + path); + } + this.path = path; + this.root = root; + this.execPath = execPath; + // These two lines establish the invariant that + // execPath == rootRelativePath <=> execPath.equals(rootRelativePath) + // This is important for isSourceArtifact. + PathFragment rootRel = path.relativeTo(root.getPath()); + if (!execPath.endsWith(rootRel)) { + throw new IllegalArgumentException(execPath + ": illegal execPath doesn't end with " + + rootRel + " at " + path + " with root " + root); + } + this.rootRelativePath = rootRel.equals(execPath) ? execPath : rootRel; + this.owner = Preconditions.checkNotNull(owner, path); + } + + /** + * Constructs an artifact for the specified path, root and execPath. The root must be an ancestor + * of path, and execPath must be a non-absolute tail of path. Should only be called for testing. + * + * <p>In a source Artifact, the path tail after the root will be identical to the execPath, but + * the root will be orthogonal to execRoot. + * <pre> + * [path] == [/root/][execPath] + * </pre> + * + * <p>In a derived Artifact, the execPath will overlap with part of the root, which in turn will + * be below of the execRoot. + * <pre> + * [path] == [/root][pathTail] == [/execRoot][execPath] == [/execRoot][rootPrefix][pathTail] + * <pre> + */ + @VisibleForTesting + public Artifact(Path path, Root root, PathFragment execPath) { + this(path, root, execPath, ArtifactOwner.NULL_OWNER); + } + + /** + * Constructs a source or derived Artifact for the specified path and specified root. The root + * must be an ancestor of the path. + */ + @VisibleForTesting // Only exists for testing. + public Artifact(Path path, Root root) { + this(path, root, root.getExecPath().getRelative(path.relativeTo(root.getPath())), + ArtifactOwner.NULL_OWNER); + } + + /** + * Constructs a source or derived Artifact for the specified root-relative path and root. + */ + @VisibleForTesting // Only exists for testing. + public Artifact(PathFragment rootRelativePath, Root root) { + this(root.getPath().getRelative(rootRelativePath), root, + root.getExecPath().getRelative(rootRelativePath), ArtifactOwner.NULL_OWNER); + } + + /** + * Returns the location of this Artifact on the filesystem. + */ + public final Path getPath() { + return path; + } + + /** + * Returns the base file name of this artifact. + */ + @Override + public final String getFilename() { + return getExecPath().getBaseName(); + } + + /** + * Returns the artifact owner. May be null. + */ + @Nullable public final Label getOwner() { + return owner.getLabel(); + } + + /** + * Get the {@code LabelAndConfiguration} of the {@code ConfiguredTarget} that owns this artifact, + * if it was set. Otherwise, this should be a dummy value -- either {@link + * ArtifactOwner#NULL_OWNER} or a dummy owner set in tests. Such a dummy value should only occur + * for source artifacts if created without specifying the owner, or for special derived artifacts, + * such as target completion middleman artifacts, build info artifacts, and the like. + * + * When deserializing artifacts we end up with a dummy owner. In that case, it must be set using + * {@link #setArtifactOwner} before this method is called. + */ + public final ArtifactOwner getArtifactOwner() { + Preconditions.checkState(owner != DESERIALIZED_MARKER_OWNER, this); + return owner; + } + + /** + * Sets the artifact owner of this artifact. Should only be called for artifacts that were created + * through deserialization, and so their owner was unknown at the time of creation. + */ + public final void setArtifactOwner(ArtifactOwner owner) { + if (this.owner == DESERIALIZED_MARKER_OWNER) { + // We tolerate multiple calls of this method to accommodate shared actions. + this.owner = Preconditions.checkNotNull(owner, this); + } + } + + /** + * Returns the root beneath which this Artifact resides, if any. This may be one of the + * package-path entries (for source Artifacts), or one of the bin, genfiles or includes dirs + * (for derived Artifacts). It will always be an ancestor of getPath(). + */ + public final Root getRoot() { + return root; + } + + /** + * Returns the exec path of this Artifact. The exec path is a relative path + * that is suitable for accessing this artifact relative to the execution + * directory for this build. + */ + public final PathFragment getExecPath() { + return execPath; + } + + /** + * Returns true iff this is a source Artifact as determined by its path and + * root relationships. Note that this will report all Artifacts in the output + * tree, including in the include symlink tree, as non-source. + */ + public final boolean isSourceArtifact() { + return execPath == rootRelativePath; + } + + /** + * Returns true iff this is a middleman Artifact as determined by its root. + */ + public final boolean isMiddlemanArtifact() { + return getRoot().isMiddlemanRoot(); + } + + /** + * Returns whether the artifact represents a Fileset. + */ + public boolean isFileset() { + return false; + } + + /** + * Returns true iff metadata cache must return constant metadata for the + * given artifact. + */ + public boolean isConstantMetadata() { + return false; + } + + /** + * Special artifact types. + * + * @see SpecialArtifact + */ + static enum SpecialArtifactType { + FILESET, + CONSTANT_METADATA, + } + + /** + * A special kind of artifact that either is a fileset or needs special metadata caching behavior. + * + * <p>We subclass {@link Artifact} instead of storing the special attributes inside in order + * to save memory. The proportion of artifacts that are special is very small, and by not having + * to keep around the attribute for the rest we save some memory. + */ + @Immutable + @VisibleForTesting + public static final class SpecialArtifact extends Artifact { + private final SpecialArtifactType type; + + SpecialArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner, + SpecialArtifactType type) { + super(path, root, execPath, owner); + this.type = type; + } + + @Override + public final boolean isFileset() { + return type == SpecialArtifactType.FILESET; + } + + @Override + public boolean isConstantMetadata() { + return type == SpecialArtifactType.CONSTANT_METADATA; + } + } + + /** + * Returns the relative path to this artifact relative to its root. (Useful + * when deriving output filenames from input files, etc.) + */ + public final PathFragment getRootRelativePath() { + return rootRelativePath; + } + + /** + * Returns this.getExecPath().getPathString(). + */ + @Override + @SkylarkCallable(name = "path", structField = true, + doc = "The execution path of this file, relative to the execution directory.") + public final String getExecPathString() { + return getExecPath().getPathString(); + } + + @SkylarkCallable(name = "short_path", structField = true, + doc = "The path of this file relative to its root.") + public final String getRootRelativePathString() { + return getRootRelativePath().getPathString(); + } + + /** + * Returns a pretty string representation of the path denoted by this artifact, suitable for use + * in user error messages. Artifacts beneath a root will be printed relative to that root; other + * artifacts will be printed as an absolute path. + * + * <p>(The toString method is intended for developer messages since its more informative.) + */ + public final String prettyPrint() { + // toDetailString would probably be more useful to users, but lots of tests rely on the + // current values. + return rootRelativePath.toString(); + } + + @Override + public final boolean equals(Object other) { + if (!(other instanceof Artifact)) { + return false; + } + // We don't bother to check root in the equivalence relation, because we + // assume that 'root' is an ancestor of 'path', and that all possible roots + // are disjoint, so unless things are really screwed up, it's ok. + Artifact that = (Artifact) other; + return this.path.equals(that.path); + } + + @Override + public final int compareTo(Artifact o) { + // The artifact factory ensures that there is a unique artifact for a given path. + return this.path.compareTo(o.path); + } + + @Override + public final int hashCode() { + return path.hashCode(); + } + + @Override + public final String toString() { + return "Artifact:" + toDetailString(); + } + + /** + * Returns the root-part of a given path by trimming off the end specified by + * a given tail. Assumes that the tail is known to match, and simply relies on + * the segment lengths. + */ + private static PathFragment trimTail(PathFragment path, PathFragment tail) { + return path.subFragment(0, path.segmentCount() - tail.segmentCount()); + } + + /** + * Returns a string representing the complete artifact path information. + */ + public final String toDetailString() { + if (isSourceArtifact()) { + // Source Artifact: relPath == execPath, & real path is not under execRoot + return "[" + root + "]" + rootRelativePath; + } else { + // Derived Artifact: path and root are under execRoot + PathFragment execRoot = trimTail(path.asFragment(), execPath); + return "[[" + execRoot + "]" + root.getPath().asFragment().relativeTo(execRoot) + "]" + + rootRelativePath; + } + } + + /** + * Serializes this artifact to a string that has enough data to reconstruct the artifact. + */ + public final String serializeToString() { + // In theory, it should be enough to serialize execPath and rootRelativePath (which is a suffix + // of execPath). However, in practice there is code around that uses other attributes which + // needs cleaning up. + String result = execPath + " /" + rootRelativePath.toString().length(); + if (getOwner() != null) { + result += " " + getOwner(); + } + return result; + } + + //--------------------------------------------------------------------------- + // Static methods to assist in working with Artifacts + + /** + * Formatter for execPath PathFragment output. + */ + private static final Function<Artifact, PathFragment> EXEC_PATH_FORMATTER = + new Function<Artifact, PathFragment>() { + @Override + public PathFragment apply(Artifact input) { + return input.getExecPath(); + } + }; + + private static final Function<Artifact, String> ROOT_RELATIVE_PATH_STRING = + new Function<Artifact, String>() { + @Override + public String apply(Artifact artifact) { + return artifact.getRootRelativePath().getPathString(); + } + }; + + /** + * Converts a collection of artifacts into execution-time path strings, and + * adds those to a given collection. Middleman artifacts are ignored by this + * method. + */ + public static void addExecPaths(Iterable<Artifact> artifacts, Collection<String> output) { + addNonMiddlemanArtifacts(artifacts, output, ActionInputHelper.EXEC_PATH_STRING_FORMATTER); + } + + /** + * Converts a collection of artifacts into the outputs computed by + * outputFormatter and adds them to a given collection. Middleman artifacts + * are ignored. + */ + static <E> void addNonMiddlemanArtifacts(Iterable<Artifact> artifacts, + Collection<? super E> output, Function<? super Artifact, E> outputFormatter) { + for (Artifact artifact : artifacts) { + if (MIDDLEMAN_FILTER.apply(artifact)) { + output.add(outputFormatter.apply(artifact)); + } + } + } + + /** + * Lazily converts artifacts into root-relative path strings. Middleman artifacts are ignored by + * this method. + */ + public static Iterable<String> toRootRelativePaths(Iterable<Artifact> artifacts) { + return Iterables.transform( + Iterables.filter(artifacts, MIDDLEMAN_FILTER), + ROOT_RELATIVE_PATH_STRING); + } + + /** + * Lazily converts artifacts into execution-time path strings. Middleman artifacts are ignored by + * this method. + */ + public static Iterable<String> toExecPaths(Iterable<Artifact> artifacts) { + return ActionInputHelper.toExecPaths(Iterables.filter(artifacts, MIDDLEMAN_FILTER)); + } + + /** + * Converts a collection of artifacts into execution-time path strings, and + * returns those as an immutable list. Middleman artifacts are ignored by this method. + */ + public static List<String> asExecPaths(Iterable<Artifact> artifacts) { + return ImmutableList.copyOf(toExecPaths(artifacts)); + } + + /** + * Renders a collection of artifacts as execution-time paths and joins + * them into a single string. Middleman artifacts are ignored by this method. + */ + public static String joinExecPaths(String delimiter, Iterable<Artifact> artifacts) { + return Joiner.on(delimiter).join(toExecPaths(artifacts)); + } + + /** + * Renders a collection of artifacts as root-relative paths and joins + * them into a single string. Middleman artifacts are ignored by this method. + */ + public static String joinRootRelativePaths(String delimiter, Iterable<Artifact> artifacts) { + return Joiner.on(delimiter).join(toRootRelativePaths(artifacts)); + } + + /** + * Adds a collection of artifacts to a given collection, with + * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions expanded once. + */ + public static void addExpandedArtifacts(Iterable<Artifact> artifacts, + Collection<? super Artifact> output, MiddlemanExpander middlemanExpander) { + addExpandedArtifacts(artifacts, output, Functions.<Artifact>identity(), middlemanExpander); + } + + /** + * Converts a collection of artifacts into execution-time path strings, and + * adds those to a given collection. Middleman artifacts for + * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions are expanded + * once. + */ + @VisibleForTesting + public static void addExpandedExecPathStrings(Iterable<Artifact> artifacts, + Collection<String> output, + MiddlemanExpander middlemanExpander) { + addExpandedArtifacts(artifacts, output, ActionInputHelper.EXEC_PATH_STRING_FORMATTER, + middlemanExpander); + } + + /** + * Converts a collection of artifacts into execution-time path fragments, and + * adds those to a given collection. Middleman artifacts for + * {@link MiddlemanType#AGGREGATING_MIDDLEMAN} middleman actions are expanded + * once. + */ + public static void addExpandedExecPaths(Iterable<Artifact> artifacts, + Collection<PathFragment> output, MiddlemanExpander middlemanExpander) { + addExpandedArtifacts(artifacts, output, EXEC_PATH_FORMATTER, middlemanExpander); + } + + /** + * Converts a collection of artifacts into the outputs computed by + * outputFormatter and adds them to a given collection. Middleman artifacts + * are expanded once. + */ + private static <E> void addExpandedArtifacts(Iterable<Artifact> artifacts, + Collection<? super E> output, + Function<? super Artifact, E> outputFormatter, + MiddlemanExpander middlemanExpander) { + for (Artifact artifact : artifacts) { + if (artifact.isMiddlemanArtifact()) { + expandMiddlemanArtifact(artifact, output, outputFormatter, middlemanExpander); + } else { + output.add(outputFormatter.apply(artifact)); + } + } + } + + private static <E> void expandMiddlemanArtifact(Artifact middleman, + Collection<? super E> output, + Function<? super Artifact, E> outputFormatter, + MiddlemanExpander middlemanExpander) { + Preconditions.checkArgument(middleman.isMiddlemanArtifact()); + List<Artifact> artifacts = new ArrayList<>(); + middlemanExpander.expand(middleman, artifacts); + for (Artifact artifact : artifacts) { + output.add(outputFormatter.apply(artifact)); + } + } + + /** + * Converts a collection of artifacts into execution-time path strings, and + * returns those as a list. Middleman artifacts are expanded once. The + * returned list is mutable. + */ + public static List<String> asExpandedExecPathStrings(Iterable<Artifact> artifacts, + MiddlemanExpander middlemanExpander) { + List<String> result = new ArrayList<>(); + addExpandedExecPathStrings(artifacts, result, middlemanExpander); + return result; + } + + /** + * Converts a collection of artifacts into execution-time path fragments, and + * returns those as a list. Middleman artifacts are expanded once. The + * returned list is mutable. + */ + public static List<PathFragment> asExpandedExecPaths(Iterable<Artifact> artifacts, + MiddlemanExpander middlemanExpander) { + List<PathFragment> result = new ArrayList<>(); + addExpandedExecPaths(artifacts, result, middlemanExpander); + return result; + } + + /** + * Converts a collection of artifacts into execution-time path strings with + * the root-break delimited with a colon ':', and adds those to a given list. + * <pre> + * Source: sourceRoot/rootRelative => :rootRelative + * Derived: execRoot/rootPrefix/rootRelative => rootPrefix:rootRelative + * </pre> + */ + public static void addRootPrefixedExecPaths(Iterable<Artifact> artifacts, + List<String> output) { + for (Artifact artifact : artifacts) { + output.add(asRootPrefixedExecPath(artifact)); + } + } + + /** + * Convenience method to filter the files to build for a certain filetype. + * + * @param artifacts the files to filter + * @param allowedType the allowed filetype + * @return all members of filesToBuild that are of one of the + * allowed filetypes + */ + public static List<Artifact> filterFiles(Iterable<Artifact> artifacts, FileType allowedType) { + List<Artifact> filesToBuild = new ArrayList<>(); + for (Artifact artifact : artifacts) { + if (allowedType.matches(artifact.getFilename())) { + filesToBuild.add(artifact); + } + } + return filesToBuild; + } + + @VisibleForTesting + static String asRootPrefixedExecPath(Artifact artifact) { + PathFragment execPath = artifact.getExecPath(); + PathFragment rootRel = artifact.getRootRelativePath(); + if (execPath.equals(rootRel)) { + return ":" + rootRel.getPathString(); + } else { //if (execPath.endsWith(rootRel)) { + PathFragment rootPrefix = trimTail(execPath, rootRel); + return rootPrefix.getPathString() + ":" + rootRel.getPathString(); + } + } + + /** + * Converts artifacts into their exec paths. Returns an immutable list. + */ + public static List<PathFragment> asPathFragments(Iterable<Artifact> artifacts) { + return ImmutableList.copyOf(Iterables.transform(artifacts, EXEC_PATH_FORMATTER)); + } + + static final ArtifactOwner DESERIALIZED_MARKER_OWNER = new ArtifactOwner() { + @Override + public Label getLabel() { + return null; + }}; +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java new file mode 100644 index 0000000000..40996acd4d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactDeserializer.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; + +/** + * An interface for creating artifacts from their serialized representations. + * + * @see ArtifactSerializer + */ +public interface ArtifactDeserializer { + + /** + * Looks up an artifact by an integer id. + * + * <p>This is a dual of {@link ArtifactSerializer#getArtifactId}. + */ + Artifact lookupArtifactById(int artifactId); + + /** + * Maps a list of artifact ids to a list of artifacts. + * + * <p>This is a batch version of {@link #lookupArtifactById}, provided for efficiency. It takes + * an iterable of boxed integers because that's what the proto wrapper provides. + */ + ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java new file mode 100644 index 0000000000..99a53cbc62 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java @@ -0,0 +1,339 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; + +/** + * A cache of Artifacts, keyed by Path. + */ +@ThreadSafe +public class ArtifactFactory implements ArtifactResolver, ArtifactSerializer, ArtifactDeserializer { + + private final Path execRoot; + + /** + * The main Path to source artifact cache. There will always be exactly one canonical + * artifact for a given source path. + */ + private final Map<PathFragment, Artifact> pathToSourceArtifact = new HashMap<>(); + + /** + * Map of package names to source root paths so that we can create source + * artifact paths given execPaths in the symlink forest. + */ + private ImmutableMap<PackageIdentifier, Root> packageRoots; + + /** + * Reverse-ordered list of derived roots for use in looking up or (in rare cases) creating + * derived artifacts from execPaths. The reverse order is only significant for overlapping roots + * so that the longest is found first. + */ + private ImmutableCollection<Root> derivedRoots = ImmutableList.of(); + + private ArtifactIdRegistry artifactIdRegistry = new ArtifactIdRegistry(); + + /** + * Constructs a new artifact factory that will use a given execution root when + * creating artifacts. + * + * @param execRoot the execution root Path to use + */ + public ArtifactFactory(Path execRoot) { + this.execRoot = execRoot; + } + + /** + * Clear the cache. + */ + public synchronized void clear() { + pathToSourceArtifact.clear(); + packageRoots = null; + derivedRoots = ImmutableList.of(); + artifactIdRegistry = new ArtifactIdRegistry(); + clearDeserializedArtifacts(); + } + + /** + * Set the set of known packages and their corresponding source artifact + * roots. Must be called exactly once after construction or clear(). + * + * @param packageRoots the map of package names to source artifact roots to + * use. + */ + public synchronized void setPackageRoots(Map<PackageIdentifier, Root> packageRoots) { + this.packageRoots = ImmutableMap.copyOf(packageRoots); + } + + /** + * Set the set of known derived artifact roots. Must be called exactly once + * after construction or clear(). + * + * @param roots the set of derived artifact roots to use + */ + public synchronized void setDerivedArtifactRoots(Collection<Root> roots) { + derivedRoots = ImmutableSortedSet.<Root>reverseOrder().addAll(roots).build(); + } + + @Override + public Artifact getSourceArtifact(PathFragment execPath, Root root, ArtifactOwner owner) { + Preconditions.checkArgument(!execPath.isAbsolute()); + Preconditions.checkNotNull(owner, execPath); + execPath = execPath.normalize(); + return getArtifact(root.getPath().getRelative(execPath), root, execPath, owner, null); + } + + @Override + public Artifact getSourceArtifact(PathFragment execPath, Root root) { + return getSourceArtifact(execPath, root, ArtifactOwner.NULL_OWNER); + } + + /** + * Only for use by BinTools! Returns an artifact for a tool at the given path + * fragment, relative to the exec root, creating it if not found. This method + * only works for normalized, relative paths. + */ + public Artifact getDerivedArtifact(PathFragment execPath) { + Preconditions.checkArgument(!execPath.isAbsolute(), execPath); + Preconditions.checkArgument(execPath.isNormalized(), execPath); + // TODO(bazel-team): Check that either BinTools do not change over the life of the Blaze server, + // or require that a legitimate ArtifactOwner be passed in here to allow for ownership. + return getArtifact(execRoot.getRelative(execPath), Root.execRootAsDerivedRoot(execRoot), + execPath, ArtifactOwner.NULL_OWNER, null); + } + + private void validatePath(PathFragment rootRelativePath, Root root) { + Preconditions.checkArgument(!rootRelativePath.isAbsolute(), rootRelativePath); + Preconditions.checkArgument(rootRelativePath.isNormalized(), rootRelativePath); + Preconditions.checkArgument(root.getPath().startsWith(execRoot), "%s %s", root, execRoot); + Preconditions.checkArgument(!root.getPath().equals(execRoot), "%s %s", root, execRoot); + // TODO(bazel-team): this should only accept roots from derivedRoots. + //Preconditions.checkArgument(derivedRoots.contains(root), "%s not in %s", root, derivedRoots); + } + + /** + * Returns an artifact for a tool at the given root-relative path under the given root, creating + * it if not found. This method only works for normalized, relative paths. + * + * <p>The root must be below the execRoot, and the execPath of the resulting Artifact is computed + * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}. + */ + // TODO(bazel-team): Don't allow root == execRoot. + public Artifact getDerivedArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact(path, root, path.relativeTo(execRoot), owner, null); + } + + /** + * Returns an artifact that represents the output directory of a Fileset at the given + * root-relative path under the given root, creating it if not found. This method only works for + * normalized, relative paths. + * + * <p>The root must be below the execRoot, and the execPath of the resulting Artifact is computed + * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}. + */ + public Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact(path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.FILESET); + } + + public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact( + path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.CONSTANT_METADATA); + } + + /** + * Returns the Artifact for the specified path, creating one if not found and + * setting the <code>root</code> and <code>execPath</code> to the + * specified values. + */ + private synchronized Artifact getArtifact(Path path, Root root, PathFragment execPath, + ArtifactOwner owner, @Nullable SpecialArtifactType type) { + Preconditions.checkNotNull(root); + Preconditions.checkNotNull(execPath); + + if (!root.isSourceRoot()) { + return createArtifact(path, root, execPath, owner, type); + } + + Artifact artifact = pathToSourceArtifact.get(execPath); + + if (artifact == null || !Objects.equals(artifact.getArtifactOwner(), owner)) { + // There really should be a safety net that makes it impossible to create two Artifacts + // with the same exec path but a different Owner, but we also need to reuse Artifacts from + // previous builds. + artifact = createArtifact(path, root, execPath, owner, type); + pathToSourceArtifact.put(execPath, artifact); + } else { + // TODO(bazel-team): Maybe we should check for equality of the fileset bit. However, that + // would require us to differentiate between artifact-creating and artifact-getting calls to + // getDerivedArtifact(). + Preconditions.checkState(root.equals(artifact.getRoot()), + "root for path %s changed from %s to %s", path, artifact.getRoot(), root); + Preconditions.checkState(execPath.equals(artifact.getExecPath()), + "execPath for path %s changed from %s to %s", path, artifact.getExecPath(), execPath); + } + return artifact; + } + + private Artifact createArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner, + @Nullable SpecialArtifactType type) { + Preconditions.checkNotNull(owner, path); + if (type == null) { + return new Artifact(path, root, execPath, owner); + } else { + return new Artifact.SpecialArtifact(path, root, execPath, owner, type); + } + } + + @Override + public synchronized Artifact resolveSourceArtifact(PathFragment execPath) { + execPath = execPath.normalize(); + // First try a quick map lookup to see if the artifact already exists. + Artifact a = pathToSourceArtifact.get(execPath); + if (a != null) { + return a; + } + // Don't create an artifact if it's derived. + if (findDerivedRoot(execRoot.getRelative(execPath)) != null) { + return null; + } + // Must be a new source artifact, so probe the known packages to find the longest package + // prefix, and then use the corresponding source root to create a new artifact. + for (PathFragment dir = execPath.getParentDirectory(); dir != null; + dir = dir.getParentDirectory()) { + Root sourceRoot = packageRoots.get(PackageIdentifier.createInDefaultRepo(dir)); + if (sourceRoot != null) { + return getSourceArtifact(execPath, sourceRoot, ArtifactOwner.NULL_OWNER); + } + } + return null; // not a path that we can find... + } + + /** + * Finds the derived root for a full path by comparing against the known + * derived artifact roots. + * + * @param path a Path to resolve the root for + * @return the root for the path or null if no root can be determined + */ + @VisibleForTesting // for our own unit tests only. + synchronized Root findDerivedRoot(Path path) { + for (Root prefix : derivedRoots) { + if (path.startsWith(prefix.getPath())) { + return prefix; + } + } + return null; + } + + /** + * Returns all source artifacts created by the artifact factory. + */ + public synchronized Iterable<Artifact> getSourceArtifacts() { + return ImmutableList.copyOf(pathToSourceArtifact.values()); + } + + // Non-final only because clear()ing a map does not actually free the memory it took up, so we + // assign it to a new map in lieu of clearing. + private ConcurrentMap<PathFragment, Artifact> deserializedArtifacts = + new ConcurrentHashMap<>(); + + /** + * Returns the map of all artifacts that were deserialized this build. The caller should process + * them and then call {@link #clearDeserializedArtifacts}. + */ + public Map<PathFragment, Artifact> getDeserializedArtifacts() { + return deserializedArtifacts; + } + + /** Clears the map of deserialized artifacts. */ + public void clearDeserializedArtifacts() { + deserializedArtifacts = new ConcurrentHashMap<>(); + } + + /** + * Resolves an artifact based on its deserialized representation. The artifact can be either a + * source or a derived one. + * + * <p>Note: this method represents a hole in the usual contract that artifacts with a random path + * cannot be created. Unfortunately, we currently need this in some cases. + * + * @param execPath the exec path of the artifact + */ + public Artifact deserializeArtifact(PathFragment execPath, PackageRootResolver resolver) { + Preconditions.checkArgument(!execPath.isAbsolute(), execPath); + Path path = execRoot.getRelative(execPath); + Root root = findDerivedRoot(path); + + Artifact result; + if (root != null) { + result = getDerivedArtifact(path.relativeTo(root.getPath()), root, + Artifact.DESERIALIZED_MARKER_OWNER); + Artifact oldResult = deserializedArtifacts.putIfAbsent(execPath, result); + if (oldResult != null) { + result = oldResult; + } + return result; + } else { + Map<PathFragment, Root> sourceRoots = resolver.findPackageRoots(Lists.newArrayList(execPath)); + if (sourceRoots == null || sourceRoots.get(execPath) == null) { + return null; + } + return getSourceArtifact(execPath, sourceRoots.get(execPath), ArtifactOwner.NULL_OWNER); + } + } + + @Override + public Artifact lookupArtifactById(int artifactId) { + return artifactIdRegistry.lookupArtifactById(artifactId); + } + + @Override + public ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds) { + return artifactIdRegistry.lookupArtifactsByIds(artifactIds); + } + + @Override + public int getArtifactId(Artifact artifact) { + return artifactIdRegistry.getArtifactId(artifact); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java new file mode 100644 index 0000000000..9edf684e98 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactIdRegistry.java @@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.MapMaker; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * A registry that keeps a map of artifacts to unique integer ids. + */ +class ArtifactIdRegistry implements ArtifactSerializer, ArtifactDeserializer { + + /** + * A sequence of registered artifacts. The position in the list is the artifact's id. + * + * <p>Synchronized using {@link #artifactIdsLock}. + */ + private final List<Artifact> serializedArtifactList = new ArrayList<>(); + + /** + * A map of artifacts to unique integer ids. + * + * <p>Writes to this map must be synchronized using {@link #artifactIdsLock}, in order to + * maintain consistency with {@link #serializedArtifactList}. + */ + private final ConcurrentMap<Artifact, Integer> serializedArtifactIds = + new MapMaker().concurrencyLevel(1).makeMap(); + + /** + * A lock for keeping {@code serializedArtifactList} and {@code serializedArtifactIds} in sync. + */ + private ReadWriteLock artifactIdsLock = new ReentrantReadWriteLock(); + + ArtifactIdRegistry() { + } + + @Override + public int getArtifactId(Artifact artifact) { + Integer artifactId = serializedArtifactIds.get(artifact); + if (artifactId == null) { + artifactId = assignArtifactId(artifact); + } + return artifactId; + } + + private Integer assignArtifactId(Artifact artifact) { + artifactIdsLock.writeLock().lock(); + try { + Integer artifactId = serializedArtifactIds.get(artifact); + if (artifactId == null) { + artifactId = serializedArtifactList.size(); + serializedArtifactList.add(artifact); + serializedArtifactIds.put(artifact, artifactId); + } + return artifactId; + } finally { + artifactIdsLock.writeLock().unlock(); + } + } + + @Override + public Artifact lookupArtifactById(int artifactId) { + artifactIdsLock.readLock().lock(); + try { + return serializedArtifactList.get(artifactId); + } finally { + artifactIdsLock.readLock().unlock(); + } + } + + @Override + public ImmutableList<Artifact> lookupArtifactsByIds(Iterable<Integer> artifactIds) { + int size = Iterables.size(artifactIds); + Artifact[] result = new Artifact[size]; + + int i = 0; + + artifactIdsLock.readLock().lock(); + try { + for (int artifactId : artifactIds) { + result[i] = serializedArtifactList.get(artifactId); + i++; + } + } finally { + artifactIdsLock.readLock().unlock(); + } + + return ImmutableList.copyOf(result); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java new file mode 100644 index 0000000000..458fb42835 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java @@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.syntax.Label; + +/** + * An interface for {@code LabelAndConfiguration}, or at least for a {@link Label}. Only tests and + * internal {@link Artifact}-generators should implement this interface -- otherwise, + * {@code LabelAndConfiguration} should be the only implementation. + */ +public interface ArtifactOwner { + Label getLabel(); + + @VisibleForTesting + public static final ArtifactOwner NULL_OWNER = new ArtifactOwner() { + @Override + public Label getLabel() { + return null; + } + + @Override + public String toString() { + return "NULL_OWNER"; + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java new file mode 100644 index 0000000000..42ad28568b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactPrefixConflictException.java @@ -0,0 +1,33 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Exception to indicate that one {@link Action} has an output artifact whose path is a prefix of an + * output of another action. Since the first path cannot be both a directory and a file, this would + * lead to an error if both actions were executed in the same build. + */ +public class ArtifactPrefixConflictException extends Exception { + public ArtifactPrefixConflictException(PathFragment firstPath, PathFragment secondPath, + Label firstOwner, Label secondOwner) { + super(String.format( + "output path '%s' (belonging to %s) is a prefix of output path '%s' (belonging to %s). " + + "These actions cannot be simultaneously present; please rename one of the output files " + + "or, as a last resort, run 'blaze clean' and then build just one of them", + firstPath, firstOwner, secondPath, secondOwner)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java new file mode 100644 index 0000000000..b74c17b0f9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactResolver.java @@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * An interface for resolving artifact names to {@link Artifact} objects. Should only be used + * in the internal machinery of Blaze: rule implementations are not allowed to do this. + */ +public interface ArtifactResolver { + /** + * Returns the source Artifact for the specified path, creating it if not found and setting its + * root and execPath. + * + * @param execPath the path of the source artifact relative to the source root + * @param root the source root prefix of the path + * @param owner the artifact owner. + * @return the canonical source artifact for the given path + */ + Artifact getSourceArtifact(PathFragment execPath, Root root, ArtifactOwner owner); + + /** + * Returns the source Artifact for the specified path, creating it if not found and setting its + * root and execPath. + * + * @see #getSourceArtifact(PathFragment, Root, ArtifactOwner) + */ + Artifact getSourceArtifact(PathFragment execPath, Root root); + + /** + * Resolves a source Artifact given an execRoot-relative path. + * + * <p>Never creates or returns derived artifacts, only source artifacts. + * + * <p>Note: this method should only be used when the roots are unknowable, such as from the + * post-compile .d or manifest scanning methods. + * + * @param execPath the exec path of the artifact to resolve + * @return an existing or new source Artifact for the given execPath. Returns null if + * the root can not be determined and the artifact did not exist before. + */ + Artifact resolveSourceArtifact(PathFragment execPath); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java new file mode 100644 index 0000000000..703eeb788a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactSerializer.java @@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An interface for creating artifacts from their serialized representations. + * + * @see ArtifactDeserializer + */ +public interface ArtifactSerializer { + + /** + * Returns a number that uniquely identifies an artifact. + * + * <p>The artifact can be retrieved again later by calling + * {@link ArtifactDeserializer#lookupArtifactById}. + */ + int getArtifactId(Artifact artifact); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java b/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java new file mode 100644 index 0000000000..dd879c2b88 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BaseSpawn.java @@ -0,0 +1,214 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.extra.EnvironmentVariable; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.util.CommandDescriptionForm; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Base implementation of a Spawn. + */ +@Immutable +public class BaseSpawn implements Spawn { + private final ImmutableList<String> arguments; + private final ImmutableMap<String, String> environment; + private final ImmutableMap<String, String> executionInfo; + private final ImmutableMap<PathFragment, Artifact> runfilesManifests; + private final ActionMetadata action; + private final ResourceSet localResources; + + /** + * Returns a new Spawn. The caller must not modify the parameters after the call; neither will + * this method. + */ + public BaseSpawn(List<String> arguments, + Map<String, String> environment, + Map<String, String> executionInfo, + Map<PathFragment, Artifact> runfilesManifests, + ActionMetadata action, + ResourceSet localResources) { + this.arguments = ImmutableList.copyOf(arguments); + this.environment = ImmutableMap.copyOf(environment); + this.executionInfo = ImmutableMap.copyOf(executionInfo); + this.runfilesManifests = ImmutableMap.copyOf(runfilesManifests); + this.action = action; + this.localResources = localResources; + } + + /** + * Returns a new Spawn. + */ + public BaseSpawn(List<String> arguments, + Map<String, String> environment, + Map<String, String> executionInfo, + // TODO(bazel-team): have this always be non-null. + @Nullable Artifact runfilesManifest, + ActionMetadata action, + ResourceSet localResources) { + this(arguments, environment, executionInfo, + ((runfilesManifest != null) + ? ImmutableMap.of(runfilesForFragment(new PathFragment(arguments.get(0))), + runfilesManifest) + : ImmutableMap.<PathFragment, Artifact>of()), + action, localResources); + } + + public static PathFragment runfilesForFragment(PathFragment pathFragment) { + return pathFragment.getParentDirectory().getChild(pathFragment.getBaseName() + ".runfiles"); + } + + /** + * Returns a new Spawn. + */ + public BaseSpawn(List<String> arguments, + Map<String, String> environment, + Map<String, String> executionInfo, + ActionMetadata action, + ResourceSet localResources) { + this(arguments, environment, executionInfo, + ImmutableMap.<PathFragment, Artifact>of(), action, localResources); + } + + @Override + public boolean isRemotable() { + return !executionInfo.containsKey("local"); + } + + @Override + public final ImmutableMap<String, String> getExecutionInfo() { + return executionInfo; + } + + @Override + public String asShellCommand(Path workingDir) { + return asShellCommand(getArguments(), workingDir, getEnvironment()); + } + + @Override + public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() { + return runfilesManifests; + } + + @Override + public ImmutableList<Artifact> getFilesetManifests() { + return ImmutableList.<Artifact>of(); + } + + @Override + public SpawnInfo getExtraActionInfo() { + SpawnInfo.Builder info = SpawnInfo.newBuilder(); + + info.addAllArgument(getArguments()); + for (Map.Entry<String, String> variable : getEnvironment().entrySet()) { + info.addVariable(EnvironmentVariable.newBuilder() + .setName(variable.getKey()) + .setValue(variable.getValue()).build()); + } + for (ActionInput input : getInputFiles()) { + // Explicitly ignore middleman artifacts here. + if (!(input instanceof Artifact) || !((Artifact) input).isMiddlemanArtifact()) { + info.addInputFile(input.getExecPathString()); + } + } + info.addAllOutputFile(ActionInputHelper.toExecPaths(getOutputFiles())); + return info.build(); + } + + @Override + public ImmutableList<String> getArguments() { + // TODO(bazel-team): this method should be final, as the correct value of the args can be + // injected in the ctor. + return arguments; + } + + @Override + public ImmutableMap<String, String> getEnvironment() { + if (getRunfilesManifests().size() != 1) { + return environment; + } + + ImmutableMap.Builder<String, String> env = ImmutableMap.builder(); + env.putAll(environment); + for (Map.Entry<PathFragment, Artifact> e : getRunfilesManifests().entrySet()) { + // TODO(bazel-team): Unify these into a single env variable. + env.put("JAVA_RUNFILES", e.getKey().getPathString() + "/"); + env.put("PYTHON_RUNFILES", e.getKey().getPathString() + "/"); + } + return env.build(); + } + + @Override + public Iterable<? extends ActionInput> getInputFiles() { + return action.getInputs(); + } + + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return action.getOutputs(); + } + + @Override + public ActionMetadata getResourceOwner() { + return action; + } + + @Override + public ResourceSet getLocalResources() { + return localResources; + } + + @Override + public ActionOwner getOwner() { return action.getOwner(); } + + @Override + public String getMnemonic() { return action.getMnemonic(); } + + /** + * Convert a working dir + environment map + arg list into a Bourne shell + * command. + */ + public static String asShellCommand(Collection<String> arguments, + Path workingDirectory, + Map<String, String> environment) { + // We print this command out in such a way that it can safely be + // copied+pasted as a Bourne shell command. This is extremely valuable for + // debugging. + return CommandFailureUtils.describeCommand(CommandDescriptionForm.COMPLETE, + arguments, environment, workingDirectory.getPathString()); + } + + /** + * A local spawn requiring zero resources. + */ + public static class Local extends BaseSpawn { + public Local(List<String> arguments, Map<String, String> environment, ActionMetadata action) { + super(arguments, environment, ImmutableMap.<String, String>of("local", ""), + action, ResourceSet.ZERO); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java b/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java new file mode 100644 index 0000000000..61803c8dd1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BipartiteVisitor.java @@ -0,0 +1,98 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import java.util.HashMap; +import java.util.Map; + +/** + * A visitor helper class for bipartite graphs. The alternate kinds of nodes + * are arbitrarily designated "black" or "white". + * + * <p> Subclasses implement the black() and white() hook functions which are + * called as nodes are visited. The class holds a mapping from each node to a + * small integer; this is available to subclasses if they wish. + */ +public abstract class BipartiteVisitor<BLACK, WHITE> { + + protected BipartiteVisitor() {} + + private int nextNodeId = 0; + + // Maps each visited black node to a small integer. + protected final Map<BLACK, Integer> visitedBlackNodes = new HashMap<>(); + + // Maps each visited white node to a small integer. + protected final Map<WHITE, Integer> visitedWhiteNodes = new HashMap<>(); + + /** + * Visit the specified black node. If this node has not already been + * visited, the black() hook is called and true is returned; otherwise, + * false is returned. + */ + public final boolean visitBlackNode(BLACK blackNode) { + if (blackNode == null) { throw new NullPointerException(); } + if (!visitedBlackNodes.containsKey(blackNode)) { + visitedBlackNodes.put(blackNode, nextNodeId++); + black(blackNode); + return true; + } + return false; + } + + /** + * Visit all specified black nodes. + */ + public final void visitBlackNodes(Iterable<BLACK> blackNodes) { + for (BLACK blackNode : blackNodes) { + visitBlackNode(blackNode); + } + } + + /** + * Visit the specified white node. If this node has not already been + * visited, the white() hook is called and true is returned; otherwise, + * false is returned. + */ + public final boolean visitWhiteNode(WHITE whiteNode) { + if (whiteNode == null) { + throw new NullPointerException(); + } + if (!visitedWhiteNodes.containsKey(whiteNode)) { + visitedWhiteNodes.put(whiteNode, nextNodeId++); + white(whiteNode); + return true; + } + return false; + } + + /** + * Visit all specified white nodes. + */ + public final void visitWhiteNodes(Iterable<WHITE> whiteNodes) { + for (WHITE whiteNode : whiteNodes) { + visitWhiteNode(whiteNode); + } + } + + /** + * Called whenever a white node is visited. Hook for subclasses. + */ + protected abstract void white(WHITE whiteNode); + + /** + * Called whenever a black node is visited. Hook for subclasses. + */ + protected abstract void black(BLACK blackNode); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java b/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java new file mode 100644 index 0000000000..fd3c3d9de1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BlazeExecutor.java @@ -0,0 +1,233 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The Executor class provides a dynamic abstraction of the various actual primitive system + * operations that might be performed during a build step. + * + * <p>Constructions of this class might perform distributed execution, "virtual" execution for + * testing purposes, or just print out the sequence of commands that would be executed, like Make's + * "-n" option. + */ +@ThreadSafe +public final class BlazeExecutor implements Executor { + + private final Path outputPath; + private final boolean verboseFailures; + private final boolean showSubcommands; + private final Path execRoot; + private final Reporter reporter; + private final EventBus eventBus; + private final Clock clock; + private final OptionsClassProvider options; + private AtomicBoolean inExecutionPhase; + + private final Map<String, SpawnActionContext> spawnActionContextMap; + private final Map<Class<? extends ActionContext>, ActionContext> contextMap = + new HashMap<>(); + + /** + * Constructs an Executor, bound to a specified output base path, and which + * will use the specified reporter to announce SUBCOMMAND events, + * the given event bus to delegate events and the given output streams + * for streaming output. The list of + * strategy implementation classes is used to construct instances of the + * strategies mapped by their declared abstract type. This list is uniquified + * before using. Each strategy instance is created with a reference to this + * Executor as well as the given options object. + * <p> + * Don't forget to call startBuildRequest() and stopBuildRequest() for each + * request, and shutdown() when you're done with this executor. + */ + public BlazeExecutor(Path execRoot, + Path outputPath, + Reporter reporter, + EventBus eventBus, + Clock clock, + OptionsClassProvider options, + boolean verboseFailures, + boolean showSubcommands, + List<ActionContext> contextImplementations, + Map<String, ActionContext> spawnContextMap, + Iterable<ActionContextProvider> contextProviders) + throws ExecutorInitException { + this.outputPath = outputPath; + this.verboseFailures = verboseFailures; + this.showSubcommands = showSubcommands; + this.execRoot = execRoot; + this.reporter = reporter; + this.eventBus = eventBus; + this.clock = clock; + this.options = options; + this.inExecutionPhase = new AtomicBoolean(false); + + // We need to keep only the last occurrences of the entries in contextImplementations + // (so we respect insertion order but also instantiate them only once). + LinkedHashSet<ActionContext> allContexts = new LinkedHashSet<>(); + allContexts.addAll(contextImplementations); + + ImmutableMap.Builder<String, SpawnActionContext> spawnMapBuilder = ImmutableMap.builder(); + for (Map.Entry<String, ActionContext> entry: spawnContextMap.entrySet()) { + spawnMapBuilder.put(entry.getKey(), (SpawnActionContext) entry.getValue()); + allContexts.add(entry.getValue()); + } + + for (ActionContext context : contextImplementations) { + ExecutionStrategy annotation = context.getClass().getAnnotation(ExecutionStrategy.class); + if (annotation != null) { + contextMap.put(annotation.contextType(), context); + } + } + this.spawnActionContextMap = spawnMapBuilder.build(); + + for (ActionContextProvider factory : contextProviders) { + factory.executorCreated(allContexts); + } + } + + @Override + public Path getExecRoot() { + return execRoot; + } + + @Override + public EventHandler getEventHandler() { + return reporter; + } + + @Override + public EventBus getEventBus() { + return eventBus; + } + + @Override + public Clock getClock() { + return clock; + } + + @Override + public boolean reportsSubcommands() { + return showSubcommands; + } + + /** + * Report a subcommand event to this Executor's Reporter and, if action + * logging is enabled, post it on its EventBus. + */ + @Override + public void reportSubcommand(String reason, String message) { + reporter.handle(new Event(EventKind.SUBCOMMAND, null, "# " + reason + "\n" + message)); + } + + /** + * This method is called before the start of the execution phase of each + * build request. + */ + public void executionPhaseStarting() { + Preconditions.checkState(!inExecutionPhase.getAndSet(true)); + Profiler.instance().startTask(ProfilerTask.INFO, "Initializing executors"); + Profiler.instance().completeTask(ProfilerTask.INFO); + } + + /** + * This method is called after the end of the execution phase of each build + * request (even if there was an interrupt). + */ + public void executionPhaseEnding() { + if (!inExecutionPhase.get()) { + return; + } + + Profiler.instance().startTask(ProfilerTask.INFO, "Shutting down executors"); + Profiler.instance().completeTask(ProfilerTask.INFO); + inExecutionPhase.set(false); + } + + public static void shutdownHelperPool(EventHandler reporter, ExecutorService pool, + String name) { + pool.shutdownNow(); + + boolean interrupted = false; + while (true) { + try { + if (!pool.awaitTermination(10, TimeUnit.SECONDS)) { + reporter.handle(Event.warn(name + " threadpool shutdown took greater than ten seconds")); + } + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + @Override + public <T extends ActionContext> T getContext(Class<? extends T> type) { + Preconditions.checkArgument(type != SpawnActionContext.class, + "should use getSpawnActionContext instead"); + return type.cast(contextMap.get(type)); + } + + /** + * Returns the {@link SpawnActionContext} to use for the given mnemonic. If no execution mode is + * set, then it returns the default strategy for spawn actions. + */ + @Override + public SpawnActionContext getSpawnActionContext(String mnemonic) { + SpawnActionContext context = spawnActionContextMap.get(mnemonic); + return context == null ? spawnActionContextMap.get("") : context; + } + + /** Returns true iff the --verbose_failures option was enabled. */ + @Override + public boolean getVerboseFailures() { + return verboseFailures; + } + + /** Returns the options associated with the execution. */ + @Override + public OptionsClassProvider getOptions() { + return options; + } + + public Path getOutputPath() { + return outputPath; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java b/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java new file mode 100644 index 0000000000..088ba60948 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BuildFailedException.java @@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; + +/** + * This exception gets thrown if there were errors during the execution phase of + * the build. + * + * <p>The argument to the constructor may be null if the thrower has already + * printed an error message; in this case, no error message should be printed by + * the catcher. (Typically, this happens when the builder is unsuccessful and + * {@code --keep_going} was specified. This error corresponds to one or more + * actions failing, but since those actions' failures will be reported + * separately, the exception carries no message and is just used for control + * flow.) + */ +@ThreadSafe +public class BuildFailedException extends Exception { + private final boolean catastrophic; + private final Action action; + private final Iterable<Label> rootCauses; + private final boolean errorAlreadyShown; + + public BuildFailedException() { + this(null); + } + + public BuildFailedException(String message) { + this(message, false, null, ImmutableList.<Label>of()); + } + + public BuildFailedException(String message, boolean catastrophic) { + this(message, catastrophic, null, ImmutableList.<Label>of()); + } + + public BuildFailedException(String message, boolean catastrophic, + Action action, Iterable<Label> rootCauses) { + this(message, catastrophic, action, rootCauses, false); + } + + public BuildFailedException(String message, boolean catastrophic, + Action action, Iterable<Label> rootCauses, boolean errorAlreadyShown) { + super(message); + this.catastrophic = catastrophic; + this.rootCauses = ImmutableList.copyOf(rootCauses); + this.action = action; + this.errorAlreadyShown = errorAlreadyShown; + } + + public boolean isCatastrophic() { + return catastrophic; + } + + public Action getAction() { + return action; + } + + public Iterable<Label> getRootCauses() { + return rootCauses; + } + + public boolean isErrorAlreadyShown() { + return errorAlreadyShown || getMessage() == null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java b/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java new file mode 100644 index 0000000000..72ce13335a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/BuilderUtils.java @@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * Methods needed by {@code SkyframeBuilder}. + */ +public final class BuilderUtils { + + private BuilderUtils() {} + + /** + * Figure out why an action's execution failed and rethrow the right kind of exception. + */ + public static void rethrowCause(Exception e) throws BuildFailedException, TestExecException { + Throwable cause = e.getCause(); + Throwable innerCause = cause.getCause(); + if (innerCause instanceof TestExecException) { + throw (TestExecException) innerCause; + } + if (cause instanceof ActionExecutionException) { + ActionExecutionException actionExecutionCause = (ActionExecutionException) cause; + // Sometimes ActionExecutionExceptions are caused by Actions with no owner. + String message = + (actionExecutionCause.getLocation() != null) ? + (actionExecutionCause.getLocation().print() + " " + cause.getMessage()) : + e.getMessage(); + throw new BuildFailedException(message, actionExecutionCause.isCatastrophe(), + actionExecutionCause.getAction(), actionExecutionCause.getRootCauses(), + /*errorAlreadyShown=*/ !actionExecutionCause.showError()); + } else if (cause instanceof MissingInputFileException) { + throw new BuildFailedException(cause.getMessage()); + } else if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } else if (cause instanceof Error) { + throw (Error) cause; + } else { + /* + * This should never happen - we should only get exceptions listed in the exception + * specification for ExecuteBuildAction.call(). + */ + throw new IllegalArgumentException("action terminated with " + + "unexpected exception: " + cause.getMessage(), cause); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java b/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java new file mode 100644 index 0000000000..35db7d6033 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/CachedActionEvent.java @@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * This event is fired during the build if an action was in the action cache. + */ +public class CachedActionEvent { + + private final Action action; + private final long nanoTimeStart; + + /** + * Create an event for an action that was cached. + * + * @param action the cached action + * @param nanoTimeStart the time when the action was started. This allow us to + * record more accurately the time spend by the action, since we execute some code before + * deciding if we execute the action or not. + */ + public CachedActionEvent(Action action, long nanoTimeStart) { + this.action = action; + this.nanoTimeStart = nanoTimeStart; + } + + public Action getAction() { + return action; + } + + public long getNanoTimeStart() { + return nanoTimeStart; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java new file mode 100644 index 0000000000..9177cbacab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ChangedArtifactsMessage.java @@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; + +import java.util.Set; + +/** + * Used to signal when the incremental builder has found the set of changed artifacts. + */ +public class ChangedArtifactsMessage { + + private final Set<Artifact> artifacts; + + public ChangedArtifactsMessage(Set<Artifact> changedArtifacts) { + this.artifacts = ImmutableSet.copyOf(changedArtifacts); + } + + public Set<Artifact> getChangedArtifacts() { + return artifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java b/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java new file mode 100644 index 0000000000..56f688284c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ChangedFilesMessage.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Set; + +/** + * A message sent conveying a set of changed files. + */ +public class ChangedFilesMessage { + + private final Set<PathFragment> changedFiles; + + public ChangedFilesMessage(Set<PathFragment> changedFiles) { + this.changedFiles = ImmutableSet.copyOf(changedFiles); + } + + public Set<PathFragment> getChangedFiles() { + return changedFiles; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java b/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java new file mode 100644 index 0000000000..d9d875d944 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElement.java @@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; + +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; + +import javax.annotation.Nullable; + +/** + * A Multimap-like object that is actually a {@link ConcurrentMap} of {@code SmallSet}s to avoid + * the memory penalties of a {@code Multimap} while preserving concurrency guarantees, and + * retrieving a consistent "head" element. Operations are guaranteed to reflect a consistent view of + * a {@code SetMultimap}, although most methods are not implemented. + */ +final class ConcurrentMultimapWithHeadElement<K, V> { + private final ConcurrentMap<K, SmallSet<V>> map = Maps.newConcurrentMap(); + + /** + * Remove (key, val) pair from the multimap. If this removes the current 'head' element + * for a key, then another randomly chosen element becomes the current head. + * + * <p>Until the next (possibly concurrent) {@link #putAndGet}(key, val) call, {@link #get}(key) + * will never return val. + */ + void remove(K key, V val) { + SmallSet<V> entry = getEntry(key); + if (entry != null) { + entry.remove(val); + if (entry.get() == null) { + // Remove entry completely from map if dead. + map.remove(key, entry); + } + } + } + + /** + * Return some value val such that (key, val) is in the multimap. If there is always at least one + * entry for key in the multimap during the lifetime of this method call, it will not return null. + */ + @Nullable V get(K key) { + SmallSet<V> entry = getEntry(key); + return (entry != null) ? entry.get() : null; + } + + /** + * Adds (key, val) to the multimap. Returns the head element for key, either val or another + * already-stored value. + */ + V putAndGet(K key, V val) { + V result = null; + while (result == null) { + // If another thread concurrently removes the only remaining value from the entry, this + // putAndGet will return null, since the entry is about to be removed from the map. In that + // case, we obtain a fresh entry from the map and do the put on it. + result = getOrCreateEntry(key).putAndGet(val); + } + return result; + } + + /** + * Obtain the entry for key, adding it to the underlying map if no entry was previously present. + */ + private SmallSet<V> getOrCreateEntry(K key) { + SmallSet<V> entry = new SmallSet<V>(); + SmallSet<V> oldEntry = map.putIfAbsent(key, entry); + if (oldEntry != null) { + return oldEntry; + } + return entry; + } + + /** + * Obtain the entry for key, returning null if no entry was present in the underlying map. + */ + private SmallSet<V> getEntry(K key) { + return map.get(key); + } + + /** + * Clears the multimap. May not be called concurrently with any other methods. + */ + @ThreadHostile + void clear() { + map.clear(); + } + + /** + * Wrapper for a {@code #Set} that will probably have at most one element. Keeps the first element + * in a separate variable for fast reading/writing and to save space if more than one element is + * never written to this set. We always have the invariant that {@link #first} is null only if + * {@link #rest} is null. + */ + private static class SmallSet<T> { + /* + * What is this 'volatile' on first and where's the lock on the read path? + * + * Volatile is an alternative to locking that works only in very limited situations, such as + * simple field reads and writes. Writes from one thread to 'first' happen before reads from + * other threads. When used correctly, it can have the same correctness properties as a + * 'synchronized' but is much faster on most hardware. + * + * Here, volatile is used to eliminate locks on the read path. Since get() is merely fetching + * the contents of 'first', it meets the criteria for a safe volatile read. In the mutator + * methods, care is taken to write only correct values to 'first'; intermediate and incomplete + * values do not get written to the field. This means that whenever 'first' is replaced, it is + * immediately replaced with the next correct value. Therefore, it is a safe volatile write. + * + * Other more complex relationships that need to be maintained during the mutate are maintained + * with the Object monitor. Since they do not impact the read path (only 'first' matters), the + * lock is sufficient for writes and unnecessary for 'first' reads. + * + * Documentation on volatile: + * http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/package-summary.html#MemoryVisibility + * (java.util.concurrent package docs) + */ + + private volatile T first = null; + private Set<T> rest = null; + + /* + * We may have a race where one thread tries to remove a small set from the map while another + * thread tries to add to it. If the second thread loses the race, it will add to a set that is + * no longer in the map. To prevent that, once a small set is ever empty, we mark it "dead" by + * setting {@code rest} to a {@code TOMBSTONE} value, and (and subsequently remove it from the + * map). No modifications to a set can happen after the {@code TOMBSTONE} value is set. Thus, + * the thread trying to add a new value to a set will fail, and knows to retrieve the entry anew + * from the map and try again. + */ + private static final Set<Object> TOMBSTONE = ImmutableSet.of(); + + /** + * Return some value in the SmallSet. + * + * <p>If there is always at least one value in the SmallSet during the lifetime of this call, + * it will not return null, since by the invariant, {@link #first} must be non-null. + */ + private T get() { + return first; + } + + /** + * Adds val to the SmallSet. Returns some element of the SmallSet. + */ + private synchronized T putAndGet(T elt) { + Preconditions.checkNotNull(elt); + if (isDead()) { + return null; + } + if (elt.equals(first)) { + return first; + } + if (first == null) { + Preconditions.checkState(rest == null, elt); + first = elt; + return first; + } + if (rest == null) { + rest = Sets.newHashSet(); + } + rest.add(elt); + return first; + } + + /** + * Remove val from the SmallSet, if it is present. + */ + private synchronized void remove(T elt) { + Preconditions.checkNotNull(elt); + if (isDead()) { + return; + } + if (elt.equals(first)) { + // Normalize to enforce invariant "first is null only if rest is empty." + if (rest != null) { + Iterator<T> it = rest.iterator(); + first = it.next(); + it.remove(); + if (!it.hasNext()) { + rest = null; + } + } else { + first = null; + markDead(); + } + } else if ((rest != null) && rest.remove(elt) && rest.isEmpty()) { // side-effect: remove + rest = null; + } + } + + private boolean isDead() { + Preconditions.checkState(rest != TOMBSTONE || first == null, + "%s present in tombstoned SmallSet, but tombstoned SmallSets should be empty", first); + return rest == TOMBSTONE; + } + + @SuppressWarnings("unchecked") // Cast of TOMBSTONE. Ok since TOMBSTONE is empty immutable set. + private void markDead() { + rest = (Set<T>) TOMBSTONE; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java b/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java new file mode 100644 index 0000000000..4b4048b93f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/DelegateSpawn.java @@ -0,0 +1,106 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; + +/** + * A delegating spawn that allow us to overwrite certain methods while maintaining the original + * behavior for non-overwritten methods. + */ +public class DelegateSpawn implements Spawn { + + private final Spawn spawn; + + public DelegateSpawn(Spawn spawn){ + this.spawn = spawn; + } + + @Override + public final ImmutableMap<String, String> getExecutionInfo() { + return spawn.getExecutionInfo(); + } + + @Override + public boolean isRemotable() { + return spawn.isRemotable(); + } + + @Override + public ImmutableList<Artifact> getFilesetManifests() { + return spawn.getFilesetManifests(); + } + + @Override + public String asShellCommand(Path workingDir) { + return spawn.asShellCommand(workingDir); + } + + @Override + public ImmutableMap<PathFragment, Artifact> getRunfilesManifests() { + return spawn.getRunfilesManifests(); + } + + @Override + public SpawnInfo getExtraActionInfo() { + return spawn.getExtraActionInfo(); + } + + @Override + public ImmutableList<String> getArguments() { + return spawn.getArguments(); + } + + @Override + public ImmutableMap<String, String> getEnvironment() { + return spawn.getEnvironment(); + } + + @Override + public Iterable<? extends ActionInput> getInputFiles() { + return spawn.getInputFiles(); + } + + @Override + public Collection<? extends ActionInput> getOutputFiles() { + return spawn.getOutputFiles(); + } + + @Override + public ActionMetadata getResourceOwner() { + return spawn.getResourceOwner(); + } + + @Override + public ResourceSet getLocalResources() { + return spawn.getLocalResources(); + } + + @Override + public ActionOwner getOwner() { + return spawn.getOwner(); + } + + @Override + public String getMnemonic() { + return spawn.getMnemonic(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java b/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java new file mode 100644 index 0000000000..92cc12b4c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/DigestOfDirectoryException.java @@ -0,0 +1,28 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import java.io.IOException; + +/** + * Exception thrown when we try to digest a directory in {@code ActionInputFileCache}. + * + */ +public class DigestOfDirectoryException extends IOException { + + public DigestOfDirectoryException(String message) { + super(message); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java b/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java new file mode 100644 index 0000000000..e2c493cf6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/EnvironmentalExecException.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An ExecException which is results from an external problem on the user's + * local system. + * + * <p>Note that this is fundamentally different exception then the higher level + * LocalEnvironmentException, which is thrown from the BuildTool. That exception + * is thrown when the higher levels of Blaze decide to exit. + * + * <p>This exception is thrown when a low level error is encountered in the + * strategy or client protocol layers. This does not necessarily mean we will + * exit; we may just retry the action. + */ +public class EnvironmentalExecException extends ExecException { + + public EnvironmentalExecException(String message, Throwable cause) { + super(message, cause); + } + + public EnvironmentalExecException(String message) { + super(message); + } + + public EnvironmentalExecException(String message, Throwable cause, boolean catastrophe) { + super(message, cause, catastrophe); + } + + public EnvironmentalExecException(String message, boolean catastrophe) { + super(message, catastrophe); + } + + @Override + public ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action) { + if (verboseFailures) { + return new ActionExecutionException(messagePrefix + " failed" + getMessage(), this, action, + isCatastrophic()); + } else { + return new ActionExecutionException(messagePrefix + " failed" + getMessage(), action, + isCatastrophic()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecException.java b/src/main/java/com/google/devtools/build/lib/actions/ExecException.java new file mode 100644 index 0000000000..a2edc84291 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecException.java @@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An exception indication that the execution of an action has failed OR could + * not be attempted OR could not be finished OR had something else wrong. + * + * <p>The four main kinds of failure are broadly defined as follows: + * + * <p>USER_INPUT which means it had something to do with what the user told us + * to do. This failure should satisfy the invariant that it would happen + * identically again if all other things are equal. + * + * <p>ENVIRONMENT which is loosely defined as anything which is generally out of + * scope for a blaze evaluation. As a rule of thumb, these are any errors would + * not necessarily happen again given constant input. + * + * <p>INTERRUPTION conditions arise from being unable to complete an evaluation + * for whatever reason. + * + * <p>INTERNAL_ERROR would happen because of anything which arises from within + * blaze itself but is generally unexpected to ever occur for any user input. + * + * <p>The class is a catch-all for both failures of actions and failures to + * evaluate actions properly. + * + * <p>Invariably, all low level ExecExceptions are caught by various specific + * ConfigurationAction classes and re-raised as ActionExecutionExceptions. + */ +public abstract class ExecException extends Exception { + + private final boolean catastrophe; + + public ExecException(String message, boolean catastrophe) { + super(message); + this.catastrophe = catastrophe; + } + + public ExecException(String message) { + this(message, false); + } + + public ExecException(String message, Throwable cause, boolean catastrophe) { + super(message + ": " + cause.getMessage(), cause); + this.catastrophe = catastrophe; + } + + public ExecException(String message, Throwable cause) { + this(message, cause, false); + } + + /** + * Catastrophic exceptions should stop the build, even if --keep_going. + */ + public boolean isCatastrophic() { + return catastrophe; + } + + /** + * Returns a new ActionExecutionException without a message prefix. + * @param action failed action + * @return ActionExecutionException object describing the action failure + */ + public ActionExecutionException toActionExecutionException(Action action) { + // In all ExecException implementations verboseFailures argument used only to determine should + // we pass ExecException as cause of ActionExecutionException. So use this method only + // if you need this information inside of ActionExecutionexception. + return toActionExecutionException("", true, action); + } + + /** + * Returns a new ActionExecutionException given a message prefix describing the action type as a + * noun. When appropriate (we use some heuristics to decide), produces an abbreviated message + * incorporating just the termination status if available. + * + * @param messagePrefix describes the action type as noun + * @param verboseFailures true if user requested verbose output with flag --verbose_failures + * @param action failed action + * @return ActionExecutionException object describing the action failure + */ + public abstract ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java new file mode 100644 index 0000000000..d86ea6c903 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutionStrategy.java @@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation that marks strategies that extend the execution phase behavior of Blaze. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface ExecutionStrategy { + /** + * The names this strategy is available under on the command line. + */ + String[] name() default {}; + + /** + * Returns the action context this strategy implements. + */ + Class<? extends ActionContext> contextType(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/Executor.java b/src/main/java/com/google/devtools/build/lib/actions/Executor.java new file mode 100644 index 0000000000..f3904bbf60 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Executor.java @@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsClassProvider; + +/** + * The Executor provides the context for the execution of actions. It is only valid during the + * execution phase, and references should not be cached. + * + * <p>This class provides the actual logic to execute actions. The platonic ideal of this system + * is that {@link Action}s are immutable objects that tell Blaze <b>what</b> to do and + * <link>ActionContext</link>s tell Blaze <b>how</b> to do it (however, we do have an "execute" + * method on actions now). + * + * <p>In theory, most of the methods below would not exist and they would be methods on action + * contexts, but in practice, that would require some refactoring work so we are stuck with these + * for the time being. + * + * <p>In theory, we could also merge {@link Executor} with {@link ActionExecutionContext}, since + * they both provide services to actions being executed and are passed to almost the same places. + */ +public interface Executor { + /** + * A marker interface for classes that provide services for actions during execution. + * + * <p>Interfaces extending this one should also be annotated with {@link ActionContextMarker}. + */ + public interface ActionContext { + } + + /** + * Returns the execution root. This is the directory underneath which Blaze builds its entire + * output working tree, including the source symlink forest. All build actions are executed + * relative to this directory. + */ + Path getExecRoot(); + + /** + * Returns a clock. This is not hermetic, and should only be used for build info actions or + * performance measurements / reporting. + */ + Clock getClock(); + + /** + * The EventBus for the current build. + */ + EventBus getEventBus(); + + /** + * Returns whether failures should have verbose error messages. + */ + boolean getVerboseFailures(); + + /** + * Returns the command line options of the Blaze command being executed. + */ + OptionsClassProvider getOptions(); + + /** + * Whether this Executor reports subcommands. If not, reportSubcommand has no effect. + * This is provided so the caller of reportSubcommand can avoid wastefully constructing the + * subcommand string. + */ + boolean reportsSubcommands(); + + /** + * Report a subcommand event to this Executor's Reporter and, if action + * logging is enabled, post it on its EventBus. + */ + void reportSubcommand(String reason, String message); + + /** + * An event listener to report messages to. Errors that signal a action failure should + * use ActionExecutionException. + */ + EventHandler getEventHandler(); + + /** + * Looks up and returns an action context implementation of the given interface type. + */ + <T extends ActionContext> T getContext(Class<? extends T> type); + + /** + * Returns the action context implementation for spawn actions with a given mnemonic. + */ + SpawnActionContext getSpawnActionContext(String mnemonic); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java b/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java new file mode 100644 index 0000000000..21369fb846 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ExecutorInitException.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; + +/** + * An exception that is thrown when an executor can't be initialized. + */ +public class ExecutorInitException extends AbruptExitException { + + public ExecutorInitException(String message) { + this(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR); + } + + public ExecutorInitException(String message, ExitCode exitCode) { + super(message, exitCode); + } + + public ExecutorInitException(String message, Throwable cause) { + super(message + ": " + cause.getMessage(), + ExitCode.LOCAL_ENVIRONMENTAL_ERROR, + cause); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/FailAction.java b/src/main/java/com/google/devtools/build/lib/actions/FailAction.java new file mode 100644 index 0000000000..6c15b31dac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FailAction.java @@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +/** + * FailAction is an Action that always fails to execute. (Used as scaffolding + * for rules we haven't yet implemented. Also useful for testing.) + */ +@ThreadSafe +public final class FailAction extends AbstractAction { + + private static final String GUID = "626cb78a-810f-4af3-979c-ee194955f04c"; + + private final String errorMessage; + + public FailAction(ActionOwner owner, Iterable<Artifact> outputs, String errorMessage) { + super(owner, ImmutableList.<Artifact>of(), outputs); + this.errorMessage = errorMessage; + } + + @Override + public Artifact getPrimaryInput() { + return null; + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + throw new ActionExecutionException(errorMessage, this, false); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + protected String computeKey() { + return GUID; + } + + @Override + protected String getRawProgressMessage() { + return "Building unsupported rule " + getOwner().getLabel() + + " located at " + getOwner().getLocation(); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public String getMnemonic() { + return "Fail"; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java new file mode 100644 index 0000000000..1ae15ba00c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetOutputSymlink.java @@ -0,0 +1,81 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** Definition of a symlink in the output tree of a Fileset rule. */ +public final class FilesetOutputSymlink { + private static final String STRIPPED_METADATA = "<stripped-for-testing>"; + + /** Final name of the symlink relative to the Fileset's output directory. */ + public final PathFragment name; + + /** Target of the symlink. Depending on FilesetEntry.symlinks it may be relative or absolute. */ + public final PathFragment target; + + /** Opaque metadata about the link and its target; should change if either of them changes. */ + public final String metadata; + + @VisibleForTesting + public FilesetOutputSymlink(PathFragment name, PathFragment target) { + this.name = name; + this.target = target; + this.metadata = STRIPPED_METADATA; + } + + /** + * @param name relative path under the Fileset's output directory, including FilesetEntry.destdir + * with and FilesetEntry.strip_prefix applied (if applicable) + * @param target relative or absolute value of the link + * @param metadata opaque metadata about the link and its target; should change if either the link + * or its target changes + */ + public FilesetOutputSymlink(PathFragment name, PathFragment target, String metadata) { + this.name = Preconditions.checkNotNull(name); + this.target = Preconditions.checkNotNull(target); + this.metadata = Preconditions.checkNotNull(metadata); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || !obj.getClass().equals(getClass())) { + return false; + } + FilesetOutputSymlink o = (FilesetOutputSymlink) obj; + return name.equals(o.name) && target.equals(o.target) && metadata.equals(o.metadata); + } + + @Override + public int hashCode() { + return Objects.hashCode(name, target, metadata); + } + + @Override + public String toString() { + if (metadata.equals(STRIPPED_METADATA)) { + return String.format("FilesetOutputSymlink(%s -> %s)", + name.getPathString(), target.getPathString()); + } else { + return String.format("FilesetOutputSymlink(%s -> %s | metadata=%s)", + name.getPathString(), target.getPathString(), metadata); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java new file mode 100644 index 0000000000..1464f73143 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParams.java @@ -0,0 +1,165 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Optional; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; + +import java.util.Set; + +/** + * Parameters of a filesystem traversal requested by a Fileset rule. + * + * <p>This object stores the details of the traversal request, e.g. whether it's a direct or nested + * traversal (see {@link #getDirectTraversal()} and {@link #getNestedTraversal()}) or who the owner + * of the traversal is. + */ +public interface FilesetTraversalParams { + + /** + * Abstraction of the root directory of a {@link DirectTraversal}. + * + * <ul> + * <li>The root of package traversals is the package directory, i.e. the parent of the BUILD file. + * <li>The root of "recursive" directory traversals is the directory's path. + * <li>The root of "file" traversals is the path of the file (or directory, or symlink) itself. + * </ul> + * + * <p>For the meaning of "recursive" and "file" traversals see {@link DirectTraversal}. + */ + interface DirectTraversalRoot { + + /** + * Returns the root part of the full path. + * + * <p>This is typically the workspace root or some output tree's root (e.g. genfiles, binfiles). + */ + Path getRootPart(); + + /** + * Returns the {@link #getRootPart() root}-relative part of the path. + * + * <p>This is typically the source directory under the workspace or the output file under an + * output directory. + */ + PathFragment getRelativePart(); + + /** Returns a {@link RootedPath} composed of the root and relative parts. */ + RootedPath asRootedPath(); + } + + /** + * Describes a request for a direct filesystem traversal. + * + * <p>"Direct" means this corresponds to an actual filesystem traversal as opposed to traversing + * another Fileset rule, which is called a "nested" traversal. + * + * <p>Direct traversals can further be divided into two categories, "file" traversals and + * "recursive" traversals. + * + * <p>File traversal requests are created when the FilesetEntry.files attribute is defined; one + * file traversal request is created for each entry. + * + * <p>Recursive traversal requests are created when the FilesetEntry.files attribute is + * unspecified; one recursive traversal request is created for the FilesetEntry.srcdir. + * + * <p>See {@link DirectTraversal#getRoot()} for more details. + */ + interface DirectTraversal { + + /** Returns the root of the traversal; see {@link DirectTraversalRoot}. */ + DirectTraversalRoot getRoot(); + + /** + * Returns true if this traversal refers to a whole package. + * + * <p>In that case the root (see {@link #getRoot()}) refers to the path of the package. + * + * <p>Package traversals are always recursive (see {@link #isRecursive()}) and are never + * generated (see {@link #isGenerated()}). + */ + boolean isPackage(); + + /** + * Returns true if this is a "recursive traversal", i.e. created from FilesetEntry.srcdir. + * + * <p>This type of traversal is created when the FilesetEntry doesn't define a "files" list. + * When it does, the traversal is referred to as a "file traversal". When it doesn't, but the + * srcdir points to another Fileset, it is called a "nested" traversal. + * + * <p>Recursive traversals got their name from recursively traversing a directory structure. + * These are usually whole-package traversals, i.e. when FilesetEntry.srcdir refers to a BUILD + * file (see {@link #isPackage()}), but sometimes the srcdir references a input or output + * directory (the latter being generated by a local genrule) or a symlink (which must point to a + * directory; enforced during action execution). + * + * <p>The files in the results of a recursive traversal are all under the {@link #getRoot() + * root}. The root's path is stripped from the results. + * + * <p>N.B.: "file traversals" can also be recursive if the entry in FilesetEntry.files, for + * which the traversal parameters were created, turned out to be a directory. The difference + * lies in how the output paths are computed (with recursive traversals, the directory's name + * is stripped; with file traversals it is not, modulo usage of strip_prefix and the excludes + * attributes), and how directory symlinks are handled (in "recursive traversals" they are + * expanded just like normal directories, subsequent directory symlinks under them are *not* + * expanded though; they are not expanded at all in "file traversals"). + */ + boolean isRecursive(); + + /** Returns true if the root points to a generated file, symlink or directory. */ + boolean isGenerated(); + + /** Returns true if input symlinks should be dereferenced; false if copied. */ + boolean isFollowingSymlinks(); + + /** Returns the desired behavior when the traversal hits a subpackage. */ + boolean getCrossPackageBoundary(); + } + + /** Label of the Fileset rule that owns this traversal. */ + Label getOwnerLabel(); + + /** Returns the directory under the output path where the files will be mapped. May be empty. */ + PathFragment getDestPath(); + + /** Returns a list of file basenames to be excluded from the output. May be empty. */ + Set<String> getExcludedFiles(); + + /** + * Returns the parameters of the direct traversal request, if any. + * + * <p>A direct traversal is anything that's not a nested traversal, e.g. traversal of a package or + * directory (when FilesetEntry.srcdir is specified) or traversal of a single file (when + * FilesetEntry.files is specified). See {@link DirectTraversal} for more detail. + * + * <p>The value is present if and only if {@link #getNestedTraversal} is absent. + */ + Optional<DirectTraversal> getDirectTraversal(); + + /** + * Returns the parameters of the nested traversal request, if any. + * + * <p>A nested traversal is the traversal of another Fileset referenced by FilesetEntry.srcdir. + * + * <p>The value is present if and only if {@link #getDirectTraversal} is absent. + */ + Optional<FilesetTraversalParams> getNestedTraversal(); + + /** Adds the fingerprint of this traversal object. */ + void fingerprint(Fingerprint fp); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java new file mode 100644 index 0000000000..acc0e86a51 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/FilesetTraversalParamsFactory.java @@ -0,0 +1,314 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversal; +import com.google.devtools.build.lib.actions.FilesetTraversalParams.DirectTraversalRoot; +import com.google.devtools.build.lib.syntax.FilesetEntry.SymlinkBehavior; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; + +import java.util.Set; + +import javax.annotation.Nullable; + +/** Factory of {@link FilesetTraversalParams}. */ +public final class FilesetTraversalParamsFactory { + + /** + * Creates parameters for a recursive traversal request in a package. + * + * <p>"Recursive" means that a directory is traversed along with all of its subdirectories. Such + * a traversal is created when FilesetEntry.files is unspecified. + * + * @param ownerLabel the rule that created this object + * @param buildFile path of the BUILD file of the package to traverse + * @param destPath path in the Fileset's output directory that will be the root of files found + * in this directory + * @param excludes optional; set of files directly under this package's directory to exclude; + * files in subdirectories cannot be excluded + * @param symlinkBehaviorMode what to do with symlinks + * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains + * a BUILD file) + */ + public static FilesetTraversalParams recursiveTraversalOfPackage(Label ownerLabel, + Artifact buildFile, PathFragment destPath, @Nullable Set<String> excludes, + SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) { + Preconditions.checkState(buildFile.isSourceArtifact(), "%s", buildFile); + return new DirectoryTraversalParams(ownerLabel, DirectTraversalRootImpl.forPackage(buildFile), + true, destPath, excludes, symlinkBehaviorMode, crossPkgBoundary, true, false); + } + + /** + * Creates parameters for a recursive traversal request in a directory. + * + * <p>"Recursive" means that a directory is traversed along with all of its subdirectories. Such + * a traversal is created when FilesetEntry.files is unspecified. + * + * @param ownerLabel the rule that created this object + * @param directoryToTraverse path of the directory to traverse + * @param destPath path in the Fileset's output directory that will be the root of files found + * in this directory + * @param excludes optional; set of files directly below this directory to exclude; files in + * subdirectories cannot be excluded + * @param symlinkBehaviorMode what to do with symlinks + * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains + * a BUILD file) + */ + public static FilesetTraversalParams recursiveTraversalOfDirectory(Label ownerLabel, + Artifact directoryToTraverse, PathFragment destPath, @Nullable Set<String> excludes, + SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) { + return new DirectoryTraversalParams(ownerLabel, + DirectTraversalRootImpl.forFileOrDirectory(directoryToTraverse), false, destPath, + excludes, symlinkBehaviorMode, crossPkgBoundary, true, + !directoryToTraverse.isSourceArtifact()); + } + + /** + * Creates parameters for a file traversal request. + * + * <p>Such a traversal is created for every entry in FilesetEntry.files, when it is specified. + * + * @param ownerLabel the rule that created this object + * @param fileToTraverse the file to traverse; "traversal" means that if this file is actually a + * directory or a symlink to one then it'll be traversed as one + * @param destPath path in the Fileset's output directory that will be the name of this file's + * respective symlink there, or the root of files found (in case this is a directory) + * @param symlinkBehaviorMode what to do with symlinks + * @param crossPkgBoundary whether to traverse a subdirectory if it's also a subpackage (contains + * a BUILD file) + */ + public static FilesetTraversalParams fileTraversal(Label ownerLabel, Artifact fileToTraverse, + PathFragment destPath, SymlinkBehavior symlinkBehaviorMode, boolean crossPkgBoundary) { + return new DirectoryTraversalParams(ownerLabel, + DirectTraversalRootImpl.forFileOrDirectory(fileToTraverse), false, destPath, null, + symlinkBehaviorMode, crossPkgBoundary, false, !fileToTraverse.isSourceArtifact()); + } + + /** + * Creates traversal request parameters for a FilesetEntry wrapping another Fileset. + * + * @param ownerLabel the rule that created this object + * @param nested the traversal params that were used for the nested (inner) Fileset + * @param destDir path in the Fileset's output directory that will be the root of files coming + * from the nested Fileset + * @param excludes optional; set of files directly below (not in a subdirectory of) the nested + * Fileset that should be excluded from the outer Fileset + */ + public static FilesetTraversalParams nestedTraversal(Label ownerLabel, + FilesetTraversalParams nested, PathFragment destDir, @Nullable Set<String> excludes) { + // When srcdir is another Fileset, then files must be null so strip_prefix must also be null. + return new NestedTraversalParams(ownerLabel, nested, destDir, excludes); + } + + private abstract static class ParamsCommon implements FilesetTraversalParams { + private final Label ownerLabel; + private final PathFragment destDir; + private final ImmutableSet<String> excludes; + + ParamsCommon(Label ownerLabel, PathFragment destDir, @Nullable Set<String> excludes) { + this.ownerLabel = ownerLabel; + this.destDir = destDir; + if (excludes == null) { + this.excludes = ImmutableSet.<String>of(); + } else { + // Order the set for the sake of deterministic fingerprinting. + this.excludes = ImmutableSet.copyOf(Ordering.natural().immutableSortedCopy(excludes)); + } + } + + @Override + public Label getOwnerLabel() { + return ownerLabel; + } + + @Override + public Set<String> getExcludedFiles() { + return excludes; + } + + @Override + public PathFragment getDestPath() { + return destDir; + } + + protected final void commonFingerprint(Fingerprint fp) { + fp.addPath(destDir); + if (!excludes.isEmpty()) { + fp.addStrings(excludes); + } + } + } + + private static final class DirectTraversalImpl implements DirectTraversal { + private final DirectTraversalRoot root; + private final boolean isPackage; + private final boolean followSymlinks; + private final boolean crossPkgBoundary; + private final boolean isRecursive; + private final boolean isGenerated; + + DirectTraversalImpl(DirectTraversalRoot root, boolean isPackage, boolean followSymlinks, + boolean crossPkgBoundary, boolean isRecursive, boolean isGenerated) { + this.root = root; + this.isPackage = isPackage; + this.followSymlinks = followSymlinks; + this.crossPkgBoundary = crossPkgBoundary; + this.isRecursive = isRecursive; + this.isGenerated = isGenerated; + } + + @Override + public DirectTraversalRoot getRoot() { + return root; + } + + @Override + public boolean isPackage() { + return isPackage; + } + + @Override + public boolean isRecursive() { + return isRecursive; + } + + @Override + public boolean isGenerated() { + return isGenerated; + } + + @Override + public boolean isFollowingSymlinks() { + return followSymlinks; + } + + @Override + public boolean getCrossPackageBoundary() { + return crossPkgBoundary; + } + + void fingerprint(Fingerprint fp) { + fp.addPath(root.asRootedPath().asPath()); + fp.addBoolean(isPackage); + fp.addBoolean(followSymlinks); + fp.addBoolean(isRecursive); + fp.addBoolean(isGenerated); + fp.addBoolean(crossPkgBoundary); + } + } + + private static final class DirectoryTraversalParams extends ParamsCommon { + private final DirectTraversalImpl traversal; + + DirectoryTraversalParams(Label ownerLabel, + DirectTraversalRoot root, + boolean isPackage, + PathFragment destPath, + @Nullable Set<String> excludes, + SymlinkBehavior symlinkBehaviorMode, + boolean crossPkgBoundary, + boolean isRecursive, + boolean isGenerated) { + super(ownerLabel, destPath, excludes); + traversal = new DirectTraversalImpl(root, isPackage, + symlinkBehaviorMode == SymlinkBehavior.DEREFERENCE, crossPkgBoundary, isRecursive, + isGenerated); + } + + @Override + public Optional<DirectTraversal> getDirectTraversal() { + return Optional.<DirectTraversal>of(traversal); + } + + @Override + public Optional<FilesetTraversalParams> getNestedTraversal() { + return Optional.absent(); + } + + @Override + public void fingerprint(Fingerprint fp) { + commonFingerprint(fp); + traversal.fingerprint(fp); + } + } + + private static final class NestedTraversalParams extends ParamsCommon { + private final FilesetTraversalParams nested; + + public NestedTraversalParams(Label ownerLabel, FilesetTraversalParams nested, + PathFragment destDir, @Nullable Set<String> excludes) { + super(ownerLabel, destDir, excludes); + this.nested = nested; + } + + @Override + public Optional<DirectTraversal> getDirectTraversal() { + return Optional.absent(); + } + + @Override + public Optional<FilesetTraversalParams> getNestedTraversal() { + return Optional.of(nested); + } + + @Override + public void fingerprint(Fingerprint fp) { + commonFingerprint(fp); + nested.fingerprint(fp); + } + } + + private static final class DirectTraversalRootImpl implements DirectTraversalRoot { + private final Path rootDir; + private final PathFragment relativeDir; + + static DirectTraversalRoot forPackage(Artifact buildFile) { + return new DirectTraversalRootImpl(buildFile.getRoot().getPath(), + buildFile.getRootRelativePath().getParentDirectory()); + } + + static DirectTraversalRoot forFileOrDirectory(Artifact fileOrDirectory) { + return new DirectTraversalRootImpl(fileOrDirectory.getRoot().getPath(), + fileOrDirectory.getRootRelativePath()); + } + + private DirectTraversalRootImpl(Path rootDir, PathFragment relativeDir) { + this.rootDir = rootDir; + this.relativeDir = relativeDir; + } + + @Override + public Path getRootPart() { + return rootDir; + } + + @Override + public PathFragment getRelativePart() { + return relativeDir; + } + + @Override + public RootedPath asRootedPath() { + return RootedPath.toRootedPath(rootDir, relativeDir); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java new file mode 100644 index 0000000000..5fc6013490 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java @@ -0,0 +1,302 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.io.Files; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.ProcMeminfoParser; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * This class estimates the local host's resource capacity. + */ +@ThreadCompatible +public final class LocalHostCapacity { + + private static final Logger LOG = Logger.getLogger(LocalHostCapacity.class.getName()); + + /** + * Stores parsed /proc/stat CPU time counters. + * See {@link LocalHostCapacity#getCpuTimes(String)} for details. + */ + @Immutable + private final static class CpuTimes { + private final long idleJiffies; + private final long totalJiffies; + + CpuTimes(long idleJiffies, long totalJiffies) { + this.idleJiffies = idleJiffies; + this.totalJiffies = totalJiffies; + } + + /** + * Return idle CPU ratio using current and previous CPU readings or 0 if + * ratio is undefined. + */ + double getIdleRatio(CpuTimes prevTimes) { + if (prevTimes.totalJiffies == 0 || totalJiffies == prevTimes.totalJiffies) { + return 0; + } + return ((double)(idleJiffies - prevTimes.idleJiffies) / + (double)(totalJiffies - prevTimes.totalJiffies)); + } + } + + /** + * Used to store available local CPU and RAM resources information. + * See {@link LocalHostCapacity#getFreeResources(FreeResources)} for details. + */ + public static final class FreeResources { + + private final Clock clock; + private final CpuTimes cpuTimes; + private final long lastTimestamp; + private final double freeCpu; + private final double freeMb; + private final long interval; + + private FreeResources(Clock localClock, ProcMeminfoParser memInfo, String statContent, + FreeResources prevStats) { + clock = localClock; + lastTimestamp = localClock.nanoTime(); + freeMb = ProcMeminfoParser.kbToMb(memInfo.getFreeRamKb()); + cpuTimes = getCpuTimes(statContent); + if (prevStats == null) { + interval = 0; + freeCpu = 0.0; + } else { + interval = lastTimestamp - prevStats.lastTimestamp; + freeCpu = getLocalHostCapacity().getCpuUsage() * cpuTimes.getIdleRatio(prevStats.cpuTimes); + } + } + + /** + * Returns amount of available RAM in MB. + */ + public double getFreeMb() { return freeMb; } + + /** + * Returns average available CPU resources (as a fraction of the CPU core, + * so one fully CPU-bound thread should consume exactly 1.0 CPU resource). + */ + public double getAvgFreeCpu() { return freeCpu; } + + /** + * Returns interval in ms between CPU load measurements used to calculate + * average available CPU resources. + */ + public long getInterval() { return interval / 1000000; } + + /** + * Returns age of available resource data in ms. + */ + public long getReadingAge() { + return (clock.nanoTime() - lastTimestamp) / 1000000; + } + } + + // Disables getFreeResources() if error occured during reading or parsing + // /proc/* information. + @VisibleForTesting + static boolean isDisabled; + + // If /proc/* information is not available, assume 3000 MB and 2 CPUs. + private static ResourceSet DEFAULT_RESOURCES = new ResourceSet(3000.0, 2.0, 1.0); + + private LocalHostCapacity() {} + + /** + * Estimates of the local host's resource capacity, + * obtained by reading /proc/cpuinfo and /proc/meminfo. + */ + private static ResourceSet localHostCapacity; + + /** + * Estimates of the local host's resource capacity, + * obtained by reading /proc/cpuinfo and /proc/meminfo. + */ + public static ResourceSet getLocalHostCapacity() { + if (localHostCapacity == null) { + localHostCapacity = getLocalHostCapacity("/proc/cpuinfo", "/proc/meminfo"); + } + return localHostCapacity; + } + + /** + * Returns new FreeResources object populated with free RAM information from + * /proc/meminfo and CPU load information from the /proc/stat. First call + * should be made with null parameter to instantiate new FreeResources object. + * Subsequent calls will use information inside it to calculate average CPU + * load over the time between calls and to calculate amount of free CPU + * resources and generate new FreeResources() instance. + * + * If information is not available due to error, functionality will be disabled + * and method will always return null. + */ + public static FreeResources getFreeResources(FreeResources stats) { + return getFreeResources(BlazeClock.instance(), "/proc/meminfo", "/proc/stat", stats); + } + + private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings(); + + @VisibleForTesting + static int getLogicalCpuCount(String cpuinfoContent) { + Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent); + int count = 0; + for (String line : lines) { + if(line.startsWith("processor")) { + count++; + } + } + if (count == 0) { + throw new IllegalArgumentException("Can't locate processor in the /proc/cpuinfo"); + } + return count; + } + + @VisibleForTesting + static int getPhysicalCpuCount(String cpuinfoContent, int logicalCpuCount) { + Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoContent); + Set<String> uniq = new HashSet<>(); + for (String line : lines) { + if(line.startsWith("physical id")) { + uniq.add(line); + } + } + int physicalCpuCount = uniq.size(); + if (physicalCpuCount == 0) { + physicalCpuCount = logicalCpuCount; + } + return physicalCpuCount; + } + + @VisibleForTesting + static int getCoresPerCpu(String cpuinfoFileContent) { + Iterable<String> lines = NEWLINE_SPLITTER.split(cpuinfoFileContent); + Set<String> uniq = new HashSet<>(); + for (String line : lines) { + if(line.startsWith("core id")) { + uniq.add(line); + } + } + int coresPerCpu = uniq.size(); + if (coresPerCpu == 0) { + coresPerCpu = 1; + } + return coresPerCpu; + } + + /** + * Parses cpu line of the /proc/stats, calculates number of idle and total + * CPU jiffies and returns CpuTimes instance with that information. + * + * Total CPU time includes <b>all</b> time reported to be spent by the CPUs, + * including so-called "stolen" time - time spent by other VMs on the same + * workstation. + */ + private static CpuTimes getCpuTimes(String statContent) { + String[] cpuStats = statContent.substring(0, statContent.indexOf('\n')).trim().split(" +"); + // Supported versions of /proc/stat (Linux kernel 2.6.x) must contain either + // 9 or 10 fields: + // "cpu" utime ultime stime idle iowait irq softirq steal(since 2.6.11) 0 + // We are interested in total time (sum of all columns) and idle time. + if (cpuStats.length < 9 | cpuStats.length > 10) { + throw new IllegalArgumentException("Unrecognized /proc/stat format"); + } + if (!cpuStats[0].equals("cpu")) { + throw new IllegalArgumentException("/proc/stat does not start with cpu keyword"); + } + long idleCpuJiffies = Long.parseLong(cpuStats[4]); // "idle" column. + long totalJiffies = 0; + for (int i = 1; i < cpuStats.length; i++) { + totalJiffies += Long.parseLong(cpuStats[i]); + } + long totalCpuJiffies = totalJiffies; + return new CpuTimes(idleCpuJiffies, totalCpuJiffies); + } + + @VisibleForTesting + static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) { + try { + String cpuinfoContent = readContent(cpuinfoFile); + ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile); + int logicalCpuCount = getLogicalCpuCount(cpuinfoContent); + int physicalCpuCount = getPhysicalCpuCount(cpuinfoContent, logicalCpuCount); + int coresPerCpu = getCoresPerCpu(cpuinfoContent); + int totalCores = coresPerCpu * physicalCpuCount; + boolean hyperthreading = (logicalCpuCount != totalCores); + double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb()); + final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6; + return new ResourceSet( + ramMb, + logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU + : 1.0), + 1.0); + } catch (IOException | IllegalArgumentException e) { + disableProcFsUse(e); + return DEFAULT_RESOURCES; + } + } + + @VisibleForTesting + static FreeResources getFreeResources(Clock localClock, String meminfoFile, String statFile, + FreeResources prevStats) { + if (isDisabled) { return null; } + try { + String statContent = readContent(statFile); + return new FreeResources(localClock, new ProcMeminfoParser(meminfoFile), + statContent, prevStats); + } catch (IOException | IllegalArgumentException e) { + disableProcFsUse(e); + return null; + } + } + + /** + * For testing purposes only. Do not use it. + */ + @VisibleForTesting + static void setLocalHostCapacity(ResourceSet resources) { + localHostCapacity = resources; + isDisabled = false; + } + + private static String readContent(String filename) throws IOException { + return Files.toString(new File(filename), Charset.defaultCharset()); + } + + /** + * Disables use of /proc filesystem. Called internally when unexpected + * exception is caught. + */ + private static void disableProcFsUse(Throwable cause) { + LoggingUtil.logToRemote(Level.WARNING, "Unable to read system load or capacity", cause); + LOG.log(Level.WARNING, "Unable to read system load or capacity", cause); + isDisabled = true; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java new file mode 100644 index 0000000000..2788f2f307 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MapBasedActionGraph.java @@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * An action graph that resolves generating actions by looking them up in a map. + */ +@ThreadSafe +public final class MapBasedActionGraph implements MutableActionGraph { + + private final ConcurrentMultimapWithHeadElement<Artifact, Action> generatingActionMap = + new ConcurrentMultimapWithHeadElement<Artifact, Action>(); + + @Override + @Nullable + public Action getGeneratingAction(Artifact artifact) { + return generatingActionMap.get(artifact); + } + + @Override + public void registerAction(Action action) throws ActionConflictException { + for (Artifact artifact : action.getOutputs()) { + Action previousAction = generatingActionMap.putAndGet(artifact, action); + if (previousAction != null && previousAction != action + && !Actions.canBeShared(action, previousAction)) { + generatingActionMap.remove(artifact, action); + throw new ActionConflictException(artifact, previousAction, action); + } + } + } + + @Override + public void unregisterAction(Action action) { + for (Artifact artifact : action.getOutputs()) { + generatingActionMap.remove(artifact, action); + Action otherAction = generatingActionMap.get(artifact); + Preconditions.checkState(otherAction == null + || (otherAction != action && Actions.canBeShared(action, otherAction)), + "%s %s", action, otherAction); + } + } + + @Override + public void clear() { + generatingActionMap.clear(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java new file mode 100644 index 0000000000..9d3a2b8c05 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanAction.java @@ -0,0 +1,107 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.syntax.Label; + +/** + * An action that depends on a set of inputs and creates a single output file whenever it + * runs. This is useful for bundling up a bunch of dependencies that are shared + * between individual targets in the action graph; for example generated header files. + */ +public class MiddlemanAction extends AbstractAction { + + public static final String MIDDLEMAN_MNEMONIC = "Middleman"; + private final String description; + private final MiddlemanType middlemanType; + + /** + * Constructs a new {@link MiddlemanAction}. + * + * @param owner the owner of the action, usually a {@code ConfiguredTarget} + * @param inputs inputs of the middleman, i.e. the files it acts as a placeholder for + * @param stampFile the output of the middleman expansion; must be a middleman artifact (see + * {@link Artifact#isMiddlemanArtifact()}) + * @param description a short description for the action, for progress messages + * @param middlemanType the type of the middleman + * @throws IllegalArgumentException if {@code stampFile} is not a middleman artifact + */ + public MiddlemanAction(ActionOwner owner, Iterable<Artifact> inputs, Artifact stampFile, + String description, MiddlemanType middlemanType) { + super(owner, inputs, ImmutableList.of(stampFile)); + Preconditions.checkNotNull(middlemanType); + Preconditions.checkArgument(stampFile.isMiddlemanArtifact(), stampFile); + this.description = description; + this.middlemanType = middlemanType; + } + + @Override + public final void execute( + ActionExecutionContext actionExecutionContext) { + throw new IllegalStateException("MiddlemanAction should never be executed"); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + protected String computeKey() { + // TODO(bazel-team): Need to take middlemanType into account here. + // Only the set of inputs matters, and the dependency checker is + // responsible for considering those. + return ""; + } + + /** + * Returns the type of the middleman. + */ + @Override + public MiddlemanType getActionType() { + return middlemanType; + } + + @Override + protected String getRawProgressMessage() { + return null; // users don't really want to know about Middlemen. + } + + @Override + public String prettyPrint() { + return description + " for " + Label.print(getOwner().getLabel()); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public String getMnemonic() { + return MIDDLEMAN_MNEMONIC; + } + + /** + * Creates a new middleman action. + */ + public static Action create(ActionRegistry env, ActionOwner owner, + Iterable<Artifact> inputs, Artifact stampFile, String purpose, MiddlemanType middlemanType) { + MiddlemanAction action = new MiddlemanAction(owner, inputs, stampFile, purpose, middlemanType); + env.registerAction(action); + return action; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java new file mode 100644 index 0000000000..85a1450344 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MiddlemanFactory.java @@ -0,0 +1,188 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Iterator; + +/** + * A factory to create middleman objects. + */ +@ThreadSafe +public final class MiddlemanFactory { + + private final ArtifactFactory artifactFactory; + private final ActionRegistry actionRegistry; + + public MiddlemanFactory( + ArtifactFactory artifactFactory, ActionRegistry actionRegistry) { + this.artifactFactory = Preconditions.checkNotNull(artifactFactory); + this.actionRegistry = Preconditions.checkNotNull(actionRegistry); + } + + /** + * Creates a {@link MiddlemanType#AGGREGATING_MIDDLEMAN aggregating} middleman. + * + * @param owner the owner of the action that will be created; must not be null + * @param purpose the purpose for which this middleman is created. This should be a string which + * is suitable for use as a filename. A single rule may have many middlemen with distinct + * purposes. + * @param inputs the set of artifacts for which the created artifact is to be the middleman. + * @param middlemanDir the directory in which to place the middleman. + * @return null iff {@code inputs} is empty; the single element of {@code inputs} if there's only + * one; a new aggregating middleman for the {@code inputs} otherwise + */ + public Artifact createAggregatingMiddleman( + ActionOwner owner, String purpose, Iterable<Artifact> inputs, Root middlemanDir) { + if (hasExactlyOneInput(inputs)) { // Optimization: No middleman for just one input. + return Iterables.getOnlyElement(inputs); + } + Pair<Artifact, Action> result = createMiddleman( + owner, Label.print(owner.getLabel()), purpose, inputs, middlemanDir, + MiddlemanType.AGGREGATING_MIDDLEMAN); + return result == null ? null : result.getFirst(); + } + + /** + * Returns <code>null</code> iff inputs is empty. Returns the sole element + * of inputs iff <code>inputs.size()==1</code>. Otherwise, returns a + * middleman artifact and creates a middleman action that generates that + * artifact. + * + * @param owner the owner of the action that will be created. + * @param owningArtifact the artifact of the file for which the runfiles + * should be created. There may be at most one set of runfiles for + * an owning artifact, unless the owning artifact is null. There + * may be at most one set of runfiles per owner with a null + * owning artifact. + * Further, if the owning Artifact is non-null, the owning Artifacts' + * root-relative path must be unique and the artifact must be part + * of the runfiles tree for which this middleman is created. Usually + * this artifact will be an executable program. + * @param inputs the set of artifacts for which the created artifact is to be + * the middleman. + * @param middlemanDir the directory in which to place the middleman. + */ + public Artifact createRunfilesMiddleman( + ActionOwner owner, Artifact owningArtifact, Iterable<Artifact> inputs, Root middlemanDir) { + if (hasExactlyOneInput(inputs)) { // Optimization: No middleman for just one input. + return Iterables.getOnlyElement(inputs); + } + String middlemanPath = owningArtifact == null + ? Label.print(owner.getLabel()) + : owningArtifact.getRootRelativePath().getPathString(); + return createMiddleman(owner, middlemanPath, "runfiles", inputs, middlemanDir, + MiddlemanType.RUNFILES_MIDDLEMAN).getFirst(); + } + + private <T> boolean hasExactlyOneInput(Iterable<T> iterable) { + Iterator<T> it = iterable.iterator(); + if (!it.hasNext()) { + return false; + } + it.next(); + return !it.hasNext(); + } + + /** + * Creates a {@link MiddlemanType#ERROR_PROPAGATING_MIDDLEMAN error-propagating} middleman. + * + * @param owner the owner of the action that will be created. May not be null. + * @param middlemanName a unique file name for the middleman artifact in the {@code middlemanDir}; + * in practice this is usually the owning rule's label (so it gets escaped as such) + * @param purpose the purpose for which this middleman is created. This should be a string which + * is suitable for use as a filename. A single rule may have many middlemen with distinct + * purposes. + * @param inputs the set of artifacts for which the created artifact is to be the middleman; must + * not be null or empty + * @param middlemanDir the directory in which to place the middleman. + * @return a middleman that enforces scheduling order (just like a scheduling middleman) and + * propagates errors, but is ignored by the dependency checker + * @throws IllegalArgumentException if {@code inputs} is null or empty + */ + public Artifact createErrorPropagatingMiddleman(ActionOwner owner, String middlemanName, + String purpose, Iterable<Artifact> inputs, Root middlemanDir) { + Preconditions.checkArgument(inputs != null); + Preconditions.checkArgument(!Iterables.isEmpty(inputs)); + // We must always create this middleman even if there is only one input. + return createMiddleman(owner, middlemanName, purpose, inputs, middlemanDir, + MiddlemanType.ERROR_PROPAGATING_MIDDLEMAN).getFirst(); + } + + /** + * Returns the same artifact as {@code createErrorPropagatingMiddleman} would return, + * but doesn't create any action. + */ + public Artifact getErrorPropagatingMiddlemanArtifact(String middlemanName, String purpose, + Root middlemanDir) { + return getStampFileArtifact(middlemanName, purpose, middlemanDir); + } + + /** + * Creates both normal and scheduling middlemen. + * + * <p>Note: there's no need to synchronize this method; the only use of a field is via a call to + * another synchronized method (getArtifact()). + * + * @return null iff {@code inputs} is null or empty; the middleman file and the middleman action + * otherwise + */ + private Pair<Artifact, Action> createMiddleman( + ActionOwner owner, String middlemanName, String purpose, Iterable<Artifact> inputs, + Root middlemanDir, MiddlemanType middlemanType) { + if (inputs == null || Iterables.isEmpty(inputs)) { + return null; + } + + Artifact stampFile = getStampFileArtifact(middlemanName, purpose, middlemanDir); + Action action = new MiddlemanAction(owner, inputs, stampFile, purpose, middlemanType); + actionRegistry.registerAction(action); + return Pair.of(stampFile, action); + } + + /** + * Creates a normal middleman. + * + * <p>If called multiple times, it always returns the same object depending on the {@code + * purpose}. It does not check that the list of inputs is identical. In contrast to other + * middleman methods, this one also returns an object if the list of inputs is empty. + * + * <p>Note: there's no need to synchronize this method; the only use of a field is via a call to + * another synchronized method (getArtifact()). + */ + public Artifact createMiddlemanAllowMultiple(ActionRegistry registry, + ActionOwner owner, String purpose, Iterable<Artifact> inputs, Root middlemanDir) { + PathFragment stampName = new PathFragment("_middlemen/" + purpose); + Artifact stampFile = artifactFactory.getDerivedArtifact(stampName, middlemanDir, + actionRegistry.getOwner()); + MiddlemanAction.create( + registry, owner, inputs, stampFile, purpose, MiddlemanType.AGGREGATING_MIDDLEMAN); + return stampFile; + } + + private Artifact getStampFileArtifact(String middlemanName, String purpose, Root middlemanDir) { + String escapedFilename = Actions.escapedPath(middlemanName); + PathFragment stampName = new PathFragment("_middlemen/" + escapedFilename + "-" + purpose); + Artifact stampFile = artifactFactory.getDerivedArtifact(stampName, middlemanDir, + actionRegistry.getOwner()); + return stampFile; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java b/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java new file mode 100644 index 0000000000..52f9f2734e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MissingInputFileException.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.events.Location; + +/** + * This exception is thrown during a build when an input file is missing, but the file + * is not the input to any action being executed. + * + * If a missing input file is an input + * to an action, an {@link ActionExecutionException} is thrown instead. + */ +public class MissingInputFileException extends BuildFailedException { + private final Location location; + + public MissingInputFileException(String message, Location location) { + super(message); + this.location = location; + } + + /** + * Return a location where this input file is referenced. If there + * are multiple such locations, one is chosen arbitrarily. If there + * are none, return null. + */ + public Location getLocation() { + return location; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java b/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java new file mode 100644 index 0000000000..8b84c3112c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/MutableActionGraph.java @@ -0,0 +1,151 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.StringUtil; + +import java.util.Set; + +/** + * A mutable action graph. Implementations of this interface must be thread-safe. + */ +public interface MutableActionGraph extends ActionGraph { + + /** + * Attempts to register the action. If any of the action's outputs already has a generating + * action, and the two actions are not compatible, then an {@link ActionConflictException} is + * thrown. The internal data structure may be partially modified when that happens; it is not + * guaranteed that all potential conflicts are detected, but at least one of them is. + * + * <p>For example, take three actions A, B, and C, where A creates outputs a and b, B creates just + * b, and C creates c and b. There are two potential conflicts in this case, between A and B, and + * between B and C. Depending on the ordering of calls to this method and the ordering of outputs + * in the action output lists, either one or two conflicts are detected: if B is registered first, + * then both conflicts are detected; if either A or C is registered first, then only one conflict + * is detected. + */ + void registerAction(Action action) throws ActionConflictException; + + /** + * Removes an action from this action graph if it is present. + * + * <p>Throws {@link IllegalStateException} if one of the outputs of the action is in fact + * generated by a different {@link Action} instance (even if they are sharable). + */ + void unregisterAction(Action action); + + /** + * Clear the action graph. + */ + void clear(); + + /** + * This exception is thrown when a conflict between actions is detected. It contains information + * about the artifact for which the conflict is found, and data about the two conflicting actions + * and their owners. + */ + public static final class ActionConflictException extends Exception { + + private final Artifact artifact; + private final Action previousAction; + private final Action attemptedAction; + + public ActionConflictException(Artifact artifact, Action previousAction, + Action attemptedAction) { + super("for " + artifact); + this.artifact = artifact; + this.previousAction = previousAction; + this.attemptedAction = attemptedAction; + } + + public Artifact getArtifact() { + return artifact; + } + + public void reportTo(EventHandler eventListener) { + String msg = "file '" + artifact.prettyPrint() + + "' is generated by these conflicting actions:\n" + + suffix(attemptedAction, previousAction); + eventListener.handle(Event.error(msg)); + } + + private void addStringDetail(StringBuilder sb, String key, String valueA, String valueB) { + valueA = valueA != null ? valueA : "(null)"; + valueB = valueB != null ? valueB : "(null)"; + + sb.append(key).append(": ").append(valueA); + if (!valueA.equals(valueB)) { + sb.append(", ").append(valueB); + } + sb.append("\n"); + } + + private void addListDetail(StringBuilder sb, String key, + Iterable<Artifact> valueA, Iterable<Artifact> valueB) { + Set<Artifact> setA = ImmutableSet.copyOf(valueA); + Set<Artifact> setB = ImmutableSet.copyOf(valueB); + SetView<Artifact> diffA = Sets.difference(setA, setB); + SetView<Artifact> diffB = Sets.difference(setB, setA); + + sb.append(key).append(": "); + if (diffA.isEmpty() && diffB.isEmpty()) { + sb.append("are equal"); + } else { + if (!diffA.isEmpty() && !diffB.isEmpty()) { + sb.append("attempted action contains artifacts not in previous action and " + + "previous action contains artifacts not in attempted action."); + } else if (!diffA.isEmpty()) { + sb.append("attempted action contains artifacts not in previous action: "); + sb.append(StringUtil.joinEnglishList(diffA, "and")); + } else if (!diffB.isEmpty()) { + sb.append("previous action contains artifacts not in attempted action: "); + sb.append(StringUtil.joinEnglishList(diffB, "and")); + } + } + sb.append("\n"); + } + + // See also Actions.canBeShared() + private String suffix(Action a, Action b) { + // Note: the error message reveals to users the names of intermediate files that are not + // documented in the BUILD language. This error-reporting logic is rather elaborate but it + // does help to diagnose some tricky situations. + StringBuilder sb = new StringBuilder(); + ActionOwner aOwner = a.getOwner(); + ActionOwner bOwner = b.getOwner(); + boolean aNull = aOwner == null; + boolean bNull = bOwner == null; + + addStringDetail(sb, "Label", aNull ? null : Label.print(aOwner.getLabel()), + bNull ? null : Label.print(bOwner.getLabel())); + addStringDetail(sb, "RuleClass", aNull ? null : aOwner.getTargetKind(), + bNull ? null : bOwner.getTargetKind()); + addStringDetail(sb, "Configuration", aNull ? null : aOwner.getConfigurationName(), + bNull ? null : bOwner.getConfigurationName()); + addStringDetail(sb, "Mnemonic", a.getMnemonic(), b.getMnemonic()); + + addListDetail(sb, "MandatoryInputs", a.getMandatoryInputs(), b.getMandatoryInputs()); + addListDetail(sb, "Outputs", a.getOutputs(), b.getOutputs()); + + return sb.toString(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java b/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java new file mode 100644 index 0000000000..fa9b54e9d4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/NotifyOnActionCacheHit.java @@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An action which must know when it is skipped due to an action cache hit. + * + * Use should be rare, as the action graph is a functional model. + */ +public interface NotifyOnActionCacheHit extends Action { + + /** + * Called when action has "cache hit", and therefore need not be executed. + * + * @param executor the executor + */ + void actionCacheHit(Executor executor); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java b/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java new file mode 100644 index 0000000000..90af1361c7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/PackageRootResolver.java @@ -0,0 +1,34 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Map; + +import javax.annotation.Nullable; + +/** + * Represents logic that evaluates the root of the package containing path. + */ +public interface PackageRootResolver { + + /** + * Returns mapping from execPath to Root. Some roots can equal null if the corresponding + * package can't be found. Returns null if for some reason we can't evaluate it. + */ + @Nullable + Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java b/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java new file mode 100644 index 0000000000..80df9e20ac --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ParameterFile.java @@ -0,0 +1,114 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +/** + * Support for parameter file generation (as used by gcc and other tools, e.g. + * {@code gcc @param_file}. Note that the parameter file needs to be explicitly + * deleted after use. Different tools require different parameter file formats, + * which can be selected via the {@link ParameterFileType} enum. + * + * <p>The default charset is ISO-8859-1 (latin1). This also has to match the + * expectation of the tool. + * + * <p>Don't use this class for new code. Use the ParameterFileWriteAction + * instead! + */ +public class ParameterFile { + + /** + * Different styles of parameter files. + */ + public static enum ParameterFileType { + /** + * A parameter file with every parameter on a separate line. This format + * cannot handle newlines in parameters. It is currently used for most + * tools, but may not be interpreted correctly if parameters contain + * white space or other special characters. It should be avoided for new + * development. + */ + UNQUOTED, + + /** + * A parameter file where each parameter is correctly quoted for shell + * use, and separated by white space (space, tab, newline). This format is + * safe for all characters, but must be specially supported by the tool. In + * particular, it must not be used with gcc and related tools, which do not + * support this format as it is. + */ + SHELL_QUOTED; + } + + // Parameter file location. + private final Path execRoot; + private final PathFragment execPath; + private final Charset charset; + private final ParameterFileType type; + + @VisibleForTesting + public static final FileType PARAMETER_FILE = FileType.of(".params"); + + /** + * Creates a parameter file with the given parameters. + */ + public ParameterFile(Path execRoot, PathFragment execPath, Charset charset, + ParameterFileType type) { + Preconditions.checkNotNull(type); + this.execRoot = execRoot; + this.execPath = execPath; + this.charset = Preconditions.checkNotNull(charset); + this.type = Preconditions.checkNotNull(type); + } + + /** + * Derives an exec path from a given exec path by appending <code>".params"</code>. + */ + public static PathFragment derivePath(PathFragment original) { + return original.replaceName(original.getBaseName() + "-2.params"); + } + + /** + * Returns the path for the parameter file. + */ + public Path getPath() { + return execRoot.getRelative(execPath); + } + + /** + * Writes the arguments from the list into the parameter file according to + * the style selected in the constructor. + */ + public void writeContent(List<String> arguments) throws ExecException { + Iterable<String> actualArgs = (type == ParameterFileType.SHELL_QUOTED) ? + ShellEscaper.escapeAll(arguments) : arguments; + Path file = getPath(); + try { + FileSystemUtils.writeLinesAs(file, charset, actualArgs); + } catch (IOException e) { + throw new EnvironmentalExecException("could not write param file '" + file + "'", e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java new file mode 100644 index 0000000000..929a1068d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java @@ -0,0 +1,472 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.Pair; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.CountDownLatch; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * CPU/RAM resource manager. Used to keep track of resources consumed by the Blaze action execution + * threads and throttle them when necessary. + * + * <p>Threads which are known to consume a significant amount of the local CPU or RAM resources + * should call {@link #acquireResources} method. This method will check whether requested resources + * are available and will either mark them as used and allow thread to proceed or will block the + * thread until requested resources will become available. When thread completes it task, it must + * release allocated resources by calling {@link #releaseResources} method. + * + * <p>Available resources can be calculated using one of three ways: + * <ol> + * <li>They can be preset using {@link #setAvailableResources(ResourceSet)} method. This is used + * mainly by the unit tests (however it is possible to provide a future option that would + * artificially limit amount of CPU/RAM consumed by the Blaze). + * <li>They can be preset based on the /proc/cpuinfo and /proc/meminfo information. Blaze will + * calculate amount of available CPU cores (adjusting for hyperthreading logical cores) and + * amount of the total available memory and will limit itself to the number of effective cores + * and 2/3 of the available memory. For details, please look at the {@link + * LocalHostCapacity#getLocalHostCapacity} method. + * <li>Blaze will periodically (every 3 seconds) poll {@code /proc/meminfo} and {@code /proc/stat} + * information to obtain how much RAM and CPU resources are currently idle at that moment. For + * calculation details, please look at the {@link LocalHostCapacity#getFreeResources} + * implementation. + * </ol> + * + * <p>The resource manager also allows a slight overallocation of the resources to account for the + * fact that requested resources are usually estimated using a pessimistic approximation. It also + * guarantees that at least one thread will always be able to acquire any amount of requested + * resources (even if it is greater than amount of available resources). Therefore, assuming that + * threads correctly release acquired resources, Blaze will never be fully blocked. + */ +@ThreadSafe +public class ResourceManager { + + private static final Logger LOG = Logger.getLogger(ResourceManager.class.getName()); + private final boolean FINE; + + private EventBus eventBus; + + private final ThreadLocal<Boolean> threadLocked = new ThreadLocal<Boolean>() { + @Override + protected Boolean initialValue() { + return false; + } + }; + + /** + * Singleton reference defined in a separate class to ensure thread-safe lazy + * initialization. + */ + private static class Singleton { + static ResourceManager instance = new ResourceManager(); + } + + /** + * Returns singleton instance of the resource manager. + */ + public static ResourceManager instance() { + return Singleton.instance; + } + + // Allocated resources are allowed to go "negative", but at least + // MIN_AVAILABLE_CPU_RATIO portion of CPU and MIN_AVAILABLE_RAM_RATIO portion + // of RAM should be available. + // Please note that this value is purely empirical - we assume that generally + // requested resources are somewhat pessimistic and thread would end up + // using less than requested amount. + private final static double MIN_NECESSARY_CPU_RATIO = 0.6; + private final static double MIN_NECESSARY_RAM_RATIO = 1.0; + private final static double MIN_NECESSARY_IO_RATIO = 1.0; + + // List of blocked threads. Associated CountDownLatch object will always + // be initialized to 1 during creation in the acquire() method. + private final List<Pair<ResourceSet, CountDownLatch>> requestList; + + // The total amount of resources on the local host. Must be set by + // an explicit call to setAvailableResources(), often using + // LocalHostCapacity.getLocalHostCapacity() as an argument. + private ResourceSet staticResources = null; + + private ResourceSet availableResources = null; + private LocalHostCapacity.FreeResources freeReading = null; + + // Used amount of CPU capacity (where 1.0 corresponds to the one fully + // occupied CPU core. Corresponds to the CPU resource definition in the + // ResourceSet class. + private double usedCpu; + + // Used amount of RAM capacity in MB. Corresponds to the RAM resource + // definition in the ResourceSet class. + private double usedRam; + + // Used amount of I/O resources. Corresponds to the I/O resource + // definition in the ResourceSet class. + private double usedIo; + + // Specifies how much of the RAM in staticResources we should allow to be used. + public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67; + private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE; + + // Timer responsible for the periodic polling of the current system load. + private Timer timer = null; + + private ResourceManager() { + FINE = LOG.isLoggable(Level.FINE); + requestList = new LinkedList<Pair<ResourceSet, CountDownLatch>>(); + } + + @VisibleForTesting public static ResourceManager instanceForTestingOnly() { + return new ResourceManager(); + } + + /** + * Resets resource manager state and releases all thread locks. + * Note - it does not reset auto-sensing or available resources. Use + * separate call to setAvailableResoures() or to setAutoSensing(). + */ + public synchronized void resetResourceUsage() { + usedCpu = 0; + usedRam = 0; + usedIo = 0; + for (Pair<ResourceSet, CountDownLatch> request : requestList) { + // CountDownLatch can be set only to 0 or 1. + request.second.countDown(); + } + requestList.clear(); + } + + /** + * Sets available resources using given resource set. Must be called + * at least once before using resource manager. + * <p> + * Method will also disable auto-sensing if it was enabled. + */ + public synchronized void setAvailableResources(ResourceSet resources) { + Preconditions.checkNotNull(resources); + staticResources = resources; + setAutoSensing(false); + } + + public synchronized boolean isAutoSensingEnabled() { + return timer != null; + } + + /** + * Specify how much of the available RAM we should allow to be used. + * This has no effect if autosensing is enabled. + */ + public synchronized void setRamUtilizationPercentage(int percentage) { + ramUtilizationPercentage = percentage; + } + + /** + * Enables or disables secondary resource allocation algorithm that will + * periodically (when needed but at most once per 3 seconds) checks real + * amount of available memory (based on /proc/meminfo) and current CPU load + * (based on 1 second difference of /proc/stat) and allows additional resource + * acquisition if previous requests were overly pessimistic. + */ + public synchronized void setAutoSensing(boolean enable) { + // Create new Timer instance only if it does not exist already. + if (enable && !isAutoSensingEnabled()) { + Profiler.instance().logEvent(ProfilerTask.INFO, "Enable auto sensing"); + if(refreshFreeResources()) { + timer = new Timer("AutoSenseTimer", true); + timer.schedule(new TimerTask() { + @Override public void run() { refreshFreeResources(); } + }, 3000, 3000); + } + } else if (!enable) { + if (isAutoSensingEnabled()) { + Profiler.instance().logEvent(ProfilerTask.INFO, "Disable auto sensing"); + timer.cancel(); + timer = null; + } + if (staticResources != null) { + updateAvailableResources(false); + } + } + } + + /** + * Acquires requested resource set. Will block if resource is not available. + * NB! This method must be thread-safe! + */ + public void acquireResources(ActionMetadata owner, ResourceSet resources) + throws InterruptedException { + Preconditions.checkArgument(resources != null); + long startTime = Profiler.nanoTimeMaybe(); + CountDownLatch latch = null; + try { + waiting(owner); + latch = acquire(resources); + if (latch != null) { + latch.await(); + } + } finally { + threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0 + || resources.getIoUsage() != 0); + acquired(owner); + + // Profile acquisition only if it waited for resource to become available. + if (latch != null) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.ACTION_LOCK, owner); + } + } + } + + /** + * Acquires the given resources if available immediately. Does not block. + * @return true iff the given resources were locked (all or nothing). + */ + public boolean tryAcquire(ActionMetadata owner, ResourceSet resources) { + boolean acquired = false; + synchronized (this) { + if (areResourcesAvailable(resources)) { + incrementResources(resources); + acquired = true; + } + } + + if (acquired) { + threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0); + acquired(owner); + } + + return acquired; + } + + private void incrementResources(ResourceSet resources) { + usedCpu += resources.getCpuUsage(); + usedRam += resources.getMemoryMb(); + usedIo += resources.getIoUsage(); + } + + /** + * Return true if any resources have been claimed through this manager. + */ + public synchronized boolean inUse() { + return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || requestList.size() > 0; + } + + + /** + * Return true iff this thread has a lock on non-zero resources. + */ + public boolean threadHasResources() { + return threadLocked.get(); + } + + public void setEventBus(EventBus eventBus) { + Preconditions.checkState(this.eventBus == null); + this.eventBus = Preconditions.checkNotNull(eventBus); + } + + public void unsetEventBus() { + Preconditions.checkState(this.eventBus != null); + this.eventBus = null; + } + + private void waiting(ActionMetadata owner) { + if (eventBus != null) { + // Null only in tests. + eventBus.post(ActionStatusMessage.schedulingStrategy(owner)); + } + } + + private void acquired(ActionMetadata owner) { + if (eventBus != null) { + // Null only in tests. + eventBus.post(ActionStatusMessage.runningStrategy(owner)); + } + } + + /** + * Releases previously requested resource set. + * + * <p>NB! This method must be thread-safe! + */ + public void releaseResources(ActionMetadata owner, ResourceSet resources) { + boolean isConflict = false; + long startTime = Profiler.nanoTimeMaybe(); + try { + isConflict = release(resources); + } finally { + threadLocked.set(false); + + // Profile resource release only if it resolved at least one allocation request. + if (isConflict) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.ACTION_RELEASE, owner); + } + } + } + + private synchronized CountDownLatch acquire(ResourceSet resources) { + if (areResourcesAvailable(resources)) { + incrementResources(resources); + return null; + } + Pair<ResourceSet, CountDownLatch> request = + new Pair<>(resources, new CountDownLatch(1)); + requestList.add(request); + + // If we use auto sensing and there has not been an update within last + // 30 seconds, something has gone really wrong - disable it. + if (isAutoSensingEnabled() && freeReading.getReadingAge() > 30000) { + LoggingUtil.logToRemote(Level.WARNING, "Free resource readings were " + + "not updated for 30 seconds - auto-sensing is disabled", + new IllegalStateException()); + LOG.warning("Free resource readings were not updated for 30 seconds - " + + "auto-sensing is disabled"); + setAutoSensing(false); + } + return request.second; + } + + private synchronized boolean release(ResourceSet resources) { + usedCpu -= resources.getCpuUsage(); + usedRam -= resources.getMemoryMb(); + usedIo -= resources.getIoUsage(); + + // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being + // e.g. 1E-15. So if it is small enough, we set it to 0. But maybe there is a better solution. + if (usedCpu < 0.0001) { + usedCpu = 0; + } + if (usedRam < 0.0001) { + usedRam = 0; + } + if (usedIo < 0.0001) { + usedIo = 0; + } + if (requestList.size() > 0) { + processWaitingThreads(); + return true; + } + return false; + } + + + /** + * Tries to unblock one or more waiting threads if there are sufficient resources available. + */ + private synchronized void processWaitingThreads() { + Iterator<Pair<ResourceSet, CountDownLatch>> iterator = requestList.iterator(); + while (iterator.hasNext()) { + Pair<ResourceSet, CountDownLatch> request = iterator.next(); + if (areResourcesAvailable(request.first)) { + incrementResources(request.first); + request.second.countDown(); + iterator.remove(); + } + } + } + + // Method will return true if all requested resources are considered to be available. + private boolean areResourcesAvailable(ResourceSet resources) { + Preconditions.checkNotNull(availableResources); + // Comparison below is robust, since any calculation errors will be fixed + // by the release() method. + if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0) { + return true; + } + // Use only MIN_NECESSARY_???_RATIO of the resource value to check for + // allocation. This is necessary to account for the fact that most of the + // requested resource sets use pessimistic estimations. Note that this + // ratio is used only during comparison - for tracking we will actually + // mark whole requested amount as used. + double cpu = resources.getCpuUsage() * MIN_NECESSARY_CPU_RATIO; + double ram = resources.getMemoryMb() * MIN_NECESSARY_RAM_RATIO; + double io = resources.getIoUsage() * MIN_NECESSARY_IO_RATIO; + + double availableCpu = availableResources.getCpuUsage(); + double availableRam = availableResources.getMemoryMb(); + double availableIo = availableResources.getIoUsage(); + + // Resources are considered available if any one of the conditions below is true: + // 1) If resource is not requested at all, it is available. + // 2) If resource is not used at the moment, it is considered to be + // available regardless of how much is requested. This is necessary to + // ensure that at any given time, at least one thread is able to acquire + // resources even if it requests more than available. + // 3) If used resource amount is less than total available resource amount. + return (cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu) && + (ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam) && + (io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo); + } + + private synchronized void updateAvailableResources(boolean useFreeReading) { + Preconditions.checkNotNull(staticResources); + if (useFreeReading && isAutoSensingEnabled()) { + availableResources = new ResourceSet( + usedRam + freeReading.getFreeMb(), + usedCpu + freeReading.getAvgFreeCpu(), + staticResources.getIoUsage()); + if(FINE) { + LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB," + + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU"); + } + processWaitingThreads(); + } else { + availableResources = new ResourceSet( + staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0, + staticResources.getCpuUsage(), + staticResources.getIoUsage()); + processWaitingThreads(); + } + } + + /** + * Called by the timer thread to update system load information. + * + * @return true if update was successful and false if error was detected and + * autosensing was disabled. + */ + private boolean refreshFreeResources() { + freeReading = LocalHostCapacity.getFreeResources(freeReading); + if (freeReading == null) { // Unable to read or parse /proc/* information. + LOG.warning("Unable to obtain system load - autosensing is disabled"); + setAutoSensing(false); + return false; + } + updateAvailableResources( + freeReading.getInterval() >= 1000 && freeReading.getInterval() <= 10000); + return true; + } + + @VisibleForTesting + synchronized int getWaitCount() { + return requestList.size(); + } + + @VisibleForTesting + synchronized boolean isAvailable(double ram, double cpu, double io) { + return areResourcesAvailable(new ResourceSet(ram, cpu, io)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java new file mode 100644 index 0000000000..e7ab98f968 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java @@ -0,0 +1,113 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Splitter; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Instances of this class represent an estimate of the resource consumption + * for a particular Action, or the total available resources. We plan to + * use this to do smarter scheduling of actions, for example making sure + * that we don't schedule jobs concurrently if they would use so much + * memory as to cause the machine to thrash. + */ +@Immutable +public class ResourceSet { + + /** For actions that consume negligible resources. */ + public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0); + + /** The amount of real memory (resident set size). */ + private final double memoryMb; + + /** The number of CPUs, or fractions thereof. */ + private final double cpuUsage; + + /** + * Relative amount of used I/O resources (with 1.0 being total available amount on an "average" + * workstation. + */ + private final double ioUsage; + + public ResourceSet(double memoryMb, double cpuUsage, double ioUsage) { + this.memoryMb = memoryMb; + this.cpuUsage = cpuUsage; + this.ioUsage = ioUsage; + } + + /** Returns the amount of real memory (resident set size) used in MB. */ + public double getMemoryMb() { + return memoryMb; + } + + /** + * Returns the number of CPUs (or fractions thereof) used. + * For a CPU-bound single-threaded process, this will be 1.0. + * For a single-threaded process which spends part of its + * time waiting for I/O, this will be somewhere between 0.0 and 1.0. + * For a multi-threaded or multi-process application, + * this may be more than 1.0. + */ + public double getCpuUsage() { + return cpuUsage; + } + + /** + * Returns the amount of I/O used. + * Full amount of available I/O resources on the "average" workstation is + * considered to be 1.0. + */ + public double getIoUsage() { + return ioUsage; + } + + public static class ResourceSetConverter implements Converter<ResourceSet> { + private static final Splitter SPLITTER = Splitter.on(','); + + @Override + public ResourceSet convert(String input) throws OptionsParsingException { + Iterator<String> values = SPLITTER.split(input).iterator(); + try { + double memoryMb = Double.parseDouble(values.next()); + double cpuUsage = Double.parseDouble(values.next()); + double ioUsage = Double.parseDouble(values.next()); + if (values.hasNext()) { + throw new OptionsParsingException("Expected exactly 3 comma-separated float values"); + } + if (memoryMb <= 0.0 || cpuUsage <= 0.0 || ioUsage <= 0.0) { + throw new OptionsParsingException("All resource values must be positive"); + } + return new ResourceSet(memoryMb, cpuUsage, ioUsage); + } catch (NumberFormatException nfe) { + throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nfe); + } catch (NoSuchElementException nsee) { + throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nsee); + } + } + + @Override + public String getTypeDescription() { + return "comma-separated available amount of RAM (in MB), CPU (in cores) and " + + "available I/O (1.0 being average workstation)"; + } + + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/Root.java b/src/main/java/com/google/devtools/build/lib/actions/Root.java new file mode 100644 index 0000000000..284b85f0a4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Root.java @@ -0,0 +1,163 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.Serializable; +import java.util.Objects; + +import javax.annotation.Nullable; + +/** + * A root for an artifact. The roots are the directories containing artifacts, and they are mapped + * together into a single directory tree to form the execution environment. There are two kinds of + * roots, source roots and derived roots. Source roots correspond to entries of the package path, + * and they can be anywhere on disk. Derived roots correspond to output directories; there are + * generally different output directories for different configurations, and different types of + * output (bin, genfiles, includes, etc.). + * + * <p>When mapping the roots into a single directory tree, the source roots are merged, such that + * each package is accessed in its entirety from a single source root. The package cache is + * responsible for determining that mapping. The derived roots, on the other hand, have to be + * distinct. (It is currently allowed to have a derived root that is the prefix of another one.) + * + * <p>The derived roots must have paths that point inside the exec root, i.e. below the directory + * that is the root of the merged directory tree. + */ +@SkylarkModule(name = "root", + doc = "A root for files. The roots are the directories containing files, and they are mapped " + + "together into a single directory tree to form the execution environment.") +public final class Root implements Comparable<Root>, Serializable { + + /** + * Returns the given path as a source root. The path may not be {@code null}. + */ + public static Root asSourceRoot(Path path) { + return new Root(null, path); + } + + /** + * DO NOT USE IN PRODUCTION CODE! + * + * <p>Returns the given path as a derived root. This method only exists as a convenience for + * tests, which don't need a proper Root object. + */ + @VisibleForTesting + public static Root asDerivedRoot(Path path) { + return new Root(path, path); + } + + /** + * Returns the given path as a derived root, relative to the given exec root. The root must be a + * proper sub-directory of the exec root (i.e. not equal). Neither may be {@code null}. + * + * <p>Be careful with this method - all derived roots must be registered with the artifact factory + * before the analysis phase. + */ + public static Root asDerivedRoot(Path execRoot, Path root) { + Preconditions.checkArgument(root.startsWith(execRoot)); + Preconditions.checkArgument(!root.equals(execRoot)); + return new Root(execRoot, root); + } + + public static Root middlemanRoot(Path execRoot, Path outputDir) { + Path root = outputDir.getRelative("internal"); + Preconditions.checkArgument(root.startsWith(execRoot)); + Preconditions.checkArgument(!root.equals(execRoot)); + return new Root(execRoot, root, true); + } + + /** + * Returns the exec root as a derived root. The exec root should never be treated as a derived + * root, but this is currently allowed. Do not add any further uses besides the ones that already + * exist! + */ + static Root execRootAsDerivedRoot(Path execRoot) { + return new Root(execRoot, execRoot); + } + + @Nullable private final Path execRoot; + private final Path path; + private final boolean isMiddlemanRoot; + + private Root(@Nullable Path execRoot, Path path, boolean isMiddlemanRoot) { + this.execRoot = execRoot; + this.path = Preconditions.checkNotNull(path); + this.isMiddlemanRoot = isMiddlemanRoot; + } + + private Root(@Nullable Path execRoot, Path path) { + this(execRoot, path, false); + } + + public Path getPath() { + return path; + } + + /** + * Returns the path fragment from the exec root to the actual root. For source roots, this returns + * the empty fragment. + */ + public PathFragment getExecPath() { + return isSourceRoot() ? PathFragment.EMPTY_FRAGMENT : path.relativeTo(execRoot); + } + + @SkylarkCallable(name = "path", structField = true, + doc = "Returns the relative path from the exec root to the actual root.") + public String getExecPathString() { + return getExecPath().getPathString(); + } + + public boolean isSourceRoot() { + return execRoot == null; + } + + public boolean isMiddlemanRoot() { + return isMiddlemanRoot; + } + + @Override + public int compareTo(Root o) { + return path.compareTo(o.path); + } + + @Override + public int hashCode() { + return Objects.hash(execRoot, path.hashCode()); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Root)) { + return false; + } + Root r = (Root) o; + return path.equals(r.path) && Objects.equals(execRoot, r.execRoot); + } + + @Override + public String toString() { + return path.toString() + (isSourceRoot() ? "[source]" : "[derived]"); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/Spawn.java b/src/main/java/com/google/devtools/build/lib/actions/Spawn.java new file mode 100644 index 0000000000..2f905d0adc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/Spawn.java @@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.extra.SpawnInfo; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; + +/** + * An object representing a subprocess to be invoked, including its command and + * arguments, its working directory, its environment, a boolean indicating + * whether remote execution is appropriate for this command, and if so, the set + * of files it is expected to read and write. + */ +public interface Spawn { + + /** + * Returns true iff this command may be executed remotely. + */ + boolean isRemotable(); + + /** + * Out-of-band data for this spawn. This can be used to signal hints (hardware requirements, + * local vs. remote) to the execution subsystem. + * + * <p>String tags from {@link + * com.google.devtools.build.lib.rules.test.TestTargetProperties#getExecutionInfo()} can be added + * as keys with arbitrary values to this map too. + */ + ImmutableMap<String, String> getExecutionInfo(); + + /** + * Returns this Spawn as a Bourne shell command. + * + * @param workingDir the initial working directory of the command + */ + String asShellCommand(Path workingDir); + + /** + * Returns the runfiles data for remote execution. Format is (directory, manifest file). + */ + ImmutableMap<PathFragment, Artifact> getRunfilesManifests(); + + /** + * Returns artifacts for filesets, so they can be scheduled on remote execution. + */ + ImmutableList<Artifact> getFilesetManifests(); + + /** + * Returns a protocol buffer describing this spawn for use by the extra_action functionality. + */ + SpawnInfo getExtraActionInfo(); + + /** + * Returns the command (the first element) and its arguments. + */ + ImmutableList<String> getArguments(); + + /** + * Returns the initial environment of the process. + * If null, the environment is inherited from the parent process. + */ + ImmutableMap<String, String> getEnvironment(); + + /** + * Returns the list of files that this command may read. + * + * <p>This method explicitly does not expand middleman artifacts. Pass the result + * to an appropriate utility method on {@link com.google.devtools.build.lib.actions.Artifact} to + * expand the middlemen. + * + * <p>This is for use with remote execution, so we can ship inputs before starting the + * command. Order stability across multiple calls should be upheld for performance reasons. + */ + Iterable<? extends ActionInput> getInputFiles(); + + /** + * Returns the collection of files that this command must write. Callers should not mutate + * the result. + * + * <p>This is for use with remote execution, so remote execution does not have to guess what + * outputs the process writes. While the order does not affect the semantics, it should be + * stable so it can be cached. + */ + Collection<? extends ActionInput> getOutputFiles(); + + /** + * Returns the resource owner for local fallback. + */ + ActionMetadata getResourceOwner(); + + /** + * Returns the amount of resources needed for local fallback. + */ + ResourceSet getLocalResources(); + + /** + * Returns the owner for this action. Production code should supply a non-null owner. + */ + ActionOwner getOwner(); + + /** + * Returns a mnemonic (string constant) for this kind of spawn. + */ + String getMnemonic(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java new file mode 100644 index 0000000000..c2ea1b0a2b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnActionContext.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +/** + * A context that allows execution of {@link Spawn} instances. + */ +@ActionContextMarker(name = "spawn") +public interface SpawnActionContext extends Executor.ActionContext { + + /** + * Executes the given spawn. + */ + void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException; + + /** Returns the locality of running the spawn, i.e., "local". */ + String strategyLocality(String mnemonic, boolean remotable); + + /** + * This implements a tri-state mode. There are three possible cases: (1) implementations of this + * class can unconditionally execute spawns locally, (2) they can follow whatever is set for the + * corresponding spawn (see {@link Spawn#isRemotable}), or (3) they can unconditionally execute + * spawns remotely, i.e., force remote execution. + * + * <p>Passing the spawns remotable flag to this method returns whether the spawn will actually be + * executed remotely. + */ + boolean isRemotable(String mnemonic, boolean remotable); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java b/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java new file mode 100644 index 0000000000..9092ab2d19 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TargetOutOfDateException.java @@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An exception indicating that a target is out of date. + */ +public class TargetOutOfDateException extends ActionExecutionException { + + public TargetOutOfDateException(Action action) { + super (action.prettyPrint() + " is not up-to-date", action, false); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java b/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java new file mode 100644 index 0000000000..c86133838f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TestExecException.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * An TestExecException that is related to the failure of a TestAction. + */ +public final class TestExecException extends ExecException { + + public TestExecException(String message) { + super(message); + } + + @Override + public ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action) { + String message = messagePrefix + " failed" + getMessage(); + if (verboseFailures) { + return new ActionExecutionException(message, this, action, isCatastrophic()); + } else { + return new ActionExecutionException(message, action, isCatastrophic()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java b/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java new file mode 100644 index 0000000000..128ac20821 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TestMiddlemanObserver.java @@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +/** + * Used as a notification mechanism for the scheduling middleman mutations + * related to scheduling exclusive tests. + */ +public interface TestMiddlemanObserver { + + /** + * Called when the test removes the stale middleman. + * + * @param action the test action. + * @param middleman the scheduling middleman. + * @param middlemanAction the action generating the scheduling middleman + */ + void remove(Action action, Artifact middleman, Action middlemanAction); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java b/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java new file mode 100644 index 0000000000..86a6eb008c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/UserExecException.java @@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +/** + * An ExecException that is related to the failure of an Action and therefore + * very likely the user's fault. + */ +public class UserExecException extends ExecException { + + public UserExecException(String message) { + super(message); + } + + public UserExecException(String message, Throwable cause) { + super(message, cause); + } + + @Override + public ActionExecutionException toActionExecutionException(String messagePrefix, + boolean verboseFailures, Action action) { + String message = messagePrefix + " failed: " + getMessage(); + if (verboseFailures) { + return new ActionExecutionException(message, this, action, isCatastrophic()); + } else { + return new ActionExecutionException(message, action, isCatastrophic()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java new file mode 100644 index 0000000000..2ac0ab82da --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java @@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * An interface defining a cache of already-executed Actions. + * + * <p>This class' naming is misleading; it doesn't cache the actual actions, but it stores a + * fingerprint of the action state (ie. a hash of the input and output files on disk), so + * we can tell if we need to rerun an action given the state of the file system. + * + * <p>Each action entry uses one of its output paths as a key (after conversion + * to the string). + */ +@ThreadCompatible +public interface ActionCache { + + /** + * Updates the cache entry for the specified key. + */ + void put(String key, ActionCache.Entry entry); + + /** + * Returns the corresponding cache entry for the specified key, if any, or + * null if not found. + */ + ActionCache.Entry get(String key); + + /** + * Removes entry from cache + */ + void remove(String key); + + /** + * Returns a new Entry instance. This method allows ActionCache subclasses to + * define their own Entry implementation. + */ + ActionCache.Entry createEntry(String key); + + /** + * An entry in the ActionCache that contains all action input and output + * artifact paths and their metadata plus action key itself. + * + * Cache entry operates under assumption that once it is fully initialized + * and getFileDigest() method is called, it becomes logically immutable (all methods + * will continue to return same result regardless of internal data transformations). + */ + public final class Entry { + private final String actionKey; + private final List<String> files; + // If null, digest is non-null and the entry is immutable. + private Map<String, Metadata> mdMap; + private Digest digest; + + public Entry(String key) { + actionKey = key; + files = new ArrayList<>(); + mdMap = new HashMap<>(); + } + + public Entry(String key, List<String> files, Digest digest) { + actionKey = key; + this.files = files; + this.digest = digest; + mdMap = null; + } + + /** + * Adds the artifact, specified by the executable relative path and its + * metadata into the cache entry. + */ + public void addFile(PathFragment relativePath, Metadata md) { + Preconditions.checkState(mdMap != null); + Preconditions.checkState(!isCorrupted()); + Preconditions.checkState(digest == null); + + String execPath = relativePath.getPathString(); + files.add(execPath); + mdMap.put(execPath, md); + } + + /** + * @return action key string. + */ + public String getActionKey() { + return actionKey; + } + + /** + * Returns the combined digest of the action's inputs and outputs. + * + * This may compresses the data into a more compact representation, and + * makes the object immutable. + */ + public Digest getFileDigest() { + if (digest == null) { + digest = Digest.fromMetadata(mdMap); + mdMap = null; + } + return digest; + } + + /** + * Returns true if this cache entry is corrupted and should be ignored. + */ + public boolean isCorrupted() { + return actionKey == null; + } + + /** + * @return stored path strings. + */ + public Collection<String> getPaths() { + return files; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(" actionKey = ").append(actionKey).append("\n"); + builder.append(" digestKey = "); + if (digest == null) { + builder.append(Digest.fromMetadata(mdMap)).append(" (from mdMap)\n"); + } else { + builder.append(digest).append("\n"); + } + List<String> fileInfo = Lists.newArrayListWithCapacity(files.size()); + fileInfo.addAll(files); + Collections.sort(fileInfo); + for (String info : fileInfo) { + builder.append(" ").append(info).append("\n"); + } + return builder.toString(); + } + } + + /** + * Give persistent cache implementations a notification to write to disk. + * @return size in bytes of the serialized cache. + */ + long save() throws IOException; + + /** + * Dumps action cache content into the given PrintStream. + */ + void dump(PrintStream out); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java new file mode 100644 index 0000000000..24eb42ead0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java @@ -0,0 +1,389 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.CompactStringIndexer; +import com.google.devtools.build.lib.util.PersistentMap; +import com.google.devtools.build.lib.util.StringIndexer; +import com.google.devtools.build.lib.util.VarInt; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.UnixGlob; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * An implementation of the ActionCache interface that uses + * {@link CompactStringIndexer} to reduce memory footprint and saves + * cached actions using the {@link PersistentMap}. + * + * <p>This cache is not fully correct: as hashes are xor'd together, a permutation of input + * file contents will erroneously be considered up to date. + */ +@ConditionallyThreadSafe // condition: each instance must instantiated with + // different cache root +public class CompactPersistentActionCache implements ActionCache { + private static final int SAVE_INTERVAL_SECONDS = 3; + private static final long NANOS_PER_SECOND = 1000 * 1000 * 1000; + + // Key of the action cache record that holds information used to verify referential integrity + // between action cache and string indexer. Must be < 0 to avoid conflict with real action + // cache records. + private static final int VALIDATION_KEY = -10; + + private static final int VERSION = 10; + + private final class ActionMap extends PersistentMap<Integer, byte[]> { + private final Clock clock; + private long nextUpdate; + + public ActionMap(Map<Integer, byte[]> map, Clock clock, Path mapFile, Path journalFile) + throws IOException { + super(VERSION, map, mapFile, journalFile); + this.clock = clock; + // Using nanoTime. currentTimeMillis may not provide enough granularity. + nextUpdate = clock.nanoTime() / NANOS_PER_SECOND + SAVE_INTERVAL_SECONDS; + load(); + } + + @Override + protected boolean updateJournal() { + // Using nanoTime. currentTimeMillis may not provide enough granularity. + long time = clock.nanoTime() / NANOS_PER_SECOND; + if (SAVE_INTERVAL_SECONDS == 0 || time > nextUpdate) { + nextUpdate = time + SAVE_INTERVAL_SECONDS; + // Force flushing of the PersistentStringIndexer instance. This is needed to ensure + // that filename index data on disk is always up-to-date when we save action cache + // data. + indexer.flush(); + return true; + } + return false; + } + + @Override + protected boolean keepJournal() { + // We must first flush the journal to get an accurate measure of its size. + forceFlush(); + try { + return journalSize() * 100 < cacheSize(); + } catch (IOException e) { + return false; + } + } + + @Override + protected Integer readKey(DataInputStream in) throws IOException { + return in.readInt(); + } + + @Override + protected byte[] readValue(DataInputStream in) + throws IOException { + int size = in.readInt(); + if (size < 0) { + throw new IOException("found negative array size: " + size); + } + byte[] data = new byte[size]; + in.readFully(data); + return data; + } + + @Override + protected void writeKey(Integer key, DataOutputStream out) + throws IOException { + out.writeInt(key); + } + + @Override + // TODO(bazel-team): (2010) This method, writeKey() and related Metadata methods + // should really use protocol messages. Doing so would allow easy inspection + // of the action cache content and, more importantly, would cut down on the + // need to change VERSION to different number every time we touch those + // methods. Especially when we'll start to add stuff like statistics for + // each action. + protected void writeValue(byte[] value, DataOutputStream out) + throws IOException { + out.writeInt(value.length); + out.write(value); + } + } + + private final PersistentMap<Integer, byte[]> map; + private final PersistentStringIndexer indexer; + static final ActionCache.Entry CORRUPTED = new ActionCache.Entry(null); + + public CompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException { + Path cacheFile = cacheFile(cacheRoot); + Path journalFile = journalFile(cacheRoot); + Path indexFile = cacheRoot.getChild("filename_index_v" + VERSION + ".blaze"); + // we can now use normal hash map as backing map, since dependency checker + // will manually purge records from the action cache. + Map<Integer, byte[]> backingMap = new HashMap<>(); + + try { + indexer = PersistentStringIndexer.newPersistentStringIndexer(indexFile, clock); + } catch (IOException e) { + renameCorruptedFiles(cacheRoot); + throw new IOException("Failed to load filename index data", e); + } + + try { + map = new ActionMap(backingMap, clock, cacheFile, journalFile); + } catch (IOException e) { + renameCorruptedFiles(cacheRoot); + throw new IOException("Failed to load action cache data", e); + } + + // Validate referential integrity between two collections. + if (!map.isEmpty()) { + String integrityError = validateIntegrity(indexer.size(), map.get(VALIDATION_KEY)); + if (integrityError != null) { + renameCorruptedFiles(cacheRoot); + throw new IOException("Failed action cache referential integrity check: " + integrityError); + } + } + } + + /** + * Rename corrupted files so they could be analyzed later. This would also ensure + * that next initialization attempt will create empty cache. + */ + private static void renameCorruptedFiles(Path cacheRoot) { + try { + for (Path path : UnixGlob.forPath(cacheRoot).addPattern("action_*_v" + VERSION + ".*") + .glob()) { + path.renameTo(path.getParentDirectory().getChild(path.getBaseName() + ".bad")); + } + for (Path path : UnixGlob.forPath(cacheRoot).addPattern("filename_*_v" + VERSION + ".*") + .glob()) { + path.renameTo(path.getParentDirectory().getChild(path.getBaseName() + ".bad")); + } + } catch (IOException e) { + // do nothing + } + } + + /** + * @return false iff indexer contains no data or integrity check has failed. + */ + private static String validateIntegrity(int indexerSize, byte[] validationRecord) { + if (indexerSize == 0) { + return "empty index"; + } + if (validationRecord == null) { + return "no validation record"; + } + try { + int validationSize = ByteBuffer.wrap(validationRecord).asIntBuffer().get(); + if (validationSize <= indexerSize) { + return null; + } else { + return String.format("Validation mismatch: validation entry %d is too large " + + "compared to index size %d", validationSize, indexerSize); + } + } catch (BufferUnderflowException e) { + return e.getMessage(); + } + + } + + public static Path cacheFile(Path cacheRoot) { + return cacheRoot.getChild("action_cache_v" + VERSION + ".blaze"); + } + + public static Path journalFile(Path cacheRoot) { + return cacheRoot.getChild("action_journal_v" + VERSION + ".blaze"); + } + + @Override + public ActionCache.Entry createEntry(String key) { + return new ActionCache.Entry(key); + } + + @Override + public ActionCache.Entry get(String key) { + int index = indexer.getIndex(key); + if (index < 0) { + return null; + } + byte[] data; + synchronized (this) { + data = map.get(index); + } + try { + return data != null ? CompactPersistentActionCache.decode(indexer, data) : null; + } catch (IOException e) { + // return entry marked as corrupted. + return CORRUPTED; + } + } + + @Override + public void put(String key, ActionCache.Entry entry) { + // Encode record. Note that both methods may create new mappings in the indexer. + int index = indexer.getOrCreateIndex(key); + byte[] content = encode(indexer, entry); + + // Update validation record. + ByteBuffer buffer = ByteBuffer.allocate(4); // size of int in bytes + int indexSize = indexer.size(); + buffer.asIntBuffer().put(indexSize); + + // Note the benign race condition here in which two threads might race on + // updating the VALIDATION_KEY. If the most recent update loses the race, + // a value lower than the indexer size will remain in the validation record. + // This will still pass the integrity check. + synchronized (this) { + map.put(VALIDATION_KEY, buffer.array()); + // Now update record itself. + map.put(index, content); + } + } + + @Override + public synchronized void remove(String key) { + map.remove(indexer.getIndex(key)); + } + + @Override + public synchronized long save() throws IOException { + long indexSize = indexer.save(); + long mapSize = map.save(); + return indexSize + mapSize; + } + + @Override + public synchronized String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Action cache (" + map.size() + " records):\n"); + for (Map.Entry<Integer, byte[]> entry: map.entrySet()) { + if (entry.getKey() == VALIDATION_KEY) { continue; } + String content; + try { + content = decode(indexer, entry.getValue()).toString(); + } catch (IOException e) { + content = e.toString() + "\n"; + } + builder.append("-> ").append(indexer.getStringForIndex(entry.getKey())).append("\n") + .append(content).append(" packed_len = ").append(entry.getValue().length).append("\n"); + } + return builder.toString(); + } + + /** + * Dumps action cache content. + */ + @Override + public synchronized void dump(PrintStream out) { + out.println("String indexer content:\n"); + out.println(indexer.toString()); + out.println("Action cache (" + map.size() + " records):\n"); + for (Map.Entry<Integer, byte[]> entry: map.entrySet()) { + if (entry.getKey() == VALIDATION_KEY) { continue; } + String content; + try { + content = CompactPersistentActionCache.decode(indexer, entry.getValue()).toString(); + } catch (IOException e) { + content = e.toString() + "\n"; + } + out.println(entry.getKey() + ", " + indexer.getStringForIndex(entry.getKey()) + ":\n" + + content + "\n packed_len = " + entry.getValue().length + "\n"); + } + } + + /** + * @return action data encoded as a byte[] array. + */ + private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) { + Preconditions.checkState(!entry.isCorrupted()); + + try { + byte[] actionKeyBytes = entry.getActionKey().getBytes(ISO_8859_1); + Collection<String> files = entry.getPaths(); + + // Estimate the size of the buffer: + // 5 bytes max for the actionKey length + // + the actionKey itself + // + 16 bytes for the digest + // + 5 bytes max for the file list length + // + 5 bytes max for each file id + int maxSize = VarInt.MAX_VARINT_SIZE + actionKeyBytes.length + Digest.MD5_SIZE + + VarInt.MAX_VARINT_SIZE + files.size() * VarInt.MAX_VARINT_SIZE; + ByteArrayOutputStream sink = new ByteArrayOutputStream(maxSize); + + VarInt.putVarInt(actionKeyBytes.length, sink); + sink.write(actionKeyBytes); + + entry.getFileDigest().write(sink); + + VarInt.putVarInt(files.size(), sink); + for (String file : files) { + VarInt.putVarInt(indexer.getOrCreateIndex(file), sink); + } + return sink.toByteArray(); + } catch (IOException e) { + // This Exception can never be thrown by ByteArrayOutputStream. + throw new AssertionError(e); + } + } + + /** + * Creates new action cache entry using given compressed entry data. Data + * will stay in the compressed format until entry is actually used by the + * dependency checker. + */ + private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) throws IOException { + try { + ByteBuffer source = ByteBuffer.wrap(data); + + byte[] actionKeyBytes = new byte[VarInt.getVarInt(source)]; + source.get(actionKeyBytes); + String actionKey = new String(actionKeyBytes, ISO_8859_1); + + Digest digest = Digest.read(source); + + int count = VarInt.getVarInt(source); + ImmutableList.Builder<String> builder = new ImmutableList.Builder<>(); + for (int i = 0; i < count; i++) { + int id = VarInt.getVarInt(source); + String filename = (id >= 0 ? indexer.getStringForIndex(id) : null); + if (filename == null) { + throw new IOException("Corrupted file index"); + } + builder.add(filename); + } + if (source.remaining() > 0) { + throw new IOException("serialized entry data has not been fully decoded"); + } + return new Entry(actionKey, builder.build(), digest); + } catch (BufferUnderflowException e) { + throw new IOException("encoded entry data is incomplete", e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java new file mode 100644 index 0000000000..f278507ddf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Digest.java @@ -0,0 +1,142 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.VarInt; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Map; + +/** + * A value class for capturing and comparing MD5-based digests. + * + * <p>Note that this class is responsible for digesting file metadata in an + * order-independent manner. Care must be taken to do this properly. The + * digest must be a function of the set of (path, metadata) tuples. While the + * order of these pairs must not matter, it would <b>not</b> be safe to make + * the digest be a function of the set of paths and the set of metadata. + * + * <p>Note that the (path, metadata) tuples must be unique, otherwise the + * XOR-based approach will fail. + */ +public class Digest { + + static final int MD5_SIZE = 16; + + private final byte[] digest; + + /** + * Construct the digest from the given bytes. + * @param digest an MD5 digest. Must be sized properly. + */ + @VisibleForTesting + Digest(byte[] digest) { + Preconditions.checkState(digest.length == MD5_SIZE); + this.digest = Arrays.copyOf(digest, digest.length); + } + + /** + * @param source the byte buffer source. + * @return the digest from the given buffer. + * @throws IOException if the byte buffer is incorrectly formatted. + */ + public static Digest read(ByteBuffer source) throws IOException { + int size = VarInt.getVarInt(source); + if (size != MD5_SIZE) { + throw new IOException("Unexpected digest length: " + size); + } + byte[] bytes = new byte[size]; + source.get(bytes); + return new Digest(bytes); + } + + /** + * Write the digest to the output stream. + */ + public void write(OutputStream sink) throws IOException { + VarInt.putVarInt(digest.length, sink); + sink.write(digest); + } + + /** + * @param mdMap A collection of (execPath, Metadata) pairs. + * Values may be null. + * @return an <b>order-independent</b> digest from the given "set" of + * (path, metadata) pairs. + */ + public static Digest fromMetadata(Map<String, Metadata> mdMap) { + byte[] result = new byte[MD5_SIZE]; + // Profiling showed that MD5 engine instantiation was a hotspot, so create one instance for + // this computation to amortize its cost. + Fingerprint fp = new Fingerprint(); + for (Map.Entry<String, Metadata> entry : mdMap.entrySet()) { + xorWith(result, getDigest(fp, entry.getKey(), entry.getValue())); + fp.reset(); + } + return new Digest(result); + } + + /** + * @return this Digest as a Metadata with no mtime. + */ + public Metadata asMetadata() { + return new Metadata(digest); + } + + @Override + public int hashCode() { + return Arrays.hashCode(digest); + } + + @Override + public boolean equals(Object obj) { + return (obj instanceof Digest) && Arrays.equals(digest, ((Digest) obj).digest); + } + + @Override + public String toString() { + return Fingerprint.hexDigest(digest); + } + + private static byte[] getDigest(Fingerprint fp, String execPath, Metadata md) { + fp.addString(execPath); + + if (md == null) { + // Move along, nothing to see here. + } else if (md.digest == null) { + // Use the timestamp if the digest is not present, but not both. + // Modifying a timestamp while keeping the contents of a file the + // same should not cause rebuilds. + fp.addLong(md.mtime); + } else { + fp.addBytes(md.digest); + } + return fp.digestAndReset(); + } + + /** + * Compute lhs ^= rhs bitwise operation of the arrays. + */ + private static void xorWith(byte[] lhs, byte[] rhs) { + for (int i = 0; i < lhs.length; i++) { + lhs[i] ^= rhs[i]; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java new file mode 100644 index 0000000000..7295fb5317 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java @@ -0,0 +1,130 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.base.Preconditions; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Objects; +import java.util.logging.Level; + +import javax.annotation.Nullable; + +/** + * Utility class for getting md5 digests of files. + */ +public class DigestUtils { + // Object to synchronize on when serializing large file reads. + private static final Object MD5_LOCK = new Object(); + + /** Private constructor to prevent instantiation of utility class. */ + private DigestUtils() {} + + /** + * Returns true iff using MD5 digests is appropriate for an artifact. + * + * @param artifact Artifact in question. + * @param isFile whether or not Artifact is a file versus a directory, isFile() on its stat. + * @param size size of Artifact on filesystem in bytes, getSize() on its stat. + */ + public static boolean useFileDigest(Artifact artifact, boolean isFile, long size) { + // Use timestamps for directories. Use digests for everything else. + return isFile && size != 0; + } + + /** + * Obtain file's MD5 metadata using synchronized method, ensuring that system + * is not overloaded in case when multiple threads are requesting MD5 + * calculations and underlying file system cannot provide it via extended + * attribute. + */ + private static byte[] getDigestInExclusiveMode(Path path) throws IOException { + long startTime = BlazeClock.nanoTime(); + synchronized (MD5_LOCK) { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, path.getPathString()); + return getDigestInternal(path); + } + } + + private static byte[] getDigestInternal(Path path) throws IOException { + long startTime = BlazeClock.nanoTime(); + byte[] md5bin = path.getMD5Digest(); + + long millis = (BlazeClock.nanoTime() - startTime) / 1000000; + if (millis > 5000L) { + System.err.println("Slow read: a " + path.getFileSize() + "-byte read from " + path + + " took " + millis + "ms."); + } + return md5bin; + } + + private static boolean binaryDigestWellFormed(byte[] digest) { + Preconditions.checkNotNull(digest); + return digest.length == 16; + } + + /** + * Returns the the fast md5 digest of the file, or null if not available. + */ + @Nullable + public static byte[] getFastDigest(Path path) throws IOException { + return path.getFastDigestFunctionType().equals("MD5") ? path.getFastDigest() : null; + } + + /** + * Get the md5 digest of {@code path}, using a constant-time xattr call if the filesystem supports + * it, and calculating the digest manually otherwise. + * + * @param path Path of the file. + * @param fileSize size of the file. Used to determine if digest calculation should be done + * serially or in parallel. Files larger than a certain threshold will be read serially, in order + * to avoid excessive disk seeks. + */ + public static byte[] getDigestOrFail(Path path, long fileSize) throws IOException { + // TODO(bazel-team): the action cache currently only works with md5 digests but it ought to + // work with any opaque digest. + byte[] md5bin = null; + if (Objects.equals(path.getFastDigestFunctionType(), "MD5")) { + md5bin = getFastDigest(path); + } + if (md5bin != null && !binaryDigestWellFormed(md5bin)) { + // Fail-soft in cases where md5bin is non-null, but not a valid digest. + String msg = String.format("Malformed digest '%s' for file %s", + BaseEncoding.base16().lowerCase().encode(md5bin), + path); + LoggingUtil.logToRemote(Level.SEVERE, msg, new IllegalStateException(msg)); + md5bin = null; + } + if (md5bin != null) { + return md5bin; + } else if (fileSize > 4096) { + // We'll have to read file content in order to calculate the digest. In that case + // it would be beneficial to serialize those calculations since there is a high + // probability that MD5 will be requested for multiple output files simultaneously. + // Exception is made for small (<=4K) files since they will not likely to introduce + // significant delays (at worst they will result in two extra disk seeks by + // interrupting other reads). + return getDigestInExclusiveMode(path); + } else { + return getDigestInternal(path); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java b/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java new file mode 100644 index 0000000000..9764cd8410 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/InjectedStat.java @@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.vfs.FileStatus; + +/** + * A FileStatus corresponding to a file that is not determined by querying the file system. + */ +public class InjectedStat implements FileStatus { + + private final long mtime; + private final long size; + private final long nodeId; + + public InjectedStat(long mtime, long size, long nodeId) { + this.mtime = mtime; + this.size = size; + this.nodeId = nodeId; + } + + @Override + public boolean isFile() { + return true; + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public long getSize() { + return size; + } + + @Override + public long getLastModifiedTime() { + return mtime; + } + + @Override + public long getLastChangeTime() { + return getLastModifiedTime(); + } + + @Override + public long getNodeId() { + return nodeId; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java new file mode 100644 index 0000000000..36c52b9942 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/Metadata.java @@ -0,0 +1,92 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.base.Preconditions; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import java.util.Arrays; +import java.util.Date; + +/** + * A class to represent file metadata. + * ActionCacheChecker may assume that, for a given file, equal + * metadata at different moments implies equal file-contents, + * where metadata equality is computed using Metadata.equals(). + * <p> + * NB! Several other parts of Blaze are relying on the fact that metadata + * uses mtime and not ctime. If metadata is ever changed + * to use ctime, all uses of Metadata must be carefully examined. + */ +@Immutable @ThreadSafe +public final class Metadata { + public final long mtime; + public final byte[] digest; + + // Convenience object for use with volatile files that we do not want checked + // (e.g. the build-changelist.txt) + public static final Metadata CONSTANT_METADATA = new Metadata(-1); + + public Metadata(long mtime) { + this.mtime = mtime; + this.digest = null; + } + + public Metadata(byte[] digest) { + this.mtime = 0L; + this.digest = Preconditions.checkNotNull(digest); + } + + @Override + public int hashCode() { + int hash = 0; + if (digest != null) { + // We are already dealing with the digest so we can just use portion of it + // as a hash code. + hash += digest[0] + (digest[1] << 8) + (digest[2] << 16) + (digest[3] << 24); + } else { + // Inlined hashCode for Long, so we don't + // have to construct an Object, just to compute + // a 32-bit hash out of a 64 bit value. + hash = (int) (mtime ^ (mtime >>> 32)); + } + return hash; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + if (!(that instanceof Metadata)) { + return false; + } + // Do a strict comparison - both digest and mtime should match + return Arrays.equals(this.digest, ((Metadata) that).digest) + && this.mtime == ((Metadata) that).mtime; + } + + @Override + public String toString() { + if (digest != null) { + return "MD5 " + BaseEncoding.base16().lowerCase().encode(digest); + } else if (mtime > 0) { + return "timestamp " + new Date(mtime); + } + return "no metadata"; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java new file mode 100644 index 0000000000..9d288dbbff --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/MetadataHandler.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.vfs.FileStatus; + +import java.io.IOException; +import java.util.Collection; + +/** Retrieves {@link Metadata} of {@link Artifact}s, and inserts virtual metadata as well. */ +public interface MetadataHandler { + /** + * Returns metadata for the given artifact or null if it does not exist. + * + * @param artifact artifact + * + * @return metadata instance or null if metadata cannot be obtained. + */ + Metadata getMetadataMaybe(Artifact artifact); + /** + * Returns metadata for the given artifact or throws an exception if the + * metadata could not be obtained. + * + * @return metadata instance + * + * @throws IOException if metadata could not be obtained. + */ + Metadata getMetadata(Artifact artifact) throws IOException; + + /** Sets digest for virtual artifacts (e.g. middlemen). {@code digest} must not be null. */ + void setDigestForVirtualArtifact(Artifact artifact, Digest digest); + + /** + * Injects provided digest into the metadata handler, simultaneously caching lstat() data as well. + */ + void injectDigest(ActionInput output, FileStatus statNoFollow, byte[] digest); + + /** Returns true iff artifact exists. */ + boolean artifactExists(Artifact artifact); + /** Returns true iff artifact is a regular file. */ + boolean isRegularFile(Artifact artifact); + + /** + * @return Whether the artifact's data was injected. + * @throws IOException if implementation tried to stat artifact which threw an exception. + * Technically, this means that the artifact could not have been injected, but by throwing + * here we save the caller trying to stat this file on their own and throwing the same + * exception. Implementations are not guaranteed to throw in this case if they are able to + * determine that the artifact is not injected without statting it. + */ + boolean isInjected(Artifact artifact) throws IOException; + + /** Discards all metadata for the given artifacts, presumably because they will be modified. */ + void discardMetadata(Collection<Artifact> artifactList); + +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java new file mode 100644 index 0000000000..097515075f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions.cache; + +import java.io.IOException; +import java.io.PrintStream; + +/** + * A no-op action cache that never caches anything. + */ +public final class NullActionCache implements ActionCache { + + @Override + public void put(String key, Entry entry) { + } + + @Override + public Entry get(String key) { + return null; + } + + @Override + public void remove(String key) { + } + + @Override + public Entry createEntry(String key) { + return new ActionCache.Entry(key); + } + + @Override + public long save() throws IOException { + return 0; + } + + @Override + public void dump(PrintStream out) { + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java new file mode 100644 index 0000000000..bd98b2bfda --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexer.java @@ -0,0 +1,161 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.common.collect.MapMaker; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import com.google.devtools.build.lib.util.CanonicalStringIndexer; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.PersistentMap; +import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * Persistent version of the CanonicalStringIndexer. + * + * <p>This class is backed by a PersistentMap that holds one direction of the + * canonicalization mapping. The other direction is handled purely in memory + * and reconstituted at load-time. + * + * <p>Thread-safety is ensured by locking on all mutating operations from the + * superclass. Read-only operations are not locked, but rather backed by + * ConcurrentMaps. + */ +@ConditionallyThreadSafe // condition: each instance must instantiated with + // different dataFile. +final class PersistentStringIndexer extends CanonicalStringIndexer { + + /** + * Persistent metadata map. Used as a backing map to provide a persistent + * implementation of the metadata cache. + */ + private static final class PersistentIndexMap extends PersistentMap<String, Integer> { + private static final int VERSION = 0x01; + private static final long SAVE_INTERVAL_NS = 3L * 1000 * 1000 * 1000; + + private final Clock clock; + private long nextUpdate; + + public PersistentIndexMap(Path mapFile, Path journalFile, Clock clock) throws IOException { + super(VERSION, PersistentStringIndexer.<String, Integer>newConcurrentMap(INITIAL_ENTRIES), + mapFile, journalFile); + this.clock = clock; + nextUpdate = clock.nanoTime(); + load(/*throwOnLoadFailure=*/true); + } + + @Override + protected boolean updateJournal() { + long time = clock.nanoTime(); + if (SAVE_INTERVAL_NS == 0 || time > nextUpdate) { + nextUpdate = time + SAVE_INTERVAL_NS; + return true; + } + return false; + } + + @Override + public Integer remove(Object object) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + public void flush() { + super.forceFlush(); + } + + @Override + protected String readKey(DataInputStream in) throws IOException { + int length = in.readInt(); + if (length < 0) { + throw new IOException("corrupt key length: " + length); + } + byte[] content = new byte[length]; + in.readFully(content); + return StringCanonicalizer.intern(bytes2string(content)); + } + + @Override + protected Integer readValue(DataInputStream in) throws IOException { + return in.readInt(); + } + + @Override + protected void writeKey(String key, DataOutputStream out) throws IOException { + byte[] content = string2bytes(key); + out.writeInt(content.length); + out.write(content); + } + + @Override + protected void writeValue(Integer value, DataOutputStream out) throws IOException { + out.writeInt(value); + } + } + + private final PersistentIndexMap persistentIndexMap; + private static final int INITIAL_ENTRIES = 10000; + + /** + * Instantiates and loads instance of the persistent string indexer. + */ + static PersistentStringIndexer newPersistentStringIndexer(Path dataPath, + Clock clock) throws IOException { + PersistentIndexMap persistentIndexMap = new PersistentIndexMap(dataPath, + FileSystemUtils.replaceExtension(dataPath, ".journal"), clock); + Map<Integer, String> reverseMapping = newConcurrentMap(INITIAL_ENTRIES); + for (Map.Entry<String, Integer> entry : persistentIndexMap.entrySet()) { + if (reverseMapping.put(entry.getValue(), entry.getKey()) != null) { + throw new IOException("Corrupted filename index has duplicate entry: " + entry.getKey()); + } + } + return new PersistentStringIndexer(persistentIndexMap, reverseMapping); + } + + private PersistentStringIndexer(PersistentIndexMap stringToInt, + Map<Integer, String> intToString) { + super(stringToInt, intToString); + this.persistentIndexMap = stringToInt; + } + + /** + * Saves index data to the file. + */ + synchronized long save() throws IOException { + return persistentIndexMap.save(); + } + + /** + * Flushes the journal. + */ + synchronized void flush() { + persistentIndexMap.flush(); + } + + private static <K, V> ConcurrentMap<K, V> newConcurrentMap(int expectedCapacity) { + return new MapMaker().initialCapacity(expectedCapacity).makeMap(); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java b/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java new file mode 100644 index 0000000000..debb8eac52 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/VirtualActionInput.java @@ -0,0 +1,31 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import com.google.devtools.build.lib.actions.ActionInput; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * An ActionInput that does not actually exist on the filesystem, but can still be written to an + * OutputStream. + */ +public interface VirtualActionInput extends ActionInput { + /** + * Writes the the fake file to an OutputStream. MUST be deterministic, in that multiple calls + * to write the same VirtualActionInput must write identical bytes. + */ + void writeTo(OutputStream out) throws IOException; +} |