From e3b1cb765a04c858a87ca7c7b0ecfa63d55be269 Mon Sep 17 00:00:00 2001 From: Michael Thvedt Date: Mon, 8 Feb 2016 23:32:27 +0000 Subject: Introduce TreeArtifact and associated code to work with it. No functionality implemented yet. -- MOS_MIGRATED_REVID=114157140 --- .../build/lib/actions/ActionInputHelper.java | 35 ++++++- .../devtools/build/lib/actions/Artifact.java | 116 ++++++++++++++++----- .../build/lib/actions/ArtifactFactory.java | 14 +++ .../devtools/build/lib/actions/ArtifactFile.java | 115 ++++++++++++++++++++ .../build/lib/actions/TreeArtifactFile.java | 92 ++++++++++++++++ .../build/lib/analysis/AnalysisEnvironment.java | 7 ++ .../lib/analysis/CachingAnalysisEnvironment.java | 8 ++ .../devtools/build/lib/analysis/RuleContext.java | 15 +++ 8 files changed, 373 insertions(+), 29 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/actions/ArtifactFile.java create mode 100644 src/main/java/com/google/devtools/build/lib/actions/TreeArtifactFile.java (limited to 'src/main/java/com/google/devtools') diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java index 09558bf787..723c9fbd37 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ActionInputHelper.java @@ -20,6 +20,7 @@ import com.google.common.base.Functions; import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; import com.google.devtools.build.lib.util.Preconditions; +import com.google.devtools.build.lib.vfs.PathFragment; import java.util.ArrayList; import java.util.Collection; @@ -124,6 +125,38 @@ public final class ActionInputHelper { return Collections2.transform(paths, FROM_PATH); } + /** + * Instantiates a concrete ArtifactFile with the given parent Artifact and path + * relative to that Artifact. + */ + public static ArtifactFile artifactFile(Artifact parent, PathFragment relativePath) { + Preconditions.checkState(parent.isTreeArtifact(), + "Given parent %s must be a TreeArtifact", parent); + return new TreeArtifactFile(parent, relativePath); + } + + /** + * Instantiates a concrete ArtifactFile with the given parent Artifact and path + * relative to that Artifact. + */ + public static ArtifactFile artifactFile(Artifact parent, String relativePath) { + return artifactFile(parent, new PathFragment(relativePath)); + } + + /** Returns an Iterable of ArtifactFiles with the given parent and parent relative paths. */ + public static Iterable asArtifactFiles( + final Artifact parent, Iterable parentRelativePaths) { + Preconditions.checkState(parent.isTreeArtifact(), + "Given parent %s must be a TreeArtifact", parent); + return Iterables.transform(parentRelativePaths, + new Function() { + @Override + public ArtifactFile apply(PathFragment pathFragment) { + return artifactFile(parent, pathFragment); + } + }); + } + /** * Expands middleman artifacts in a sequence of {@link ActionInput}s. * @@ -145,7 +178,7 @@ public final class ActionInputHelper { return result; } - /** Formatter for execPath String output. Public because Artifact uses it directly. */ + /** Formatter for execPath String output. Public because {@link Artifact} uses it directly. */ public static final Function EXEC_PATH_STRING_FORMATTER = new Function() { @Override diff --git a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java index 24bece9edd..23e2a4cb46 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/Artifact.java +++ b/src/main/java/com/google/devtools/build/lib/actions/Artifact.java @@ -65,6 +65,29 @@ import javax.annotation.Nullable; * during include validation, will also have null generating Actions. * * + * In the usual case, an Artifact represents a single file. However, an Artifact may + * also represent the following: + *
    + *
  • A TreeArtifact, which is a directory containing a tree of unknown {@link ArtifactFile}s. + * In the future, Actions will be able to examine these files as inputs and declare them as outputs + * at execution time, but this is not yet implemented. This is used for Actions where + * the inputs and/or outputs might not be discoverable except during Action execution. + *
  • A directory of unknown contents, but not a TreeArtifact. + * This is a legacy facility and should not be used by any new rule implementations. + * In particular, the file system cache integrity checks fail for directories. + *
  • An 'aggregating middleman' special Artifact, which may be expanded using a + * {@link MiddlemanExpander} at Action execution time. This is used by a handful of rules to save + * memory. + *
  • A 'constant metadata' special Artifact. These represent real files, changes to which are + * ignored by the build system. They are useful for files which change frequently but do not affect + * the result of a build, such as timestamp files. + *
  • A 'Fileset' special Artifact. This is a legacy type of Artifact and should not be used + * by new rule implementations. + *
+ *

+ * This class implements {@link ArtifactFile}, and is modeled as an Artifact "containing" itself + * as an ArtifactFile. + *

*

This class is "theoretically" final; it should not be subclassed except by * {@link SpecialArtifact}. */ @@ -72,7 +95,7 @@ import javax.annotation.Nullable; @SkylarkModule(name = "File", doc = "This type represents a file used by the build system. It can be " + "either a source file or a derived file produced by a rule.") -public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue { +public class Artifact implements FileType.HasFilename, ArtifactFile, SkylarkValue { /** * Compares artifact according to their exec paths. Sorts null values first. @@ -204,13 +227,22 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue root.getExecPath().getRelative(rootRelativePath), ArtifactOwner.NULL_OWNER); } - /** - * Returns the location of this Artifact on the filesystem. - */ + @Override public final Path getPath() { return path; } + /** + * Returns the Artifact containing this ArtifactFile. Since normal Artifacts correspond + * to only one ArtifactFile -- itself -- for normal Artifacts, this method returns {@code this}. + * For special artifacts, throws {@link UnsupportedOperationException}. + * See also {@link ArtifactFile#getParent()}. + */ + @Override + public Artifact getParent() throws UnsupportedOperationException { + return this; + } + /** * Returns the directory name of this artifact, similar to dirname(1). * @@ -271,6 +303,7 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue * package-path entries (for source Artifacts), or one of the bin, genfiles or includes dirs * (for derived Artifacts). It will always be an ancestor of getPath(). */ + @Override @SkylarkCallable(name = "root", structField = true, doc = "The root beneath which this file resides." ) @@ -278,15 +311,22 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue return root; } - /** - * Returns the exec path of this Artifact. The exec path is a relative path - * that is suitable for accessing this artifact relative to the execution - * directory for this build. - */ + @Override public final PathFragment getExecPath() { return execPath; } + /** + * Returns the path of this ArtifactFile relative to this containing Artifact. Since + * ordinary Artifacts correspond to only one ArtifactFile -- itself -- for ordinary Artifacts, + * this just returns the empty path. For special Artifacts, throws + * {@link UnsupportedOperationException}. See also {@link ArtifactFile#getParentRelativePath()}. + */ + @Override + public PathFragment getParentRelativePath() { + return PathFragment.EMPTY_FRAGMENT; + } + /** * Returns true iff this is a source Artifact as determined by its path and * root relationships. Note that this will report all Artifacts in the output @@ -305,6 +345,13 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue return getRoot().isMiddlemanRoot(); } + /** + * Returns true iff this is a TreeArtifact representing a directory tree containing ArtifactFiles. + */ + public boolean isTreeArtifact() { + return false; + } + /** * Returns whether the artifact represents a Fileset. */ @@ -325,8 +372,10 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue * * @see SpecialArtifact */ - static enum SpecialArtifactType { + @VisibleForTesting + public static enum SpecialArtifactType { FILESET, + TREE, CONSTANT_METADATA, } @@ -342,7 +391,8 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue public static final class SpecialArtifact extends Artifact { private final SpecialArtifactType type; - SpecialArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner, + @VisibleForTesting + public SpecialArtifact(Path path, Root root, PathFragment execPath, ArtifactOwner owner, SpecialArtifactType type) { super(path, root, execPath, owner); this.type = type; @@ -357,12 +407,28 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue public boolean isConstantMetadata() { return type == SpecialArtifactType.CONSTANT_METADATA; } + + @Override + public boolean isTreeArtifact() { + return type == SpecialArtifactType.TREE; + } + + @Override + public Artifact getParent() { + throw new UnsupportedOperationException(); + } + + @Override + public PathFragment getParentRelativePath() { + throw new UnsupportedOperationException(); + } } /** * Returns the relative path to this artifact relative to its root. (Useful * when deriving output filenames from input files, etc.) */ + @Override public final PathFragment getRootRelativePath() { return rootRelativePath; } @@ -397,13 +463,7 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue return getRootRelativePath().getPathString(); } - /** - * Returns a pretty string representation of the path denoted by this artifact, suitable for use - * in user error messages. Artifacts beneath a root will be printed relative to that root; other - * artifacts will be printed as an absolute path. - * - *

(The toString method is intended for developer messages since its more informative.) - */ + @Override public final String prettyPrint() { // toDetailString would probably be more useful to users, but lots of tests rely on the // current values. @@ -475,26 +535,26 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue /** * Formatter for execPath PathFragment output. */ - private static final Function EXEC_PATH_FORMATTER = - new Function() { + private static final Function EXEC_PATH_FORMATTER = + new Function() { @Override - public PathFragment apply(Artifact input) { + public PathFragment apply(ArtifactFile input) { return input.getExecPath(); } }; - public static final Function ROOT_RELATIVE_PATH_STRING = - new Function() { + public static final Function ROOT_RELATIVE_PATH_STRING = + new Function() { @Override - public String apply(Artifact artifact) { + public String apply(ArtifactFile artifact) { return artifact.getRootRelativePath().getPathString(); } }; - public static final Function ABSOLUTE_PATH_STRING = - new Function() { + public static final Function ABSOLUTE_PATH_STRING = + new Function() { @Override - public String apply(Artifact artifact) { + public String apply(ArtifactFile artifact) { return artifact.getPath().getPathString(); } }; @@ -708,7 +768,7 @@ public class Artifact implements FileType.HasFilename, ActionInput, SkylarkValue /** * Converts artifacts into their exec paths. Returns an immutable list. */ - public static List asPathFragments(Iterable artifacts) { + public static List asPathFragments(Iterable artifacts) { return ImmutableList.copyOf(Iterables.transform(artifacts, EXEC_PATH_FORMATTER)); } diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java index 090059677a..45c195edac 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFactory.java @@ -241,6 +241,20 @@ public class ArtifactFactory implements ArtifactResolver, ArtifactSerializer, Ar return getArtifact(path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.FILESET); } + /** + * Returns an artifact that represents a TreeArtifact; that is, a directory containing some + * tree of ArtifactFiles unknown at analysis time. + * + *

The root must be below the execRoot, and the execPath of the resulting Artifact is computed + * as {@code root.getRelative(rootRelativePath).relativeTo(execRoot)}. + */ + public Artifact getTreeArtifact(PathFragment rootRelativePath, Root root, + ArtifactOwner owner) { + validatePath(rootRelativePath, root); + Path path = root.getPath().getRelative(rootRelativePath); + return getArtifact(path, root, path.relativeTo(execRoot), owner, SpecialArtifactType.TREE); + } + public Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root, ArtifactOwner owner) { validatePath(rootRelativePath, root); diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactFile.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFile.java new file mode 100644 index 0000000000..be165fbda0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactFile.java @@ -0,0 +1,115 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * An ArtifactFile represents a file used by the build system during execution of Actions. + * An ArtifactFile may be a source file or a derived (output) file. + * + * During the creation of {@link com.google.devtools.build.lib.analysis.ConfiguredTarget}s and + * {@link com.google.devtools.build.lib.analysis.ConfiguredAspect}s, generally one + * is only interested in {@link Artifact}s. Most Actions are only interested in Artifacts as well. + * During action execution, however, some Artifacts (notably + * TreeArtifacts) may correspond to more than one ArtifactFile. This is for the benefit + * of some Actions which only know their inputs or outputs at execution time, via the + * TreeArtifact mechanism. + *

  • + * Generally, most Artifacts represent a file on the filesystem, and correspond to exactly one + * ArtifactFile. For ease of use, Artifact itself implements ArtifactFile, and is modeled + * as an Artifact containing itself. + *
  • + * Some Artifacts are so-called TreeArtifacts, for which the method + * {@link Artifact#isTreeArtifact()} returns true. These artifacts contain a (possibly empty) + * tree of ArtifactFiles, which are unknown until Action execution time. + *
  • + * Some 'special artifacts' do not meaningfully represent a file or tree of files. + *
+ */ +public interface ArtifactFile extends ActionInput { + /** + * Returns the exec path of this ArtifactFile. The exec path is a relative path + * that is suitable for accessing this artifact relative to the execution + * directory for this build. + */ + PathFragment getExecPath(); + + /** + * Returns the path of this ArtifactFile relative to its containing Artifact. + * See {@link #getParent()}. + * */ + PathFragment getParentRelativePath(); + + /** Returns the location of this ArtifactFile on the filesystem. */ + Path getPath(); + + /** + * Returns the relative path to this ArtifactFile relative to its root. Useful + * when deriving output filenames from input files, etc. + */ + PathFragment getRootRelativePath(); + + /** + * Returns the root beneath which this ArtifactFile resides, if any. This may be one of the + * package-path entries (for files belonging to source {@link Artifact}s), or one of the bin + * genfiles or includes dirs (for files belonging to derived {@link Artifact}s). It will always be + * an ancestor of {@link #getPath()}. + */ + Root getRoot(); + + /** + * Returns the Artifact containing this File. For Artifacts which are files, returns + * {@code this}. Otherwise, returns a Artifact whose path and root relative path + * are an ancestor of this ArtifactFile's path, and for which {@link Artifact#isTreeArtifact()} + * returns true. + *

+ * For ArtifactFiles which are special artifacts (including TreeArtifacts), + * this method throws UnsupportedOperationException, because the ArtifactFile abstraction + * fails for those cases. + */ + Artifact getParent() throws UnsupportedOperationException; + + /** + * Returns a pretty string representation of the path denoted by this ArtifactFile, suitable for + * use in user error messages. ArtifactFiles beneath a root will be printed relative to that root; + * other ArtifactFiles will be printed as an absolute path. + * + *

(The toString method is intended for developer messages since its more informative.) + */ + String prettyPrint(); + + /** + * Two ArtifactFiles are equal if their parent Artifacts and parent relative paths + * are equal. If this ArtifactFile is itself an Artifact, see {@link Artifact#equals(Object)}. + * Note that within a build, two ArtifactFiles produced by the build system will be equal + * if and only if their full paths are equal. + */ + @Override + boolean equals(Object o); + + /** + * An ArtifactFile's hash code should be equal to + * {@code {@link #getParent().hashCode()} * 257 + + * {@link #getParentRelativePath().hashCode()}}. If this ArtifactFile is itself an Artifact, + * see {@link Artifact#hashCode()}. + *

+ * 257 is chosen because it is a Fermat prime, and has minimal 'bit overlap' with the Mersenne + * prime of 31 used in {@link Path#hashCode()} and {@link String#hashCode()}. + */ + @Override + int hashCode(); +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/TreeArtifactFile.java b/src/main/java/com/google/devtools/build/lib/actions/TreeArtifactFile.java new file mode 100644 index 0000000000..8b4cde81d7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/TreeArtifactFile.java @@ -0,0 +1,92 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.actions; + +import com.google.devtools.build.lib.util.Preconditions; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** An ArtifactFile implementation for descendants of TreeArtifacts. */ +final class TreeArtifactFile implements ArtifactFile { + private final Artifact parent; + private final PathFragment parentRelativePath; + + TreeArtifactFile(Artifact parent, PathFragment parentRelativePath) { + Preconditions.checkArgument(parent.isTreeArtifact(), "%s must be a TreeArtifact", parent); + this.parent = parent; + this.parentRelativePath = parentRelativePath; + } + + @Override + public PathFragment getExecPath() { + return parent.getExecPath().getRelative(parentRelativePath); + } + + @Override + public PathFragment getParentRelativePath() { + return parentRelativePath; + } + + @Override + public Path getPath() { + return parent.getPath().getRelative(parentRelativePath); + } + + @Override + public PathFragment getRootRelativePath() { + return parent.getRootRelativePath().getRelative(parentRelativePath); + } + + @Override + public Root getRoot() { + return parent.getRoot(); + } + + @Override + public Artifact getParent() { + return parent; + } + + @Override + public String prettyPrint() { + return getRootRelativePath().toString(); + } + + @Override + public String getExecPathString() { + return getExecPath().toString(); + } + + @Override + public String toString() { + return "ArtifactFile:[" + parent.toDetailString() + "]" + parentRelativePath; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ArtifactFile)) { + return false; + } + + ArtifactFile that = (ArtifactFile) other; + return this.getParent().equals(that.getParent()) + && this.getParentRelativePath().equals(that.getParentRelativePath()); + } + + @Override + public int hashCode() { + return getParent().hashCode() * 257 + getParentRelativePath().hashCode(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java index f739246914..bf0213959e 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisEnvironment.java @@ -69,6 +69,13 @@ public interface AnalysisEnvironment extends ActionRegistry { Artifact getConstantMetadataArtifact(PathFragment rootRelativePath, Root root); + /** + * Returns the artifact for the derived TreeArtifact with directory {@code rootRelativePath}, + * creating it if necessary, and setting the root of that artifact to + * {@code root}. The artifact will be a TreeArtifact. + */ + Artifact getTreeArtifact(PathFragment rootRelativePath, Root root); + /** * Returns the artifact for the derived file {@code rootRelativePath}, * creating it if necessary, and setting the root of that artifact to diff --git a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java index bfdd8b99a3..4d1d787095 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/CachingAnalysisEnvironment.java @@ -230,6 +230,14 @@ public class CachingAnalysisEnvironment implements AnalysisEnvironment { extendedSanityChecks ? new Throwable() : null); } + @Override + public Artifact getTreeArtifact(PathFragment rootRelativePath, Root root) { + Preconditions.checkState(enabled); + return trackArtifactAndOrigin( + artifactFactory.getTreeArtifact(rootRelativePath, root, getOwner()), + extendedSanityChecks ? new Throwable() : null); + } + @Override public Artifact getFilesetArtifact(PathFragment rootRelativePath, Root root) { Preconditions.checkState(enabled); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java index 68b54e8773..f0fecff215 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java @@ -562,6 +562,21 @@ public final class RuleContext extends TargetContext rootRelativePath, getPackageDirectory(), getLabel()); return getAnalysisEnvironment().getDerivedArtifact(rootRelativePath, root); } + + /** + * Creates a TreeArtifact under a given root with the given root-relative path. + * + *

Verifies that it is in the root-relative directory corresponding to the package of the rule, + * thus ensuring that it doesn't clash with other artifacts generated by other rules using this + * method. + */ + public Artifact getTreeArtifact(PathFragment rootRelativePath, Root root) { + Preconditions.checkState(rootRelativePath.startsWith(getPackageDirectory()), + "Output artifact '%s' not under package directory '%s' for target '%s'", + rootRelativePath, getPackageDirectory(), getLabel()); + return getAnalysisEnvironment().getTreeArtifact(rootRelativePath, root); + } + /** * Creates an artifact in a directory that is unique to the rule, thus guaranteeing that it never * clashes with artifacts created by other rules. -- cgit v1.2.3