Introducing PopulateTreeArtifactAction, an Action that populates a TreeArtifact with the content of an archive file at execution time by:

1. Reads the archive manifest file on disk.
2. Executes a spawn that expands the archive manifest entries of the archive file    into/under the TreeArtifact.
3. Registers the manifest file entries as TreeFileArtifacts of the TreeArtifact.

--
MOS_MIGRATED_REVID=123107850

1 files changed, 316 insertions, 0 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/PopulateTreeArtifactAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/PopulateTreeArtifactAction.java
new file mode 100644
index 0000000000..ffabb0b2f3
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/PopulateTreeArtifactAction.java
@@ -0,0 +1,316 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.analysis.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.AbstractAction;
+import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
+import com.google.devtools.build.lib.actions.ActionExecutionContext;
+import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.ActionOwner;
+import com.google.devtools.build.lib.actions.Actions;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
+import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
+import com.google.devtools.build.lib.actions.BaseSpawn;
+import com.google.devtools.build.lib.actions.ExecException;
+import com.google.devtools.build.lib.actions.Executor;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.actions.SpawnActionContext;
+import com.google.devtools.build.lib.analysis.FilesToRunProvider;
+import com.google.devtools.build.lib.util.Fingerprint;
+import com.google.devtools.build.lib.util.Preconditions;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.PathFragment;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * An action that populates a TreeArtifact with the contents of an archive file.
+ *
+ * <p>Internally, the following happens at execution time:
+ * <ol>
+ *   <li>The archive entry paths are read from the associated archive manifest file locally.
+ *   <li>A spawn is executed to unzip the archive contents under the root of the TreeArtifact.
+ *   <li>Child TreeFileArtifacts are created using the archive entry paths and then populated into
+ *       the output TreeArtifact.
+ * </ol>
+ *
+ * <p>There are also several requirements regarding the archive and archive manifest file:
+ * <ul>
+ *   <li>The entry names (paths) of the archive and archive manifest file must be valid ISO-8859-1
+ *       strings.
+ *   <li>The archive manifest file must not contain absolute, non-normalized
+ *       (e.g., containing '..' fragments) or duplicated paths. And no path is allowed to be a
+ *       prefix of another path.
+ * </ul>
+ */
+public final class PopulateTreeArtifactAction extends AbstractAction {
+  private static final String GUID = "a3d36f29-9f14-42cf-a014-3a51e914e482";
+
+  @VisibleForTesting
+  static final String MNEMONIC = "PopulateTreeArtifact";
+
+  private final Artifact archive;
+  private final Artifact archiveManifest;
+  private final Artifact outputTreeArtifact;
+  private final FilesToRunProvider zipper;
+
+  /**
+   * Creates a PopulateTreeArtifactAction object.
+   *
+   * @param owner the owner of the action.
+   * @param archive the archive containing files to populate into the TreeArtifact.
+   * @param archiveManifest the archive manifest file specifying the entry files to populate into
+   *     the TreeArtifact.
+   * @param treeArtifactToPopulate the TreeArtifact to be populated with archive member files.
+   * @param zipper the zipper executable used to unzip the archive.
+   */
+  public PopulateTreeArtifactAction(
+      ActionOwner owner,
+      Artifact archive,
+      Artifact archiveManifest,
+      Artifact treeArtifactToPopulate,
+      FilesToRunProvider zipper) {
+    super(
+        owner,
+        ImmutableList.copyOf(zipper.getFilesToRun()),
+        Iterables.concat(
+            ImmutableList.of(archive, archiveManifest),
+            ImmutableList.copyOf(zipper.getFilesToRun())),
+        ImmutableList.of(treeArtifactToPopulate));
+
+    Preconditions.checkArgument(
+        treeArtifactToPopulate.isTreeArtifact(),
+        "%s is not TreeArtifact",
+        treeArtifactToPopulate);
+
+    this.archive = archive;
+    this.archiveManifest = archiveManifest;
+    this.outputTreeArtifact = treeArtifactToPopulate;
+    this.zipper = zipper;
+  }
+
+  private static class PopulateTreeArtifactSpawn extends BaseSpawn {
+    private final Artifact treeArtifact;
+    private final Iterable<PathFragment> entriesToExtract;
+
+    // The output TreeFileArtifacts are created lazily outside of the contructor because potentially
+    // we can have a lot of TreeFileArtifacts under a given tree artifact.
+    private Collection<TreeFileArtifact> outputTreeFileArtifacts;
+
+    PopulateTreeArtifactSpawn(
+        Artifact treeArtifact,
+        Iterable<PathFragment> entriesToExtract,
+        Iterable<String> commandLine,
+        Map<PathFragment, Artifact> runfilesManifests,
+        ActionExecutionMetadata action) {
+      super(
+        ImmutableList.copyOf(commandLine),
+        ImmutableMap.<String, String>of(),
+        ImmutableMap.<String, String>of(),
+        ImmutableMap.copyOf(runfilesManifests),
+        action,
+        AbstractAction.DEFAULT_RESOURCE_SET);
+      this.treeArtifact = treeArtifact;
+      this.entriesToExtract = entriesToExtract;
+    }
+
+    @Override
+    public Collection<? extends ActionInput> getOutputFiles() {
+      if (outputTreeFileArtifacts == null) {
+        outputTreeFileArtifacts = ImmutableList.<TreeFileArtifact>copyOf(
+            ActionInputHelper.asTreeFileArtifacts(treeArtifact, entriesToExtract));
+      }
+      return outputTreeFileArtifacts;
+    }
+  }
+
+  @Override
+  public Artifact getPrimaryOutput() {
+    return outputTreeArtifact;
+  }
+
+  @Override
+  public void execute(ActionExecutionContext actionExecutionContext)
+      throws ActionExecutionException, InterruptedException {
+    Executor executor = actionExecutionContext.getExecutor();
+    Spawn spawn;
+
+    // Create a spawn to unzip the archive file into the output TreeArtifact.
+    try {
+      spawn = createSpawn();
+    } catch (IOException e) {
+      throw new ActionExecutionException(e, this, false);
+    } catch (IllegalManifestFileException e) {
+      throw new ActionExecutionException(e, this, true);
+    }
+
+    // Check spawn output TreeFileArtifact conflicts.
+    try {
+      checkOutputConflicts(spawn.getOutputFiles());
+    } catch (ArtifactPrefixConflictException e) {
+      throw new ActionExecutionException(e, this, true);
+    }
+
+    // Execute the spawn.
+    try {
+      getContext(executor).exec(spawn, actionExecutionContext);
+    } catch (ExecException e) {
+      throw e.toActionExecutionException(
+          getMnemonic() + " action failed for target: " + getOwner().getLabel(),
+          executor.getVerboseFailures(),
+          this);
+    }
+
+    // Populate the output TreeArtifact with the Spawn output TreeFileArtifacts.
+    for (ActionInput fileEntry : spawn.getOutputFiles()) {
+      actionExecutionContext.getMetadataHandler().addExpandedTreeOutput(
+          (TreeFileArtifact) fileEntry);
+    }
+  }
+
+  @Override
+  protected String computeKey() {
+    Fingerprint f = new Fingerprint();
+    f.addString(GUID);
+    f.addString(getMnemonic());
+    f.addStrings(spawnCommandLine());
+    Map<PathFragment, Artifact> zipperManifest = zipperExecutableRunfilesManifest();
+    f.addInt(zipperManifest.size());
+    for (Map.Entry<PathFragment, Artifact> input : zipperManifest.entrySet()) {
+      f.addString(input.getKey().getPathString() + "/");
+      f.addPath(input.getValue().getExecPath());
+    }
+
+    return f.hexDigestAndReset();
+  }
+
+  @Override
+  public String getMnemonic() {
+    return "PopulateTreeArtifact";
+  }
+
+  @Override
+  public boolean shouldReportPathPrefixConflict(ActionAnalysisMetadata action) {
+    return true;
+  }
+
+  @Override
+  public ResourceSet estimateResourceConsumption(Executor executor) {
+    if (getContext(executor).willExecuteRemotely(true)) {
+      return ResourceSet.ZERO;
+    }
+    return AbstractAction.DEFAULT_RESOURCE_SET;
+  }
+
+  private SpawnActionContext getContext(Executor executor) {
+    return executor.getSpawnActionContext(getMnemonic());
+  }
+
+  /**
+   * Creates a spawn to unzip the archive members specified in the archive manifest into the
+   * TreeArtifact.
+   */
+  @VisibleForTesting
+  Spawn createSpawn() throws IOException, IllegalManifestFileException {
+    Iterable<PathFragment> entries = readAndCheckManifestEntries();
+    return new PopulateTreeArtifactSpawn(
+        outputTreeArtifact,
+        entries,
+        spawnCommandLine(),
+        zipperExecutableRunfilesManifest(),
+        this);
+  }
+
+  private Iterable<String> spawnCommandLine() {
+    return ImmutableList.of(
+        zipper.getExecutable().getExecPathString(),
+        "x",
+        archive.getExecPathString(),
+        "-d",
+        outputTreeArtifact.getExecPathString(),
+        "@" + archiveManifest.getExecPathString());
+  }
+
+  private Map<PathFragment, Artifact> zipperExecutableRunfilesManifest() {
+    if (zipper.getRunfilesManifest() != null) {
+      return ImmutableMap.of(
+          BaseSpawn.runfilesForFragment(zipper.getExecutable().getExecPath()),
+          zipper.getRunfilesManifest());
+    } else {
+      return ImmutableMap.<PathFragment, Artifact>of();
+    }
+  }
+
+  private Iterable<PathFragment> readAndCheckManifestEntries()
+      throws IOException, IllegalManifestFileException {
+    ImmutableList.Builder<PathFragment> manifestEntries = ImmutableList.builder();
+    boolean hasNonEmptyLines = false;
+
+    for (String line :
+        FileSystemUtils.iterateLinesAsLatin1(archiveManifest.getPath())) {
+      if (!line.isEmpty()) {
+        hasNonEmptyLines = true;
+        PathFragment path = new PathFragment(line);
+
+        if (!path.isNormalized() || path.isAbsolute()) {
+          throw new IllegalManifestFileException(
+              path + " is not a proper relative path");
+        }
+
+        manifestEntries.add(path);
+      }
+    }
+
+    if (!hasNonEmptyLines) {
+      throw new IllegalManifestFileException(
+          String.format("Archive manifest %s must not be empty.", archiveManifest));
+    }
+
+    return manifestEntries.build();
+  }
+
+  private void checkOutputConflicts(Collection<? extends ActionInput> outputs)
+      throws ArtifactPrefixConflictException {
+    ImmutableMap.Builder<Artifact, ActionAnalysisMetadata> generatingActions =
+        ImmutableMap.<Artifact, ActionAnalysisMetadata>builder();
+    for (ActionInput output : outputs) {
+      generatingActions.put((Artifact) output, this);
+    }
+
+    Map<ActionAnalysisMetadata, ArtifactPrefixConflictException> artifactPrefixConflictMap =
+        Actions.findArtifactPrefixConflicts(generatingActions.build());
+
+    if (!artifactPrefixConflictMap.isEmpty()) {
+      throw artifactPrefixConflictMap.values().iterator().next();
+    }
+  }
+
+  private static class IllegalManifestFileException extends Exception {
+
+    IllegalManifestFileException(String message) {
+      super(message);
+    }
+  }
+}