diff options
Diffstat (limited to 'src/test/java/com/google/devtools/build/lib/skyframe')
5 files changed, 645 insertions, 19 deletions
diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ActionTemplateExpansionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ActionTemplateExpansionFunctionTest.java new file mode 100644 index 0000000000..790429a735 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ActionTemplateExpansionFunctionTest.java @@ -0,0 +1,245 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; +import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException; +import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate; +import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate.OutputPathMapper; +import com.google.devtools.build.lib.events.NullEventHandler; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.skyframe.ArtifactValue.OwnedArtifact; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.util.Preconditions; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequentialBuildDriver; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; + +/** Tests for {@link ActionTemplateExpansionFunction}. */ +@RunWith(JUnit4.class) +public final class ActionTemplateExpansionFunctionTest extends FoundationTestCase { + private Map<Artifact, ArtifactValue> artifactValueMap; + private SequentialBuildDriver driver; + + @Before + public void setUp() throws Exception { + artifactValueMap = new LinkedHashMap<>(); + AtomicReference<PathPackageLocator> pkgLocator = new AtomicReference<>(new PathPackageLocator( + rootDirectory.getFileSystem().getPath("/outputbase"), ImmutableList.of(rootDirectory))); + RecordingDifferencer differencer = new RecordingDifferencer(); + MemoizingEvaluator evaluator = + new InMemoryMemoizingEvaluator( + ImmutableMap.<SkyFunctionName, SkyFunction>builder() + .put(SkyFunctions.ARTIFACT, + new DummyArtifactFunction(artifactValueMap)) + .put(SkyFunctions.ACTION_TEMPLATE_EXPANSION, new ActionTemplateExpansionFunction()) + .build(), + differencer); + driver = new SequentialBuildDriver(evaluator); + PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID()); + PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get()); + } + + @Test + public void testActionTemplateExpansionFunction() throws Exception { + Artifact inputTreeArtifact = createAndPopulateTreeArtifact( + "inputTreeArtifact", "child0", "child1", "child2"); + Artifact outputTreeArtifact = createTreeArtifact("outputTreeArtifact"); + + SpawnActionTemplate spawnActionTemplate = ActionsTestUtil.createDummySpawnActionTemplate( + inputTreeArtifact, outputTreeArtifact); + List<Action> actions = evaluate(spawnActionTemplate); + assertThat(actions).hasSize(3); + + ArtifactOwner owner = ActionTemplateExpansionValue.createActionTemplateExpansionKey( + spawnActionTemplate); + int i = 0; + for (Action action : actions) { + String childName = "child" + i; + assertThat(Artifact.toExecPaths(action.getInputs())).contains( + "out/inputTreeArtifact/" + childName); + assertThat(Artifact.toExecPaths(action.getOutputs())).containsExactly( + "out/outputTreeArtifact/" + childName); + assertThat(Iterables.getOnlyElement(action.getOutputs()).getArtifactOwner()).isEqualTo(owner); + ++i; + } + } + + @Test + public void testThrowsOnActionConflict() throws Exception { + Artifact inputTreeArtifact = createAndPopulateTreeArtifact( + "inputTreeArtifact", "child0", "child1", "child2"); + Artifact outputTreeArtifact = createTreeArtifact("outputTreeArtifact"); + + + OutputPathMapper mapper = new OutputPathMapper() { + @Override + public PathFragment parentRelativeOutputPath(TreeFileArtifact inputTreeFileArtifact) { + return new PathFragment("conflict_path"); + } + }; + SpawnActionTemplate spawnActionTemplate = + new SpawnActionTemplate.Builder(inputTreeArtifact, outputTreeArtifact) + .setExecutable(new PathFragment("/bin/cp")) + .setCommandLineTemplate(CustomCommandLine.builder().build()) + .setOutputPathMapper(mapper) + .build(ActionsTestUtil.NULL_ACTION_OWNER); + + try { + evaluate(spawnActionTemplate); + fail("Expected ActionConflictException"); + } catch (ActionConflictException e) { + // Expected ActionConflictException + } + } + + @Test + public void testThrowsOnArtifactPrefixConflict() throws Exception { + Artifact inputTreeArtifact = createAndPopulateTreeArtifact( + "inputTreeArtifact", "child0", "child1", "child2"); + Artifact outputTreeArtifact = createTreeArtifact("outputTreeArtifact"); + + OutputPathMapper mapper = new OutputPathMapper() { + private int i = 0; + @Override + public PathFragment parentRelativeOutputPath(TreeFileArtifact inputTreeFileArtifact) { + PathFragment path; + switch (i) { + case 0: + path = new PathFragment("path_prefix"); + break; + case 1: + path = new PathFragment("path_prefix/conflict"); + break; + default: + path = inputTreeFileArtifact.getParentRelativePath(); + } + + ++i; + return path; + } + }; + SpawnActionTemplate spawnActionTemplate = + new SpawnActionTemplate.Builder(inputTreeArtifact, outputTreeArtifact) + .setExecutable(new PathFragment("/bin/cp")) + .setCommandLineTemplate(CustomCommandLine.builder().build()) + .setOutputPathMapper(mapper) + .build(ActionsTestUtil.NULL_ACTION_OWNER); + + try { + evaluate(spawnActionTemplate); + fail("Expected ArtifactPrefixConflictException"); + } catch (ArtifactPrefixConflictException e) { + // Expected ArtifactPrefixConflictException + } + } + + private List<Action> evaluate(SpawnActionTemplate spawnActionTemplate) throws Exception { + SkyKey skyKey = ActionTemplateExpansionValue.key(spawnActionTemplate); + EvaluationResult<ActionTemplateExpansionValue> result = driver.evaluate( + ImmutableList.of(skyKey), + false, + SkyframeExecutor.DEFAULT_THREAD_COUNT, + NullEventHandler.INSTANCE); + if (result.hasError()) { + throw result.getError().getException(); + } + return ImmutableList.copyOf(result.get(skyKey).getExpandedActions()); + } + + private Artifact createTreeArtifact(String path) { + PathFragment execPath = new PathFragment("out").getRelative(path); + Path fullPath = rootDirectory.getRelative(execPath); + return new SpecialArtifact( + fullPath, + Root.asDerivedRoot(rootDirectory, rootDirectory.getRelative("out")), + execPath, + ArtifactOwner.NULL_OWNER, + SpecialArtifactType.TREE); + } + + private Artifact createAndPopulateTreeArtifact(String path, String... childRelativePaths) + throws Exception { + Artifact treeArtifact = createTreeArtifact(path); + Map<TreeFileArtifact, FileArtifactValue> treeFileArtifactMap = new LinkedHashMap<>(); + + for (String childRelativePath : childRelativePaths) { + TreeFileArtifact treeFileArtifact = ActionInputHelper.treeFileArtifact( + treeArtifact, new PathFragment(childRelativePath)); + scratch.file(treeFileArtifact.getPath().toString(), childRelativePath); + // We do not care about the FileArtifactValues in this test. + treeFileArtifactMap.put(treeFileArtifact, FileArtifactValue.create(treeFileArtifact)); + } + + artifactValueMap.put( + treeArtifact, TreeArtifactValue.create(ImmutableMap.copyOf(treeFileArtifactMap))); + + return treeArtifact; + } + + /** Dummy ArtifactFunction that just returns injected values */ + private static class DummyArtifactFunction implements SkyFunction { + private final Map<Artifact, ArtifactValue> artifactValueMap; + + DummyArtifactFunction(Map<Artifact, ArtifactValue> artifactValueMap) { + this.artifactValueMap = artifactValueMap; + } + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + OwnedArtifact ownedArtifact = (OwnedArtifact) skyKey.argument(); + Artifact artifact = ownedArtifact.getArtifact(); + return Preconditions.checkNotNull(artifactValueMap.get(artifact)); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java index d7433ac95c..789e2b1e1d 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTest.java @@ -30,9 +30,14 @@ import com.google.common.testing.EqualsTester; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata.MiddlemanType; +import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; +import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; import com.google.devtools.build.lib.actions.MissingInputFileException; import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; import com.google.devtools.build.lib.actions.util.TestAction.DummyAction; import com.google.devtools.build.lib.events.NullEventHandler; import com.google.devtools.build.lib.util.Pair; @@ -169,7 +174,7 @@ public class ArtifactFunctionTest extends ArtifactFunctionTestCase { new DummyAction( ImmutableList.of(input1, input2), output, MiddlemanType.AGGREGATING_MIDDLEMAN); // Overwrite default generating action with this one. - for (Iterator<Action> it = actions.iterator(); it.hasNext(); ) { + for (Iterator<ActionAnalysisMetadata> it = actions.iterator(); it.hasNext(); ) { if (it.next().getOutputs().contains(output)) { it.remove(); break; @@ -324,6 +329,70 @@ public class ArtifactFunctionTest extends ArtifactFunctionTestCase { .testEquals(); } + @Test + public void testActionTreeArtifactOutput() throws Throwable { + Artifact artifact = createDerivedTreeArtifactWithAction("treeArtifact"); + TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact, "child1", "hello1"); + TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact, "child2", "hello2"); + + TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact); + assertNotNull(value.getChildValue(treeFileArtifact1)); + assertNotNull(value.getChildValue(treeFileArtifact2)); + assertNotNull(value.getChildValue(treeFileArtifact1).getDigest()); + assertNotNull(value.getChildValue(treeFileArtifact2).getDigest()); + } + + @Test + public void testSpawnActionTemplate() throws Throwable { + // artifact1 is a tree artifact generated by normal action. + Artifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1"); + createFakeTreeFileArtifact(artifact1, "child1", "hello1"); + createFakeTreeFileArtifact(artifact1, "child2", "hello2"); + + + // artifact2 is a tree artifact generated by action template. + Artifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2"); + TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact2, "child1", "hello1"); + TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact2, "child2", "hello2"); + + actions.add( + ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2)); + + TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact2); + assertNotNull(value.getChildValue(treeFileArtifact1)); + assertNotNull(value.getChildValue(treeFileArtifact2)); + assertNotNull(value.getChildValue(treeFileArtifact1).getDigest()); + assertNotNull(value.getChildValue(treeFileArtifact2).getDigest()); + } + + @Test + public void testConsecutiveSpawnActionTemplates() throws Throwable { + // artifact1 is a tree artifact generated by normal action. + Artifact artifact1 = createDerivedTreeArtifactWithAction("treeArtifact1"); + createFakeTreeFileArtifact(artifact1, "child1", "hello1"); + createFakeTreeFileArtifact(artifact1, "child2", "hello2"); + + // artifact2 is a tree artifact generated by action template. + Artifact artifact2 = createDerivedTreeArtifactOnly("treeArtifact2"); + createFakeTreeFileArtifact(artifact2, "child1", "hello1"); + createFakeTreeFileArtifact(artifact2, "child2", "hello2"); + actions.add( + ActionsTestUtil.createDummySpawnActionTemplate(artifact1, artifact2)); + + // artifact3 is a tree artifact generated by action template. + Artifact artifact3 = createDerivedTreeArtifactOnly("treeArtifact3"); + TreeFileArtifact treeFileArtifact1 = createFakeTreeFileArtifact(artifact3, "child1", "hello1"); + TreeFileArtifact treeFileArtifact2 = createFakeTreeFileArtifact(artifact3, "child2", "hello2"); + actions.add( + ActionsTestUtil.createDummySpawnActionTemplate(artifact2, artifact3)); + + TreeArtifactValue value = (TreeArtifactValue) evaluateArtifactValue(artifact3); + assertNotNull(value.getChildValue(treeFileArtifact1)); + assertNotNull(value.getChildValue(treeFileArtifact2)); + assertNotNull(value.getChildValue(treeFileArtifact1).getDigest()); + assertNotNull(value.getChildValue(treeFileArtifact2).getDigest()); + } + private void file(Path path, String contents) throws Exception { FileSystemUtils.createDirectoryAndParents(path.getParentDirectory()); writeFile(path, contents); @@ -343,6 +412,33 @@ public class ArtifactFunctionTest extends ArtifactFunctionTestCase { return output; } + private Artifact createDerivedTreeArtifactWithAction(String path) { + Artifact treeArtifact = createDerivedTreeArtifactOnly(path); + actions.add(new DummyAction(ImmutableList.<Artifact>of(), treeArtifact)); + return treeArtifact; + } + + private Artifact createDerivedTreeArtifactOnly(String path) { + PathFragment execPath = new PathFragment("out").getRelative(path); + Path fullPath = root.getRelative(execPath); + return new SpecialArtifact( + fullPath, + Root.asDerivedRoot(root, root.getRelative("out")), + execPath, + ALL_OWNER, + SpecialArtifactType.TREE); + } + + private TreeFileArtifact createFakeTreeFileArtifact(Artifact treeArtifact, + String parentRelativePath, String content) throws Exception { + TreeFileArtifact treeFileArtifact = ActionInputHelper.treeFileArtifact( + treeArtifact, new PathFragment(parentRelativePath)); + Path path = treeFileArtifact.getPath(); + FileSystemUtils.createDirectoryAndParents(path.getParentDirectory()); + writeFile(path, content); + return treeFileArtifact; + } + private void assertValueMatches(FileStatus file, byte[] digest, FileArtifactValue value) throws IOException { assertEquals(file.getSize(), value.getSize()); @@ -384,35 +480,46 @@ public class ArtifactFunctionTest extends ArtifactFunctionTestCase { throws InterruptedException { setGeneratingActions(); return driver.evaluate( - Arrays.asList(keys), /*keepGoing=*/ - false, + Arrays.asList(keys), + /*keepGoing=*/false, SkyframeExecutor.DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE); } /** Value Builder for actions that just stats and stores the output file (which must exist). */ - private class SimpleActionExecutionFunction implements SkyFunction { + private static class SimpleActionExecutionFunction implements SkyFunction { @Override public SkyValue compute(SkyKey skyKey, Environment env) { Map<Artifact, FileValue> artifactData = new HashMap<>(); + Map<Artifact, TreeArtifactValue> treeArtifactData = new HashMap<>(); + Map<Artifact, FileArtifactValue> additionalOutputData = new HashMap<>(); Action action = (Action) skyKey.argument(); Artifact output = Iterables.getOnlyElement(action.getOutputs()); - FileArtifactValue value; - if (action.getActionType() == MiddlemanType.NORMAL) { - try { + + try { + if (output.isTreeArtifact()) { + TreeFileArtifact treeFileArtifact1 = ActionInputHelper.treeFileArtifact( + output, new PathFragment("child1")); + TreeFileArtifact treeFileArtifact2 = ActionInputHelper.treeFileArtifact( + output, new PathFragment("child2")); + TreeArtifactValue treeArtifactValue = TreeArtifactValue.create(ImmutableMap.of( + treeFileArtifact1, FileArtifactValue.create(treeFileArtifact1), + treeFileArtifact2, FileArtifactValue.create(treeFileArtifact2))); + treeArtifactData.put(output, treeArtifactValue); + } else if (action.getActionType() == MiddlemanType.NORMAL) { FileValue fileValue = ActionMetadataHandler.fileValueFromArtifact(output, null, null); artifactData.put(output, fileValue); - value = FileArtifactValue.create(output, fileValue); - } catch (IOException e) { - throw new IllegalStateException(e); + additionalOutputData.put(output, FileArtifactValue.create(output, fileValue)); + } else { + additionalOutputData.put(output, FileArtifactValue.DEFAULT_MIDDLEMAN); } - } else { - value = FileArtifactValue.DEFAULT_MIDDLEMAN; + } catch (IOException e) { + throw new IllegalStateException(e); } return new ActionExecutionValue( artifactData, - ImmutableMap.<Artifact, TreeArtifactValue>of(), - ImmutableMap.of(output, value)); + treeArtifactData, + additionalOutputData); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java index aa3fba5875..d964e2ddd5 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java @@ -17,7 +17,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; @@ -54,7 +54,7 @@ abstract class ArtifactFunctionTestCase { protected Predicate<PathFragment> allowedMissingInputsPredicate = Predicates.alwaysFalse(); - protected Set<Action> actions; + protected Set<ActionAnalysisMetadata> actions; protected boolean fastDigest = false; protected RecordingDifferencer differencer = new RecordingDifferencer(); protected SequentialBuildDriver driver; @@ -101,6 +101,8 @@ abstract class ArtifactFunctionTestCase { new PackageFactory(TestRuleClassProvider.getRuleClassProvider()), directories)) .put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction()) + .put(SkyFunctions.ACTION_TEMPLATE_EXPANSION, + new ActionTemplateExpansionFunction()) .build(), differencer); driver = new SequentialBuildDriver(evaluator); diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java index 9f6bf6c544..9c51cb1d5a 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/TimestampBuilderTestCase.java @@ -65,6 +65,7 @@ import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; import com.google.devtools.build.skyframe.RecordingDifferencer; import com.google.devtools.build.skyframe.SequentialBuildDriver; import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; @@ -99,7 +100,7 @@ public abstract class TimestampBuilderTestCase extends FoundationTestCase { protected Clock clock = BlazeClock.instance(); protected TimestampGranularityMonitor tsgm; protected RecordingDifferencer differencer = new RecordingDifferencer(); - private Set<Action> actions; + private Set<ActionAnalysisMetadata> actions; protected AtomicReference<EventBus> eventBusRef = new AtomicReference<>(); @@ -109,13 +110,14 @@ public abstract class TimestampBuilderTestCase extends FoundationTestCase { tsgm = new TimestampGranularityMonitor(clock); ResourceManager.instance().setAvailableResources(ResourceSet.createWithRamCpuIo(100, 1, 1)); actions = new HashSet<>(); + actionTemplateExpansionFunction = new ActionTemplateExpansionFunction(); } protected void clearActions() { actions.clear(); } - protected <T extends Action> T registerAction(T action) { + protected <T extends ActionAnalysisMetadata> T registerAction(T action) { actions.add(action); return action; } @@ -182,6 +184,8 @@ public abstract class TimestampBuilderTestCase extends FoundationTestCase { new PackageFactory(TestRuleClassProvider.getRuleClassProvider()), directories)) .put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction()) + .put(SkyFunctions.ACTION_TEMPLATE_EXPANSION, + new DelegatingActionTemplateExpansionFunction()) .build(), differencer, evaluationProgressReceiver); @@ -194,7 +198,7 @@ public abstract class TimestampBuilderTestCase extends FoundationTestCase { if (evaluator.getExistingValueForTesting(OWNER_KEY) == null) { differencer.inject(ImmutableMap.of( OWNER_KEY, - new ActionLookupValue(ImmutableList.<ActionAnalysisMetadata>copyOf(actions)))); + new ActionLookupValue(ImmutableList.copyOf(actions)))); } } @@ -248,6 +252,8 @@ public abstract class TimestampBuilderTestCase extends FoundationTestCase { /** A non-persistent cache. */ protected InMemoryActionCache inMemoryCache; + protected SkyFunction actionTemplateExpansionFunction; + /** A class that records an event. */ protected static class Button implements Runnable { protected boolean pressed = false; @@ -401,4 +407,17 @@ public abstract class TimestampBuilderTestCase extends FoundationTestCase { throw new UnsupportedOperationException(); } } + + private class DelegatingActionTemplateExpansionFunction implements SkyFunction { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + return actionTemplateExpansionFunction.compute(skyKey, env); + } + + @Override + public String extractTag(SkyKey skyKey) { + return actionTemplateExpansionFunction.extractTag(skyKey); + } + } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactBuildTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactBuildTest.java index 7304462046..66cacfd938 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactBuildTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/TreeArtifactBuildTest.java @@ -24,9 +24,12 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; import com.google.common.hash.Hashing; import com.google.common.util.concurrent.Runnables; +import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionExecutionException; import com.google.devtools.build.lib.actions.ActionInput; @@ -35,10 +38,15 @@ import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; import com.google.devtools.build.lib.actions.Artifact.SpecialArtifactType; import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; +import com.google.devtools.build.lib.actions.BuildFailedException; import com.google.devtools.build.lib.actions.Root; import com.google.devtools.build.lib.actions.cache.InjectedStat; import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; import com.google.devtools.build.lib.actions.util.TestAction; +import com.google.devtools.build.lib.actions.util.TestAction.DummyAction; +import com.google.devtools.build.lib.analysis.actions.SpawnActionTemplate; +import com.google.devtools.build.lib.skyframe.ActionTemplateExpansionValue.ActionTemplateExpansionKey; import com.google.devtools.build.lib.testutil.TestUtils; import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileSystem; @@ -46,6 +54,9 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Symlinks; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; import org.junit.Before; import org.junit.Test; @@ -481,6 +492,199 @@ public class TreeArtifactBuildTest extends TimestampBuilderTestCase { buildArtifact(action.getSoleOutput()); } + @Test + public void testExpandedActionsBuildInActionTemplate() throws Throwable { + // artifact1 is a tree artifact generated by a TouchingTestAction. + Artifact artifact1 = createTreeArtifact("treeArtifact1"); + TreeFileArtifact treeFileArtifactA = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child1")); + TreeFileArtifact treeFileArtifactB = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child2")); + registerAction(new TouchingTestAction(treeFileArtifactA, treeFileArtifactB)); + + // artifact2 is a tree artifact generated by an action template. + Artifact artifact2 = createTreeArtifact("treeArtifact2"); + SpawnActionTemplate actionTemplate = ActionsTestUtil.createDummySpawnActionTemplate( + artifact1, artifact2); + registerAction(actionTemplate); + + // We mock out the action template function to expand into two actions that just touch the + // output files. + TreeFileArtifact expectedOutputTreeFileArtifact1 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child1")); + TreeFileArtifact expectedOutputTreeFileArtifact2 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child2")); + Action generateOutputAction = new DummyAction( + ImmutableList.<Artifact>of(treeFileArtifactA), expectedOutputTreeFileArtifact1); + Action noGenerateOutputAction = new DummyAction( + ImmutableList.<Artifact>of(treeFileArtifactB), expectedOutputTreeFileArtifact2); + + actionTemplateExpansionFunction = new DummyActionTemplateExpansionFunction( + ImmutableMultimap.of( + actionTemplate, generateOutputAction, + actionTemplate, noGenerateOutputAction)); + + buildArtifact(artifact2); + } + + @Test + public void testExpandedActionDoesNotGenerateOutputInActionTemplate() throws Throwable { + // expect errors + reporter.removeHandler(failFastHandler); + + // artifact1 is a tree artifact generated by a TouchingTestAction. + Artifact artifact1 = createTreeArtifact("treeArtifact1"); + TreeFileArtifact treeFileArtifactA = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child1")); + TreeFileArtifact treeFileArtifactB = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child2")); + registerAction(new TouchingTestAction(treeFileArtifactA, treeFileArtifactB)); + + // artifact2 is a tree artifact generated by an action template. + Artifact artifact2 = createTreeArtifact("treeArtifact2"); + SpawnActionTemplate actionTemplate = ActionsTestUtil.createDummySpawnActionTemplate( + artifact1, artifact2); + registerAction(actionTemplate); + + // We mock out the action template function to expand into two actions: + // One Action that touches the output file. + // The other action that does not generate the output file. + TreeFileArtifact expectedOutputTreeFileArtifact1 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child1")); + TreeFileArtifact expectedOutputTreeFileArtifact2 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child2")); + Action generateOutputAction = new DummyAction( + ImmutableList.<Artifact>of(treeFileArtifactA), expectedOutputTreeFileArtifact1); + Action noGenerateOutputAction = new NoOpDummyAction( + ImmutableList.<Artifact>of(treeFileArtifactB), + ImmutableList.<Artifact>of(expectedOutputTreeFileArtifact2)); + + actionTemplateExpansionFunction = new DummyActionTemplateExpansionFunction( + ImmutableMultimap.of( + actionTemplate, generateOutputAction, + actionTemplate, noGenerateOutputAction)); + + try { + buildArtifact(artifact2); + fail("Expected BuildFailedException"); + } catch (BuildFailedException e) { + assertThat(e.getMessage()).contains("not all outputs were created"); + } + } + + @Test + public void testOneExpandedActionThrowsInActionTemplate() throws Throwable { + // expect errors + reporter.removeHandler(failFastHandler); + + // artifact1 is a tree artifact generated by a TouchingTestAction. + Artifact artifact1 = createTreeArtifact("treeArtifact1"); + TreeFileArtifact treeFileArtifactA = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child1")); + TreeFileArtifact treeFileArtifactB = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child2")); + registerAction(new TouchingTestAction(treeFileArtifactA, treeFileArtifactB)); + + // artifact2 is a tree artifact generated by an action template. + Artifact artifact2 = createTreeArtifact("treeArtifact2"); + SpawnActionTemplate actionTemplate = ActionsTestUtil.createDummySpawnActionTemplate( + artifact1, artifact2); + registerAction(actionTemplate); + + // We mock out the action template function to expand into two actions: + // One Action that touches the output file. + // The other action that just throws when executed. + TreeFileArtifact expectedOutputTreeFileArtifact1 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child1")); + TreeFileArtifact expectedOutputTreeFileArtifact2 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child2")); + Action generateOutputAction = new DummyAction( + ImmutableList.<Artifact>of(treeFileArtifactA), expectedOutputTreeFileArtifact1); + Action throwingAction = new ThrowingDummyAction( + ImmutableList.<Artifact>of(treeFileArtifactB), + ImmutableList.<Artifact>of(expectedOutputTreeFileArtifact2)); + + actionTemplateExpansionFunction = new DummyActionTemplateExpansionFunction( + ImmutableMultimap.of( + actionTemplate, generateOutputAction, + actionTemplate, throwingAction)); + + try { + buildArtifact(artifact2); + fail("Expected BuildFailedException"); + } catch (BuildFailedException e) { + assertThat(e.getMessage()).contains("Throwing dummy action"); + } + } + + @Test + public void testAllExpandedActionsThrowInActionTemplate() throws Throwable { + // expect errors + reporter.removeHandler(failFastHandler); + + // artifact1 is a tree artifact generated by a TouchingTestAction. + Artifact artifact1 = createTreeArtifact("treeArtifact1"); + TreeFileArtifact treeFileArtifactA = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child1")); + TreeFileArtifact treeFileArtifactB = ActionInputHelper.treeFileArtifact( + artifact1, new PathFragment("child2")); + registerAction(new TouchingTestAction(treeFileArtifactA, treeFileArtifactB)); + + // artifact2 is a tree artifact generated by an action template. + Artifact artifact2 = createTreeArtifact("treeArtifact2"); + SpawnActionTemplate actionTemplate = ActionsTestUtil.createDummySpawnActionTemplate( + artifact1, artifact2); + registerAction(actionTemplate); + + // We mock out the action template function to expand into two actions that throw when executed. + TreeFileArtifact expectedOutputTreeFileArtifact1 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child1")); + TreeFileArtifact expectedOutputTreeFileArtifact2 = ActionInputHelper.treeFileArtifact( + artifact2, new PathFragment("child2")); + Action throwingAction = new ThrowingDummyAction( + ImmutableList.<Artifact>of(treeFileArtifactA), + ImmutableList.<Artifact>of(expectedOutputTreeFileArtifact1)); + Action anotherThrowingAction = new ThrowingDummyAction( + ImmutableList.<Artifact>of(treeFileArtifactB), + ImmutableList.<Artifact>of(expectedOutputTreeFileArtifact2)); + + actionTemplateExpansionFunction = new DummyActionTemplateExpansionFunction( + ImmutableMultimap.of( + actionTemplate, throwingAction, + actionTemplate, anotherThrowingAction)); + + try { + buildArtifact(artifact2); + fail("Expected BuildFailedException"); + } catch (BuildFailedException e) { + assertThat(e.getMessage()).contains("Throwing dummy action"); + } + } + + @Test + public void testInputTreeArtifactCreationFailedInActionTemplate() throws Throwable { + // expect errors + reporter.removeHandler(failFastHandler); + + // artifact1 is created by a action that throws. + Artifact artifact1 = createTreeArtifact("treeArtifact1"); + registerAction( + new ThrowingDummyAction(ImmutableList.<Artifact>of(), ImmutableList.of(artifact1))); + + // artifact2 is a tree artifact generated by an action template. + Artifact artifact2 = createTreeArtifact("treeArtifact2"); + SpawnActionTemplate actionTemplate = ActionsTestUtil.createDummySpawnActionTemplate( + artifact1, artifact2); + registerAction(actionTemplate); + + try { + buildArtifact(artifact2); + fail("Expected BuildFailedException"); + } catch (BuildFailedException e) { + assertThat(e.getMessage()).contains("Throwing dummy action"); + } + } + /** * A generic test action that takes at most one input TreeArtifact, * exactly one output TreeArtifact, and some path fragment inputs/outputs. @@ -766,4 +970,53 @@ public class TreeArtifactBuildTest extends TimestampBuilderTestCase { path.delete(); } } + + /** A dummy action template expansion function that just returns the injected actions */ + private static class DummyActionTemplateExpansionFunction implements SkyFunction { + private final Multimap<SpawnActionTemplate, Action> actionTemplateToActionMap; + + DummyActionTemplateExpansionFunction( + Multimap<SpawnActionTemplate, Action> actionTemplateToActionMap) { + this.actionTemplateToActionMap = actionTemplateToActionMap; + } + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + ActionTemplateExpansionKey key = (ActionTemplateExpansionKey) skyKey.argument(); + SpawnActionTemplate actionTemplate = key.getActionTemplate(); + return new ActionTemplateExpansionValue( + Preconditions.checkNotNull(actionTemplateToActionMap.get(actionTemplate))); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + } + + /** No-op action that does not generate the action outputs. */ + private static class NoOpDummyAction extends TestAction { + public NoOpDummyAction(Collection<Artifact> inputs, Collection<Artifact> outputs) { + super(NO_EFFECT, inputs, outputs); + } + + /** Do nothing */ + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException {} + } + + /** No-op action that throws when executed */ + private static class ThrowingDummyAction extends TestAction { + public ThrowingDummyAction(Collection<Artifact> inputs, Collection<Artifact> outputs) { + super(NO_EFFECT, inputs, outputs); + } + + /** Throws */ + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + throw new ActionExecutionException("Throwing dummy action", this, true); + } + } } |