diff options
Diffstat (limited to 'src/test/java/com/google/devtools/build')
150 files changed, 30238 insertions, 0 deletions
diff --git a/src/test/java/com/google/devtools/build/lib/AllTests.java b/src/test/java/com/google/devtools/build/lib/AllTests.java new file mode 100644 index 0000000000..6aa19f1a57 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/AllTests.java @@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib; + +import com.google.devtools.build.lib.testutil.ClasspathSuite; + +import org.junit.runner.RunWith; + +/** + * Test suite for options parsing framework. + */ +@RunWith(ClasspathSuite.class) +public class AllTests { +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporterTest.java b/src/test/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporterTest.java new file mode 100644 index 0000000000..8af46e4aea --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ActionExecutionStatusReporterTest.java @@ -0,0 +1,293 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.events.EventCollector; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.Clock; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Test for the {@link ActionExecutionStatusReporter} class. + */ +@RunWith(JUnit4.class) +public class ActionExecutionStatusReporterTest { + private static final class MockClock implements Clock { + private long millis = 0; + + public void advance() { + advanceBy(1000); + } + + public void advanceBy(long millis) { + Preconditions.checkArgument(millis > 0); + this.millis += millis; + } + + @Override + public long currentTimeMillis() { + return millis; + } + + @Override + public long nanoTime() { + // There's no reason to use a nanosecond-precision for a mock clock. + return millis * 1000000L; + } + } + + private EventCollector collector; + private ActionExecutionStatusReporter statusReporter; + private EventBus eventBus; + private MockClock clock = new MockClock(); + + private Action mockAction(String progressMessage) { return mockAction(progressMessage, false); } + + private Action mockAction(String progressMessage, boolean remote) { + Action action = Mockito.mock(Action.class); + when(action.describeStrategy(null)).thenReturn(remote ? "remote" : "something else"); + when(action.getProgressMessage()).thenReturn(progressMessage); + if (progressMessage == null) { + when(action.prettyPrint()).thenReturn("default message"); + } + return action; + } + + @Before + public void setUp() throws Exception { + collector = new EventCollector(EventKind.ALL_EVENTS); + Reporter reporter = new Reporter(); + reporter.addHandler(collector); + statusReporter = ActionExecutionStatusReporter.create(reporter, clock); + eventBus = new EventBus(); + eventBus.register(statusReporter); + } + + private void verifyNoOutput() { + collector.clear(); + statusReporter.showCurrentlyExecutingActions(""); + assertEquals(0, collector.count()); + } + + private void verifyOutput(String... lines) throws Exception { + collector.clear(); + statusReporter.showCurrentlyExecutingActions(""); + assertThat(Splitter.on("\n").omitEmptyStrings().trimResults().split( + Iterables.getOnlyElement(collector).getMessage().replaceAll(" +", " "))) + .containsExactlyElementsIn(Arrays.asList(lines)).inOrder(); + } + + private void verifyWarningOutput(String... lines) throws Exception { + collector.clear(); + statusReporter.warnAboutCurrentlyExecutingActions(); + assertThat(Splitter.on("\n").omitEmptyStrings().trimResults().split( + Iterables.getOnlyElement(collector).getMessage().replaceAll(" +", " "))) + .containsExactlyElementsIn(Arrays.asList(lines)).inOrder(); + } + + @Test + public void testCategories() throws Exception { + verifyNoOutput(); + verifyWarningOutput("There are no active jobs - stopping the build"); + setPreparing(mockAction("action1")); + clock.advance(); + verifyWarningOutput("Still waiting for unfinished jobs"); + setScheduling(mockAction("action2")); + clock.advance(); + setRunning(mockAction("action3", true)); + clock.advance(); + setRunning(mockAction("action4", false)); + verifyOutput("Still waiting for 4 jobs to complete:", + "Preparing:", "action1, 3 s", + "Running (remote):", "action3, 1 s", + "Running (something else):", "action4, 0 s", + "Scheduling:", "action2, 2 s"); + verifyWarningOutput("Still waiting for 3 jobs to complete:", + "Running (remote):", "action3, 1 s", + "Running (something else):", "action4, 0 s", + "Scheduling:", "action2, 2 s", + "Build will be stopped after these tasks terminate"); + } + + @Test + public void testSingleAction() throws Exception { + Action action = mockAction("action1", true); + verifyNoOutput(); + setPreparing(action); + clock.advanceBy(1200); + verifyOutput("Still waiting for 1 job to complete:", "Preparing:", "action1, 1 s"); + clock.advanceBy(5000); + + setScheduling(action); + clock.advanceBy(1200); + // Only started *scheduling* 1200 ms ago, not 6200 ms ago. + verifyOutput("Still waiting for 1 job to complete:", "Scheduling:", "action1, 1 s"); + setRunning(action); + clock.advanceBy(3000); + // Only started *running* 3000 ms ago, not 4200 ms ago. + verifyOutput("Still waiting for 1 job to complete:", "Running (remote):", "action1, 3 s"); + statusReporter.remove(action); + verifyNoOutput(); + } + + @Test + public void testDynamicUpdate() throws Exception { + Action action = mockAction("action1", true); + verifyNoOutput(); + setPreparing(action); + clock.advance(); + verifyOutput("Still waiting for 1 job to complete:", "Preparing:", "action1, 1 s"); + setScheduling(action); + clock.advance(); + verifyOutput("Still waiting for 1 job to complete:", "Scheduling:", "action1, 1 s"); + setRunning(action); + clock.advance(); + verifyOutput("Still waiting for 1 job to complete:", "Running (remote):", "action1, 1 s"); + clock.advance(); + + eventBus.post(ActionStatusMessage.analysisStrategy(action)); + // Locality strategy was changed, so timer was reset to 0 s. + verifyOutput("Still waiting for 1 job to complete:", "Analyzing:", "action1, 0 s"); + statusReporter.remove(action); + verifyNoOutput(); + } + + @Test + public void testGroups() throws Exception { + verifyNoOutput(); + List<Action> actions = ImmutableList.of( + mockAction("remote1", true), mockAction("remote2", true), mockAction("remote3", true), + mockAction("local1", false), mockAction("local2", false), mockAction("local3", false)); + + for (Action a : actions) { + setScheduling(a); + clock.advance(); + } + + verifyOutput("Still waiting for 6 jobs to complete:", + "Scheduling:", + "remote1, 6 s", "remote2, 5 s", "remote3, 4 s", + "local1, 3 s", "local2, 2 s", "local3, 1 s"); + + for (Action a : actions) { + setRunning(a); + clock.advanceBy(2000); + } + + // Timers got reset because now they are no longer scheduling but running. + verifyOutput("Still waiting for 6 jobs to complete:", + "Running (remote):", "remote1, 12 s", "remote2, 10 s", "remote3, 8 s", + "Running (something else):", "local1, 6 s", "local2, 4 s", "local3, 2 s"); + + statusReporter.remove(actions.get(0)); + verifyOutput("Still waiting for 5 jobs to complete:", + "Running (remote):", "remote2, 10 s", "remote3, 8 s", + "Running (something else):", "local1, 6 s", "local2, 4 s", "local3, 2 s"); + } + + @Test + public void testTruncation() throws Exception { + verifyNoOutput(); + List<Action> actions = new ArrayList<>(); + for (int i = 1; i <= 100; i++) { + Action a = mockAction("a" + i); + actions.add(a); + setScheduling(a); + clock.advance(); + } + verifyOutput("Still waiting for 100 jobs to complete:", "Scheduling:", + "a1, 100 s", "a2, 99 s", "a3, 98 s", "a4, 97 s", "a5, 96 s", + "a6, 95 s", "a7, 94 s", "a8, 93 s", "a9, 92 s", "... 91 more jobs"); + + for (int i = 0; i < 5; i++) { + setRunning(actions.get(i)); + clock.advance(); + } + verifyOutput("Still waiting for 100 jobs to complete:", + "Running (something else):", "a1, 5 s", "a2, 4 s", "a3, 3 s", "a4, 2 s", "a5, 1 s", + "Scheduling:", "a6, 100 s", "a7, 99 s", "a8, 98 s", "a9, 97 s", "a10, 96 s", + "a11, 95 s", "a12, 94 s", "a13, 93 s", "a14, 92 s", "... 86 more jobs"); + } + + @Test + public void testOrdering() throws Exception { + verifyNoOutput(); + setScheduling(mockAction("a1")); + clock.advance(); + setPreparing(mockAction("b1")); + clock.advance(); + setPreparing(mockAction("b2")); + clock.advance(); + setScheduling(mockAction("a2")); + clock.advance(); + verifyOutput("Still waiting for 4 jobs to complete:", + "Preparing:", "b1, 3 s", "b2, 2 s", + "Scheduling:", "a1, 4 s", "a2, 1 s"); + } + + @Test + public void testNoProgressMessage() throws Exception { + verifyNoOutput(); + setScheduling(mockAction(null)); + verifyOutput("Still waiting for 1 job to complete:", "Scheduling:", "default message, 0 s"); + } + + @Test + public void testWaitTimeCalculation() throws Exception { + // --progress_report_interval=0 + assertEquals(10, ActionExecutionStatusReporter.getWaitTime(0, 0)); + assertEquals(30, ActionExecutionStatusReporter.getWaitTime(0, 10)); + assertEquals(60, ActionExecutionStatusReporter.getWaitTime(0, 30)); + assertEquals(60, ActionExecutionStatusReporter.getWaitTime(0, 60)); + + // --progress_report_interval=42 + assertEquals(42, ActionExecutionStatusReporter.getWaitTime(42, 0)); + assertEquals(42, ActionExecutionStatusReporter.getWaitTime(42, 42)); + + // --progress_report_interval=30 (looks like one of the default timeout stages) + assertEquals(30, ActionExecutionStatusReporter.getWaitTime(30, 0)); + assertEquals(30, ActionExecutionStatusReporter.getWaitTime(30, 30)); + } + + private void setScheduling(ActionMetadata action) { + eventBus.post(ActionStatusMessage.schedulingStrategy(action)); + } + + private void setPreparing(ActionMetadata action) { + eventBus.post(ActionStatusMessage.preparingStrategy(action)); + } + + private void setRunning(ActionMetadata action) { + eventBus.post(ActionStatusMessage.runningStrategy(action)); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java b/src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java new file mode 100644 index 0000000000..71cf9c4b5d --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ArtifactFactoryTest.java @@ -0,0 +1,246 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER; +import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ARTIFACT_OWNER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Tests {@link ArtifactFactory}. Also see {@link ArtifactTest} for a test + * of individual artifacts. + */ +@RunWith(JUnit4.class) +public class ArtifactFactoryTest { + + private FsApparatus scratch = FsApparatus.newInMemory(); + + private Path execRoot; + private Root clientRoot; + private Root clientRoRoot; + private Root outRoot; + + private PathFragment fooPath; + private PackageIdentifier fooPackage; + private PathFragment fooRelative; + + private PathFragment barPath; + private PackageIdentifier barPackage; + private PathFragment barRelative; + + private ArtifactFactory artifactFactory; + + @Before + public void setUp() throws Exception { + execRoot = scratch.dir("/output/workspace"); + clientRoot = Root.asSourceRoot(scratch.dir("/client/workspace")); + clientRoRoot = Root.asSourceRoot(scratch.dir("/client/RO/workspace")); + outRoot = Root.asDerivedRoot(execRoot, execRoot.getRelative("out-root/x/bin")); + + fooPath = new PathFragment("foo"); + fooPackage = PackageIdentifier.createInDefaultRepo(fooPath); + fooRelative = fooPath.getRelative("foosource.txt"); + + barPath = new PathFragment("foo/bar"); + barPackage = PackageIdentifier.createInDefaultRepo(barPath); + barRelative = barPath.getRelative("barsource.txt"); + + artifactFactory = new ArtifactFactory(execRoot); + setupRoots(); + } + + private void setupRoots() { + Map<PackageIdentifier, Root> packageRootMap = new HashMap<>(); + packageRootMap.put(fooPackage, clientRoot); + packageRootMap.put(barPackage, clientRoRoot); + artifactFactory.setPackageRoots(packageRootMap); + artifactFactory.setDerivedArtifactRoots(ImmutableList.of(outRoot)); + } + + @Test + public void testGetSourceArtifactYieldsSameArtifact() throws Exception { + assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot), + artifactFactory.getSourceArtifact(fooRelative, clientRoot)); + } + + @Test + public void testGetSourceArtifactUnnormalized() throws Exception { + assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot), + artifactFactory.getSourceArtifact(new PathFragment("foo/./foosource.txt"), + clientRoot)); + } + + @Test + public void testResolveArtifact_noDerived_simpleSource() throws Exception { + assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot), + artifactFactory.resolveSourceArtifact(fooRelative)); + assertSame(artifactFactory.getSourceArtifact(barRelative, clientRoRoot), + artifactFactory.resolveSourceArtifact(barRelative)); + } + + @Test + public void testResolveArtifact_noDerived_derivedRoot() throws Exception { + assertNull(artifactFactory.resolveSourceArtifact( + outRoot.getPath().getRelative(fooRelative).relativeTo(execRoot))); + assertNull(artifactFactory.resolveSourceArtifact( + outRoot.getPath().getRelative(barRelative).relativeTo(execRoot))); + } + + @Test + public void testResolveArtifact_noDerived_simpleSource_other() throws Exception { + Artifact actual = artifactFactory.resolveSourceArtifact(fooRelative); + assertSame(artifactFactory.getSourceArtifact(fooRelative, clientRoot), actual); + actual = artifactFactory.resolveSourceArtifact(barRelative); + assertSame(artifactFactory.getSourceArtifact(barRelative, clientRoRoot), actual); + } + + @Test + public void testClearResetsFactory() { + Artifact fooArtifact = artifactFactory.getSourceArtifact(fooRelative, clientRoot); + artifactFactory.clear(); + setupRoots(); + assertNotSame(fooArtifact, artifactFactory.getSourceArtifact(fooRelative, clientRoot)); + } + + @Test + public void testFindDerivedRoot() throws Exception { + assertSame(outRoot, + artifactFactory.findDerivedRoot(outRoot.getPath().getRelative(fooRelative))); + assertSame(outRoot, + artifactFactory.findDerivedRoot(outRoot.getPath().getRelative(barRelative))); + } + + @Test + public void testSetGeneratingActionIdempotenceNewActionGraph() throws Exception { + Artifact a = artifactFactory.getDerivedArtifact(fooRelative, outRoot, NULL_ARTIFACT_OWNER); + Artifact b = artifactFactory.getDerivedArtifact(barRelative, outRoot, NULL_ARTIFACT_OWNER); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Action originalAction = new ActionsTestUtil.NullAction(NULL_ACTION_OWNER, a); + actionGraph.registerAction(originalAction); + + // Creating a second Action referring to the Artifact should create a conflict. + try { + Action action = new ActionsTestUtil.NullAction(NULL_ACTION_OWNER, a, b); + actionGraph.registerAction(action); + fail(); + } catch (ActionConflictException e) { + assertSame(a, e.getArtifact()); + assertSame(originalAction, actionGraph.getGeneratingAction(a)); + } + } + + @Test + public void testGetDerivedArtifact() throws Exception { + PathFragment toolPath = new PathFragment("_bin/tool"); + Artifact artifact = artifactFactory.getDerivedArtifact(toolPath); + assertEquals(toolPath, artifact.getExecPath()); + assertEquals(Root.asDerivedRoot(execRoot), artifact.getRoot()); + assertEquals(execRoot.getRelative(toolPath), artifact.getPath()); + assertNull(artifact.getOwner()); + } + + @Test + public void testGetDerivedArtifactFailsForAbsolutePath() throws Exception { + try { + artifactFactory.getDerivedArtifact(new PathFragment("/_bin/b")); + fail(); + } catch (IllegalArgumentException e) { + // Expected exception + } + } + + private static class MockPackageRootResolver implements PackageRootResolver { + private Map<PathFragment, Root> packageRoots = Maps.newHashMap(); + + public void setPackageRoots(Map<PackageIdentifier, Root> packageRoots) { + for (Entry<PackageIdentifier, Root> packageRoot : packageRoots.entrySet()) { + this.packageRoots.put(packageRoot.getKey().getPackageFragment(), packageRoot.getValue()); + } + } + + @Override + public Map<PathFragment, Root> findPackageRoots(Iterable<PathFragment> execPaths) { + Map<PathFragment, Root> result = new HashMap<>(); + for (PathFragment execPath : execPaths) { + for (PathFragment dir = execPath.getParentDirectory(); dir != null; + dir = dir.getParentDirectory()) { + if (packageRoots.get(dir) != null) { + result.put(execPath, packageRoots.get(dir)); + } + } + if (result.get(execPath) == null) { + result.put(execPath, null); + } + } + return result; + } + } + + @Test + public void testArtifactDeserializationWithoutReusedArtifacts() throws Exception { + PathFragment derivedPath = outRoot.getExecPath().getRelative("fruit/banana"); + artifactFactory.clear(); + artifactFactory.setDerivedArtifactRoots(ImmutableList.of(outRoot)); + MockPackageRootResolver rootResolver = new MockPackageRootResolver(); + rootResolver.setPackageRoots( + ImmutableMap.of(PackageIdentifier.createInDefaultRepo(""), clientRoot)); + Artifact artifact1 = artifactFactory.deserializeArtifact(derivedPath, rootResolver); + Artifact artifact2 = artifactFactory.deserializeArtifact(derivedPath, rootResolver); + assertEquals(artifact1, artifact2); + assertNull(artifact1.getOwner()); + assertNull(artifact2.getOwner()); + assertEquals(derivedPath, artifact1.getExecPath()); + assertEquals(derivedPath, artifact2.getExecPath()); + + // Source artifacts are always reused + PathFragment sourcePath = clientRoot.getExecPath().getRelative("fruit/mango"); + artifact1 = artifactFactory.deserializeArtifact(sourcePath, rootResolver); + artifact2 = artifactFactory.deserializeArtifact(sourcePath, rootResolver); + assertSame(artifact1, artifact2); + assertEquals(sourcePath, artifact1.getExecPath()); + } + + @Test + public void testDeserializationWithInvalidPath() throws Exception { + artifactFactory.clear(); + PathFragment randomPath = new PathFragment("maracuja/lemon/kiwi"); + Artifact artifact = artifactFactory.deserializeArtifact(randomPath, + new MockPackageRootResolver()); + assertNull(artifact); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java b/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java new file mode 100644 index 0000000000..d493e42b70 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ArtifactTest.java @@ -0,0 +1,312 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.testutil.MoreAsserts.assertSameContents; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Action.MiddlemanType; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; +import com.google.devtools.build.lib.actions.util.LabelArtifactOwner; +import com.google.devtools.build.lib.rules.cpp.CppFileTypes; +import com.google.devtools.build.lib.rules.java.JavaSemantics; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@RunWith(JUnit4.class) +public class ArtifactTest { + private Scratch scratch; + private Path execDir; + private Root rootDir; + + @Before + public void setUp() throws Exception { + scratch = new Scratch(); + execDir = scratch.dir("/exec"); + rootDir = Root.asDerivedRoot(scratch.dir("/exec/root")); + } + + @Test + public void testConstruction_badRootDir() throws IOException { + Path f1 = scratch.file("/exec/dir/file.ext"); + Path bogusDir = scratch.file("/exec/dir/bogus"); + try { + new Artifact(f1, Root.asDerivedRoot(bogusDir), f1.relativeTo(execDir)); + fail("Expected IllegalArgumentException constructing artifact with a bad root dir"); + } catch (IllegalArgumentException expected) {} + } + + @Test + public void testEquivalenceRelation() throws Exception { + PathFragment aPath = new PathFragment("src/a"); + PathFragment bPath = new PathFragment("src/b"); + assertEquals(new Artifact(aPath, rootDir), + new Artifact(aPath, rootDir)); + assertEquals(new Artifact(bPath, rootDir), + new Artifact(bPath, rootDir)); + assertFalse(new Artifact(aPath, rootDir).equals( + new Artifact(bPath, rootDir))); + } + + @Test + public void testComparison() throws Exception { + PathFragment aPath = new PathFragment("src/a"); + PathFragment bPath = new PathFragment("src/b"); + Artifact aArtifact = new Artifact(aPath, rootDir); + Artifact bArtifact = new Artifact(bPath, rootDir); + assertEquals(-1, aArtifact.compareTo(bArtifact)); + assertEquals(0, aArtifact.compareTo(aArtifact)); + assertEquals(0, bArtifact.compareTo(bArtifact)); + assertEquals(1, bArtifact.compareTo(aArtifact)); + } + + @Test + public void testRootPrefixedExecPath_normal() throws IOException { + Path f1 = scratch.file("/exec/root/dir/file.ext"); + Artifact a1 = new Artifact(f1, rootDir, f1.relativeTo(execDir)); + assertEquals("root:dir/file.ext", Artifact.asRootPrefixedExecPath(a1)); + } + + @Test + public void testRootPrefixedExecPath_noRoot() throws IOException { + Path f1 = scratch.file("/exec/dir/file.ext"); + Artifact a1 = new Artifact(f1.relativeTo(execDir), Root.asDerivedRoot(execDir)); + assertEquals(":dir/file.ext", Artifact.asRootPrefixedExecPath(a1)); + } + + @Test + public void testRootPrefixedExecPath_nullRootDir() throws IOException { + Path f1 = scratch.file("/exec/dir/file.ext"); + try { + new Artifact(f1, null, f1.relativeTo(execDir)); + fail("Expected IllegalArgumentException creating artifact with null root"); + } catch (IllegalArgumentException expected) {} + } + + @Test + public void testRootPrefixedExecPaths() throws IOException { + Path f1 = scratch.file("/exec/root/dir/file1.ext"); + Path f2 = scratch.file("/exec/root/dir/dir/file2.ext"); + Path f3 = scratch.file("/exec/root/dir/dir/dir/file3.ext"); + Artifact a1 = new Artifact(f1, rootDir, f1.relativeTo(execDir)); + Artifact a2 = new Artifact(f2, rootDir, f2.relativeTo(execDir)); + Artifact a3 = new Artifact(f3, rootDir, f3.relativeTo(execDir)); + List<String> strings = new ArrayList<>(); + Artifact.addRootPrefixedExecPaths(Lists.newArrayList(a1, a2, a3), strings); + assertThat(strings).containsExactly( + "root:dir/file1.ext", + "root:dir/dir/file2.ext", + "root:dir/dir/dir/file3.ext").inOrder(); + } + + @Test + public void testGetFilename() throws Exception { + Root root = Root.asSourceRoot(scratch.dir("/foo")); + Artifact javaFile = new Artifact(scratch.file("/foo/Bar.java"), root); + Artifact generatedHeader = new Artifact(scratch.file("/foo/bar.proto.h"), root); + Artifact generatedCc = new Artifact(scratch.file("/foo/bar.proto.cc"), root); + Artifact aCPlusPlusFile = new Artifact(scratch.file("/foo/bar.cc"), root); + assertTrue(JavaSemantics.JAVA_SOURCE.matches(javaFile.getFilename())); + assertTrue(CppFileTypes.CPP_HEADER.matches(generatedHeader.getFilename())); + assertTrue(CppFileTypes.CPP_SOURCE.matches(generatedCc.getFilename())); + assertTrue(CppFileTypes.CPP_SOURCE.matches(aCPlusPlusFile.getFilename())); + } + + @Test + public void testMangledPath() { + String path = "dir/sub_dir/name:end"; + assertEquals("dir_Ssub_Udir_Sname_Cend", Actions.escapedPath(path)); + } + + private List<Artifact> getFooBarArtifacts(MutableActionGraph actionGraph, boolean collapsedList) + throws Exception { + Root root = Root.asSourceRoot(scratch.dir("/foo")); + Artifact aHeader1 = new Artifact(scratch.file("/foo/bar1.h"), root); + Artifact aHeader2 = new Artifact(scratch.file("/foo/bar2.h"), root); + Artifact aHeader3 = new Artifact(scratch.file("/foo/bar3.h"), root); + Artifact middleman = new Artifact(new PathFragment("middleman"), + Root.middlemanRoot(scratch.dir("/foo"), scratch.dir("/foo/out"))); + actionGraph.registerAction(new MiddlemanAction(ActionsTestUtil.NULL_ACTION_OWNER, + ImmutableList.of(aHeader1, aHeader2, aHeader3), middleman, "desc", + MiddlemanType.AGGREGATING_MIDDLEMAN)); + return collapsedList ? Lists.newArrayList(aHeader1, middleman) : + Lists.newArrayList(aHeader1, aHeader2, middleman); + } + + @Test + public void testAddExecPaths() throws Exception { + List<String> paths = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Artifact.addExecPaths(getFooBarArtifacts(actionGraph, false), paths); + assertSameContents(ImmutableList.of("bar1.h", "bar2.h"), paths); + } + + @Test + public void testAddExpandedExecPathStrings() throws Exception { + List<String> paths = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Artifact.addExpandedExecPathStrings(getFooBarArtifacts(actionGraph, true), paths, + ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + assertSameContents(ImmutableList.of("bar1.h", "bar2.h", "bar3.h"), paths); + } + + @Test + public void testAddExpandedExecPaths() throws Exception { + List<PathFragment> paths = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Artifact.addExpandedExecPaths(getFooBarArtifacts(actionGraph, true), paths, + ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + assertSameContents(ImmutableList.of( + new PathFragment("bar1.h"), new PathFragment("bar2.h"), new PathFragment("bar3.h")), + paths); + } + + @Test + public void testAddExpandedArtifacts() throws Exception { + List<Artifact> expanded = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + List<Artifact> original = getFooBarArtifacts(actionGraph, true); + Artifact.addExpandedArtifacts(original, expanded, + ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + + List<Artifact> manuallyExpanded = new ArrayList<>(); + for (Artifact artifact : original) { + Action action = actionGraph.getGeneratingAction(artifact); + if (artifact.isMiddlemanArtifact()) { + Iterables.addAll(manuallyExpanded, action.getInputs()); + } else { + manuallyExpanded.add(artifact); + } + } + assertSameContents(manuallyExpanded, expanded); + } + + @Test + public void testAddExecPathsNewActionGraph() throws Exception { + List<String> paths = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Artifact.addExecPaths(getFooBarArtifacts(actionGraph, false), paths); + assertSameContents(ImmutableList.of("bar1.h", "bar2.h"), paths); + } + + @Test + public void testAddExpandedExecPathStringsNewActionGraph() throws Exception { + List<String> paths = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Artifact.addExpandedExecPathStrings(getFooBarArtifacts(actionGraph, true), paths, + ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + assertSameContents(ImmutableList.of("bar1.h", "bar2.h", "bar3.h"), paths); + } + + @Test + public void testAddExpandedExecPathsNewActionGraph() throws Exception { + List<PathFragment> paths = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + Artifact.addExpandedExecPaths(getFooBarArtifacts(actionGraph, true), paths, + ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + assertSameContents(ImmutableList.of( + new PathFragment("bar1.h"), new PathFragment("bar2.h"), new PathFragment("bar3.h")), + paths); + } + + @Test + public void testAddExpandedArtifactsNewActionGraph() throws Exception { + List<Artifact> expanded = new ArrayList<>(); + MutableActionGraph actionGraph = new MapBasedActionGraph(); + List<Artifact> original = getFooBarArtifacts(actionGraph, true); + Artifact.addExpandedArtifacts(original, expanded, + ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + + List<Artifact> manuallyExpanded = new ArrayList<>(); + for (Artifact artifact : original) { + Action action = actionGraph.getGeneratingAction(artifact); + if (artifact.isMiddlemanArtifact()) { + Iterables.addAll(manuallyExpanded, action.getInputs()); + } else { + manuallyExpanded.add(artifact); + } + } + assertSameContents(manuallyExpanded, expanded); + } + + @Test + public void testRootRelativePathIsSameAsExecPath() throws Exception { + Root root = Root.asSourceRoot(scratch.dir("/foo")); + Artifact a = new Artifact(scratch.file("/foo/bar1.h"), root); + assertSame(a.getExecPath(), a.getRootRelativePath()); + } + + @Test + public void testToDetailString() throws Exception { + Artifact a = new Artifact(scratch.file("/a/b/c"), Root.asDerivedRoot(scratch.dir("/a/b")), + new PathFragment("b/c")); + assertEquals("[[/a]b]c", a.toDetailString()); + } + + @Test + public void testWeirdArtifact() throws Exception { + try { + new Artifact(scratch.file("/a/b/c"), Root.asDerivedRoot(scratch.dir("/a")), + new PathFragment("c")); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("c: illegal execPath doesn't end with b/c at /a/b/c with root /a[derived]", + e.getMessage()); + } + } + + @Test + public void testSerializeToString() throws Exception { + assertEquals("b/c /3", + new Artifact(scratch.file("/a/b/c"), + Root.asDerivedRoot(scratch.dir("/a"))).serializeToString()); + } + + @Test + public void testSerializeToStringWithExecPath() throws Exception { + Path path = scratch.file("/aaa/bbb/ccc"); + Root root = Root.asDerivedRoot(scratch.dir("/aaa/bbb")); + PathFragment execPath = new PathFragment("bbb/ccc"); + + assertEquals("bbb/ccc /3", new Artifact(path, root, execPath).serializeToString()); + } + + @Test + public void testSerializeToStringWithOwner() throws Exception { + assertEquals("b/c /3 //foo:bar", + new Artifact(scratch.file("/aa/b/c"), Root.asDerivedRoot(scratch.dir("/aa")), + new PathFragment("b/c"), + new LabelArtifactOwner(Label.parseAbsoluteUnchecked("//foo:bar"))).serializeToString()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElementTest.java b/src/test/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElementTest.java new file mode 100644 index 0000000000..28f185c9d5 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ConcurrentMultimapWithHeadElementTest.java @@ -0,0 +1,175 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import com.google.common.testing.GcFinalization; +import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.testutil.TestUtils; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.lang.ref.WeakReference; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Tests for ConcurrentMultimapWithHeadElement. + */ +@RunWith(JUnit4.class) +public class ConcurrentMultimapWithHeadElementTest { + @Test + public void testSmoke() throws Exception { + ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<String, String>(); + assertEquals("val", multimap.putAndGet("key", "val")); + assertEquals("val", multimap.get("key")); + assertEquals("val", multimap.putAndGet("key", "val2")); + multimap.remove("key", "val2"); + assertEquals("val", multimap.get("key")); + assertEquals("val", multimap.putAndGet("key", "val2")); + multimap.remove("key", "val"); + assertEquals("val2", multimap.get("key")); + } + + @Test + public void testDuplicate() throws Exception { + ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<String, String>(); + assertEquals("val", multimap.putAndGet("key", "val")); + assertEquals("val", multimap.get("key")); + assertEquals("val", multimap.putAndGet("key", "val")); + multimap.remove("key", "val"); + assertEquals(null, multimap.get("key")); + } + + @Test + public void testDuplicateWithEqualsObject() throws Exception { + ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<>(); + assertEquals(new String("val"), multimap.putAndGet("key", new String("val"))); + assertEquals(new String("val"), multimap.get("key")); + assertEquals(new String("val"), multimap.putAndGet("key", new String("val"))); + multimap.remove("key", new String("val")); + assertEquals(null, multimap.get("key")); + } + + @Test + public void testFailedRemoval() throws Exception { + ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<String, String>(); + assertEquals("val", multimap.putAndGet("key", "val")); + multimap.remove("key", "val2"); + assertEquals("val", multimap.get("key")); + } + + @Test + public void testNotEmpty() throws Exception { + ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<String, String>(); + assertEquals("val", multimap.putAndGet("key", "val")); + multimap.remove("key", "val2"); + assertEquals("val", multimap.get("key")); + } + + @Test + public void testKeyRemoved() throws Exception { + String key = new String("key"); + ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<String, String>(); + assertEquals("val", multimap.putAndGet(key, "val")); + WeakReference<String> weakKey = new WeakReference<String>(key); + multimap.remove(key, "val"); + key = null; + GcFinalization.awaitClear(weakKey); + } + + @Test + public void testKeyRemovedAndAddedConcurrently() throws Exception { + final ConcurrentMultimapWithHeadElement<String, String> multimap = + new ConcurrentMultimapWithHeadElement<String, String>(); + // Because we have two threads racing, run the test many times. Before fixed, there was a 90% + // chance of failure in 10,000 runs. + for (int i = 0; i < 10000; i++) { + assertEquals("val", multimap.putAndGet("key", "val")); + final CountDownLatch threadStart = new CountDownLatch(1); + TestThread testThread = new TestThread() { + @Override + public void runTest() throws Exception { + threadStart.countDown(); + multimap.remove("key", "val"); + } + }; + testThread.start(); + assertTrue(threadStart.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + assertNotNull(multimap.putAndGet("key", "val2")); // Removal may not have happened yet. + assertNotNull(multimap.get("key")); // If put failed, this will be null. + testThread.joinAndAssertState(2000); + multimap.clear(); + } + } + + private class StressTester extends AbstractQueueVisitor { + private final ConcurrentMultimapWithHeadElement<Boolean, Integer> multimap = + new ConcurrentMultimapWithHeadElement<Boolean, Integer>(); + private final AtomicInteger actionCount = new AtomicInteger(0); + + private StressTester() { + super(/*concurrent=*/true, 200, 200, 1, TimeUnit.SECONDS, + /*failFastOnException=*/true, /*failFastOnInterrupt=*/true, "action-graph-test"); + } + + private void addAndRemove(final Boolean key, final Integer add, final Integer remove) { + enqueue(new Runnable() { + @Override + public void run() { + assertNotNull(multimap.putAndGet(key, add)); + multimap.remove(key, remove); + doRandom(); + } + }); + } + + private Integer getRandomInt() { + return (int) Math.round(Math.random() * 3.0); + } + + private void doRandom() { + if (actionCount.incrementAndGet() > 100000) { + return; + } + Boolean key = Math.random() < 0.5; + addAndRemove(key, getRandomInt(), getRandomInt()); + } + + private void work() throws InterruptedException { + work(/*failFastOnInterrupt=*/true); + } + } + + @Test + public void testStressTest() throws Exception { + StressTester stressTester = new StressTester(); + stressTester.doRandom(); + stressTester.work(); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/CustomCommandLineTest.java b/src/test/java/com/google/devtools/build/lib/actions/CustomCommandLineTest.java new file mode 100644 index 0000000000..3a7db343ca --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/CustomCommandLineTest.java @@ -0,0 +1,161 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomArgv; +import com.google.devtools.build.lib.analysis.actions.CustomCommandLine.CustomMultiArgv; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.testutil.Scratch; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for CustomCommandLine. + */ +@RunWith(JUnit4.class) +public class CustomCommandLineTest { + + private Scratch scratch; + private Root rootDir; + private Artifact artifact1; + private Artifact artifact2; + + @Before + public void setUp() throws Exception { + scratch = new Scratch(); + rootDir = Root.asDerivedRoot(scratch.dir("/exec/root")); + artifact1 = new Artifact(scratch.file("/exec/root/dir/file1.txt"), rootDir); + artifact2 = new Artifact(scratch.file("/exec/root/dir/file2.txt"), rootDir); + } + + @Test + public void testStringArgs() { + CustomCommandLine cl = CustomCommandLine.builder().add("--arg1").add("--arg2").build(); + assertEquals(ImmutableList.of("--arg1", "--arg2"), cl.arguments()); + } + + @Test + public void testLabelArgs() throws SyntaxException { + CustomCommandLine cl = CustomCommandLine.builder().add(Label.parseAbsolute("//a:b")).build(); + assertEquals(ImmutableList.of("//a:b"), cl.arguments()); + } + + @Test + public void testStringsArgs() { + CustomCommandLine cl = CustomCommandLine.builder().add("--arg", + ImmutableList.of("a", "b")).build(); + assertEquals(ImmutableList.of("--arg", "a", "b"), cl.arguments()); + } + + @Test + public void testArtifactExecPathArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addExecPath("--path", artifact1).build(); + assertEquals(ImmutableList.of("--path", "dir/file1.txt"), cl.arguments()); + } + + @Test + public void testArtifactExecPathsArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addExecPaths("--path", + ImmutableList.of(artifact1, artifact2)).build(); + assertEquals(ImmutableList.of("--path", "dir/file1.txt", "dir/file2.txt"), cl.arguments()); + } + + @Test + public void testNestedSetArtifactExecPathsArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addExecPaths( + NestedSetBuilder.<Artifact>stableOrder().add(artifact1).add(artifact2).build()).build(); + assertEquals(ImmutableList.of("dir/file1.txt", "dir/file2.txt"), cl.arguments()); + } + + @Test + public void testArtifactJoinExecPathArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addJoinExecPaths("--path", ":", + ImmutableList.of(artifact1, artifact2)).build(); + assertEquals(ImmutableList.of("--path", "dir/file1.txt:dir/file2.txt"), cl.arguments()); + } + + @Test + public void testPathArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addPath(artifact1.getExecPath()).build(); + assertEquals(ImmutableList.of("dir/file1.txt"), cl.arguments()); + } + + @Test + public void testJoinPathArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addJoinPaths(":", + ImmutableList.of(artifact1.getExecPath(), artifact2.getExecPath())).build(); + assertEquals(ImmutableList.of("dir/file1.txt:dir/file2.txt"), cl.arguments()); + } + + @Test + public void testPathsArgs() { + CustomCommandLine cl = CustomCommandLine.builder().addPaths("%s:%s", + artifact1.getExecPath(), artifact1.getRootRelativePath()).build(); + assertEquals(ImmutableList.of("dir/file1.txt:dir/file1.txt"), cl.arguments()); + } + + @Test + public void testCustomArgs() { + CustomCommandLine cl = CustomCommandLine.builder().add(new CustomArgv() { + @Override + public String argv() { + return "--arg"; + } + }).build(); + assertEquals(ImmutableList.of("--arg"), cl.arguments()); + } + + @Test + public void testCustomMultiArgs() { + CustomCommandLine cl = CustomCommandLine.builder().add(new CustomMultiArgv() { + @Override + public ImmutableList<String> argv() { + return ImmutableList.of("--arg1", "--arg2"); + } + }).build(); + assertEquals(ImmutableList.of("--arg1", "--arg2"), cl.arguments()); + } + + @Test + public void testCombinedArgs() { + CustomCommandLine cl = CustomCommandLine.builder() + .add("--arg") + .add("--args", ImmutableList.of("abc")) + .addExecPaths("--path1", ImmutableList.of(artifact1)) + .addExecPath("--path2", artifact2) + .build(); + assertEquals(ImmutableList.of("--arg", "--args", "abc", "--path1", "dir/file1.txt", "--path2", + "dir/file2.txt"), cl.arguments()); + } + + @Test + public void testAddNulls() { + CustomCommandLine cl = CustomCommandLine.builder() + .add("--args", null) + .addExecPaths(null, ImmutableList.of(artifact1)) + .addExecPath(null, null) + .build(); + assertEquals(ImmutableList.of(), cl.arguments()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java new file mode 100644 index 0000000000..2ed24a6376 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java @@ -0,0 +1,145 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + +import com.google.common.base.Strings; +import com.google.devtools.build.lib.actions.cache.DigestUtils; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * Tests for DigestUtils. + */ +@RunWith(JUnit4.class) +public class DigestUtilsTest { + + private static void assertMd5CalculationConcurrency(boolean expectConcurrent, + final boolean fastDigest, final int fileSize1, final int fileSize2) throws Exception { + final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads. + final CountDownLatch readyLatch = new CountDownLatch(1); // Used to block main thread. + + FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) { + @Override + protected byte[] getMD5Digest(Path path) throws IOException { + try { + barrierLatch.countDown(); + readyLatch.countDown(); + // Either both threads will be inside getMD5Digest at the same time or they + // both will be blocked. + barrierLatch.await(); + } catch (Exception e) { + throw new IOException(e); + } + return super.getMD5Digest(path); + } + + @Override + protected String getFastDigestFunctionType(Path path) { + return "MD5"; + } + + @Override + protected byte[] getFastDigest(Path path) throws IOException { + return fastDigest ? super.getMD5Digest(path) : null; + } + }; + + final Path myFile1 = myfs.getPath("/f1.dat"); + final Path myFile2 = myfs.getPath("/f2.dat"); + FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1)); + FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2)); + + TestThread thread1 = new TestThread () { + @Override public void runTest() throws Exception { + DigestUtils.getDigestOrFail(myFile1, fileSize1); + } + }; + + TestThread thread2 = new TestThread () { + @Override public void runTest() throws Exception { + DigestUtils.getDigestOrFail(myFile2, fileSize2); + } + }; + + thread1.start(); + thread2.start(); + if (!expectConcurrent) { // Synchronized case. + // Wait until at least one thread reached getMD5Digest(). + assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + // Only 1 thread should be inside getMD5Digest(). + assertEquals(1, barrierLatch.getCount()); + barrierLatch.countDown(); // Release barrier latch, allowing both threads to proceed. + } + // Test successful execution within 5 seconds. + thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS); + thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS); + } + + /** + * Ensures that MD5 calculation is synchronized for files + * greater than 4096 bytes if MD5 is not available cheaply, + * so machines with rotating drives don't become unusable. + */ + @Test + public void testMd5CalculationConcurrency() throws Exception { + assertMd5CalculationConcurrency(true, true, 4096, 4096); + assertMd5CalculationConcurrency(true, true, 4097, 4097); + assertMd5CalculationConcurrency(true, false, 4096, 4096); + assertMd5CalculationConcurrency(false, false, 4097, 4097); + assertMd5CalculationConcurrency(true, false, 1024, 4097); + assertMd5CalculationConcurrency(true, false, 1024, 1024); + } + + @Test + public void testRecoverFromMalformedDigest() throws Exception { + final byte[] malformed = {0, 0, 0}; + FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) { + @Override + protected String getFastDigestFunctionType(Path path) { + return "MD5"; + } + + @Override + protected byte[] getFastDigest(Path path) throws IOException { + // MD5 digests are supposed to be 16 bytes. + return malformed; + } + }; + Path path = myFS.getPath("/file"); + FileSystemUtils.writeContentAsLatin1(path, "a"); + byte[] result = DigestUtils.getDigestOrFail(path, 1); + assertArrayEquals(path.getMD5Digest(), result); + assertNotSame(malformed, result); + assertEquals(16, result.length); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java b/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java new file mode 100644 index 0000000000..50d0f25188 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ExecutableSymlinkActionTest.java @@ -0,0 +1,105 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.devtools.build.lib.actions.util.DummyExecutor; +import com.google.devtools.build.lib.analysis.actions.ExecutableSymlinkAction; +import com.google.devtools.build.lib.exec.SingleBuildFileCache; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.testutil.TestFileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ExecutableSymlinkActionTest { + private Scratch scratch = new Scratch(); + private Root inputRoot; + private Root outputRoot; + TestFileOutErr outErr; + private Executor executor; + + @Before + public void setUp() throws Exception { + final Path inputDir = scratch.dir("/in"); + inputRoot = Root.asDerivedRoot(inputDir); + outputRoot = Root.asDerivedRoot(scratch.dir("/out")); + outErr = new TestFileOutErr(); + executor = new DummyExecutor(inputDir); + } + + private ActionExecutionContext createContext() { + Path execRoot = executor.getExecRoot(); + return new ActionExecutionContext( + executor, + new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()), + null, outErr, null); + } + + @Test + public void testSimple() throws Exception { + Path inputFile = inputRoot.getPath().getChild("some-file"); + Path outputFile = outputRoot.getPath().getChild("some-output"); + FileSystemUtils.createEmptyFile(inputFile); + inputFile.setExecutable(/*executable=*/true); + Artifact input = new Artifact(inputFile, inputRoot); + Artifact output = new Artifact(outputFile, outputRoot); + ExecutableSymlinkAction action = new ExecutableSymlinkAction(NULL_ACTION_OWNER, input, output); + action.execute(createContext()); + assertEquals(inputFile, outputFile.resolveSymbolicLinks()); + } + + @Test + public void testFailIfInputIsNotAFile() throws Exception { + Path dir = inputRoot.getPath().getChild("some-dir"); + FileSystemUtils.createDirectoryAndParents(dir); + Artifact input = new Artifact(dir, inputRoot); + Artifact output = new Artifact(outputRoot.getPath().getChild("some-output"), outputRoot); + ExecutableSymlinkAction action = new ExecutableSymlinkAction(NULL_ACTION_OWNER, input, output); + try { + action.execute(createContext()); + fail(); + } catch (ActionExecutionException e) { + assertTrue(e.getMessage().contains("'some-dir' is not a file")); + } + } + + @Test + public void testFailIfInputIsNotExecutable() throws Exception { + Path file = inputRoot.getPath().getChild("some-file"); + FileSystemUtils.createEmptyFile(file); + file.setExecutable(/*executable=*/false); + Artifact input = new Artifact(file, inputRoot); + Artifact output = new Artifact(outputRoot.getPath().getChild("some-output"), outputRoot); + ExecutableSymlinkAction action = new ExecutableSymlinkAction(NULL_ACTION_OWNER, input, output); + try { + action.execute(createContext()); + fail(); + } catch (ActionExecutionException e) { + String want = "'some-file' is not executable"; + String got = e.getMessage(); + assertTrue(String.format("got %s, want %s", got, want), got.contains(want)); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/FailActionTest.java b/src/test/java/com/google/devtools/build/lib/actions/FailActionTest.java new file mode 100644 index 0000000000..7838fca951 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/FailActionTest.java @@ -0,0 +1,81 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER; +import static com.google.devtools.build.lib.testutil.MoreAsserts.assertSameContents; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.testutil.Scratch; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Collection; +import java.util.Collections; + +@RunWith(JUnit4.class) +public class FailActionTest { + + private Scratch scratch = new Scratch(); + + private String errorMessage; + private Artifact anOutput; + private Collection<Artifact> outputs; + private FailAction failAction; + + protected MutableActionGraph actionGraph = new MapBasedActionGraph(); + + @Before + public void setUp() throws Exception { + errorMessage = "An error just happened."; + anOutput = new Artifact(scratch.file("/out/foo"), + Root.asDerivedRoot(scratch.dir("/"), scratch.dir("/out"))); + outputs = ImmutableList.of(anOutput); + failAction = new FailAction(NULL_ACTION_OWNER, outputs, errorMessage); + actionGraph.registerAction(failAction); + assertSame(failAction, actionGraph.getGeneratingAction(anOutput)); + } + + @Test + public void testExecutingItYieldsExceptionWithErrorMessage() { + try { + failAction.execute(null); + fail(); + } catch (ActionExecutionException e) { + assertEquals(errorMessage, e.getMessage()); + } + } + + @Test + public void testInputsAreEmptySet() { + assertSameContents(Collections.emptySet(), failAction.getInputs()); + } + + @Test + public void testRetainsItsOutputs() { + assertSameContents(outputs, failAction.getOutputs()); + } + + @Test + public void testPrimaryOutput() { + assertSame(anOutput, failAction.getPrimaryOutput()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/LocalHostCapacityTest.java b/src/test/java/com/google/devtools/build/lib/actions/LocalHostCapacityTest.java new file mode 100644 index 0000000000..5e2e1a8c32 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/LocalHostCapacityTest.java @@ -0,0 +1,434 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LocalHostCapacityTest { + + private FsApparatus scratch = FsApparatus.newNative(); + + @Test + public void testNonHyperthreadedMachine() throws Exception { + String cpuinfoContent = StringUtilities.joinLines( + "processor\t: 0", + "vendor_id\t: GenuineIntel", + "cpu family\t: 15", + "model\t\t: 4", + "model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz", + "stepping\t: 10", + "cpu MHz\t\t: 3400.000", + "cache size\t: 2048 KB", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 5", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca " + + "cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm " + + "syscall nx lm constant_tsc up pni monitor ds_cpl est cid cx16 " + + "xtpr lahf_lm", + "bogomips\t: 6803.83", + "clflush size\t: 64", + "cache_alignment\t: 128", + "address sizes\t: 36 bits physical, 48 bits virtual", + "power management:" + ); + String cpuinfoFile = + scratch.file("test_cpuinfo_nonht", cpuinfoContent).getPathString(); + String meminfoContent = StringUtilities.joinLines( + "MemTotal: 3091732 kB", + "MemFree: 2167344 kB", + "Buffers: 60644 kB", + "Cached: 509940 kB", + "SwapCached: 0 kB", + "Active: 636892 kB", + "Inactive: 212760 kB", + "HighTotal: 0 kB", + "HighFree: 0 kB", + "LowTotal: 3091732 kB", + "LowFree: 2167344 kB", + "SwapTotal: 9124880 kB", + "SwapFree: 9124880 kB", + "Dirty: 0 kB", + "Writeback: 0 kB", + "AnonPages: 279028 kB", + "Mapped: 54404 kB", + "Slab: 42820 kB", + "PageTables: 5184 kB", + "NFS_Unstable: 0 kB", + "Bounce: 0 kB", + "CommitLimit: 10670744 kB", + "Committed_AS: 665840 kB", + "VmallocTotal: 34359738367 kB", + "VmallocUsed: 300484 kB", + "VmallocChunk: 34359437307 kB", + "HugePages_Total: 0", + "HugePages_Free: 0", + "HugePages_Rsvd: 0", + "Hugepagesize: 2048 kB" + ); + String stat1Content = StringUtilities.joinLines( + "cpu 29793342 260290 3479274 636259369 6683218 656426 714057 0", + "cpu0 29793342 260290 3479274 636259369 6683218 656426 714057 0", + "intr 2870488853 2486517107 3 0 0 2 0 5 0 0 0 0 0 3 0 74363716 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 52483586 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 973315 0 0 0 0 0 0 0 46 " + + "0 0 0 0 0 0 0 98792358 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 114339590 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43019122 0 0 0 0 0", + "ctxt 15053799843", + "btime 1199289688", + "processes 25799993", + "procs_running 1", + "procs_blocked 0" + ); + String stat2Content = StringUtilities.joinLines( + "cpu 29794509 260290 3479474 636287862 6683283 656450 714087 0 0", + "cpu0 29794509 260290 3479474 636287862 6683283 656450 714087 0 0", + "intr 2870488853 2486517107 3 0 0 2 0 5 0 0 0 0 0 3 0 74363716 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 52483586 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 973315 0 0 0 0 0 0 0 46 " + + "0 0 0 0 0 0 0 98792358 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 114339590 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 " + + "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 43019122 0 0 0 0 0", + "ctxt 15053799843", + "btime 1199289688", + "processes 25799993", + "procs_running 1", + "procs_blocked 0" + ); + String meminfoFile = + scratch.file("test_meminfo_nonht", meminfoContent).getPathString(); + String stat1File = + scratch.file("proc_stat_1", stat1Content).getPathString(); + String stat2File = + scratch.file("proc_stat_2", stat2Content).getPathString(); + assertEquals(1, LocalHostCapacity.getLogicalCpuCount(cpuinfoContent)); + assertEquals(1, LocalHostCapacity.getPhysicalCpuCount(cpuinfoContent, 1)); + assertEquals(1, LocalHostCapacity.getCoresPerCpu(cpuinfoContent)); + ResourceSet capacity = + LocalHostCapacity.getLocalHostCapacity(cpuinfoFile, meminfoFile); + assertEquals(1.0, capacity.getCpuUsage(), 0.01); + assertEquals(3091.732, capacity.getMemoryMb(), 0.1); // +/- 0.1MB + LocalHostCapacity.setLocalHostCapacity(capacity); + assertSame(capacity, LocalHostCapacity.getLocalHostCapacity()); + Clock mockedClock = new Clock() { + private int callCount = 0; + + @Override + public long currentTimeMillis() { + throw new AssertionError("unexpected method call"); + } + + @Override + public long nanoTime() { + callCount++; + if (callCount == 1) { + return 0; + } else if (callCount == 2) { + return 100 * 1000000; + } else if (callCount == 3) { + return 200 * 1000000; + } else { + throw new AssertionError("unexpected method call"); + } + } + }; + LocalHostCapacity.FreeResources freeStats = + LocalHostCapacity.getFreeResources(mockedClock, meminfoFile, stat1File, null); + assertNotNull(freeStats); + assertEquals(2356.756, freeStats.getFreeMb(), 0.001); + assertEquals(0.0, freeStats.getAvgFreeCpu(), 0); + // The next call to the mock clock returns a timestamp as if 100 ms have passed. + assertTrue(freeStats.getReadingAge() > 50); + // Fake another 100 ms going by for the next call. + freeStats = LocalHostCapacity.getFreeResources(mockedClock, meminfoFile, stat2File, freeStats); + assertNotNull(freeStats); + assertEquals(2356.756, freeStats.getFreeMb(), 0.001); + assertTrue(freeStats.getInterval() > 100); + assertEquals(0.95, freeStats.getAvgFreeCpu(), 0.001); + } + + @Test + public void testHyperthreadedMachine() throws Exception { + String cpuinfoContent = StringUtilities.joinLines( + "processor\t: 0", + "vendor_id\t: GenuineIntel", + "cpu family\t: 15", + "model\t\t: 4", + "model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz", + "stepping\t: 1", + "cpu MHz\t\t: 3400.245", + "cache size\t: 1024 KB", + "physical id\t: 0", + "siblings\t: 2", + "core id\t\t: 0", + "cpu cores\t: 1", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 5", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge " + + "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm " + + "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr", + "bogomips\t: 6806.31", + "clflush size\t: 64", + "cache_alignment\t: 128", + "address sizes\t: 36 bits physical, 48 bits virtual", + "power management:", + "", + "processor\t: 1", + "vendor_id\t: GenuineIntel", + "cpu family\t: 15", + "model\t\t: 4", + "model name\t: Intel(R) Pentium(R) 4 CPU 3.40GHz", + "stepping\t: 1", + "cpu MHz\t\t: 3400.245", + "cache size\t: 1024 KB", + "physical id\t: 0", + "siblings\t: 2", + "core id\t\t: 0", + "cpu cores\t: 1", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 5", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge " + + "mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm " + + "syscall lm constant_tsc pni monitor ds_cpl cid cx16 xtpr", + "bogomips\t: 6800.76", + "clflush size\t: 64", + "cache_alignment\t: 128", + "address sizes\t: 36 bits physical, 48 bits virtual", + "power management:", + "" + ); + String cpuinfoFile = + scratch.file("test_cpuinfo_ht", cpuinfoContent).getPathString(); + String meminfoContent = StringUtilities.joinLines( + "MemTotal: 3092004 kB", + "MemFree: 26124 kB", + "Buffers: 3836 kB", + "Cached: 52400 kB", + "SwapCached: 68204 kB", + "Active: 2281464 kB", + "Inactive: 260908 kB", + "HighTotal: 0 kB", + "HighFree: 0 kB", + "LowTotal: 3092004 kB", + "LowFree: 26124 kB", + "SwapTotal: 9124880 kB", + "SwapFree: 8264920 kB", + "Dirty: 616 kB", + "Writeback: 0 kB", + "AnonPages: 2466336 kB", + "Mapped: 37576 kB", + "Slab: 483004 kB", + "PageTables: 11912 kB", + "NFS_Unstable: 0 kB", + "Bounce: 0 kB", + "CommitLimit: 10670880 kB", + "Committed_AS: 3627984 kB", + "VmallocTotal: 34359738367 kB", + "VmallocUsed: 300460 kB", + "VmallocChunk: 34359437307 kB", + "HugePages_Total: 0", + "HugePages_Free: 0", + "HugePages_Rsvd: 0", + "Hugepagesize: 2048 kB" + ); + String meminfoFile = + scratch.file("test_meminfo_ht", meminfoContent).getPathString(); + assertEquals(2, LocalHostCapacity.getLogicalCpuCount(cpuinfoContent)); + assertEquals(1, LocalHostCapacity.getPhysicalCpuCount(cpuinfoContent, 2)); + assertEquals(1, LocalHostCapacity.getCoresPerCpu(cpuinfoContent)); + ResourceSet capacity = + LocalHostCapacity.getLocalHostCapacity(cpuinfoFile, meminfoFile); + assertEquals(1.2, capacity.getCpuUsage(), .0001); + assertEquals(3092.004, capacity.getMemoryMb(), 0.1); // +/- 0.1MB + } + + @Test + public void testAMDMachine() throws Exception { + String cpuinfoContent = StringUtilities.joinLines( + "processor\t: 0", + "vendor_id\t: AuthenticAMD", + "cpu family\t: 15", + "model\t\t: 65", + "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE", + "stepping\t: 2", + "cpu MHz\t\t: 2200.000", + "cache size\t: 1024 KB", + "physical id\t: 0", + "siblings\t: 2", + "core id\t\t: 0", + "cpu cores\t: 2", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 1", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr " + + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall " + + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm " + + "cmp_legacy svm cr8_legacy", + "bogomips\t: 4425.84", + "TLB size\t: 1024 4K pages", + "clflush size\t: 64", + "cache_alignment\t: 64", + "address sizes\t: 40 bits physical, 48 bits virtual", + "power management: ts fid vid ttp tm stc", + "", + "processor\t: 1", + "vendor_id\t: AuthenticAMD", + "cpu family\t: 15", + "model\t\t: 65", + "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE", + "stepping\t: 2", + "cpu MHz\t\t: 2200.000", + "cache size\t: 1024 KB", + "physical id\t: 0", + "siblings\t: 2", + "core id\t\t: 1", + "cpu cores\t: 2", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 1", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr " + + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall " + + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm " + + "cmp_legacy svm cr8_legacy", + "bogomips\t: 4460.61", + "TLB size\t: 1024 4K pages", + "clflush size\t: 64", + "cache_alignment\t: 64", + "address sizes\t: 40 bits physical, 48 bits virtual", + "power management: ts fid vid ttp tm stc", + "", + "processor\t: 2", + "vendor_id\t: AuthenticAMD", + "cpu family\t: 15", + "model\t\t: 65", + "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE", + "stepping\t: 2", + "cpu MHz\t\t: 2200.000", + "cache size\t: 1024 KB", + "physical id\t: 1", + "siblings\t: 2", + "core id\t\t: 0", + "cpu cores\t: 2", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 1", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr " + + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall " + + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm " + + "cmp_legacy svm cr8_legacy", + "bogomips\t: 4420.45", + "TLB size\t: 1024 4K pages", + "clflush size\t: 64", + "cache_alignment\t: 64", + "address sizes\t: 40 bits physical, 48 bits virtual", + "power management: ts fid vid ttp tm stc", + "", + "processor\t: 3", + "vendor_id\t: AuthenticAMD", + "cpu family\t: 15", + "model\t\t: 65", + "model name\t: Dual-Core AMD Opteron(tm) Processor 8214 HE", + "stepping\t: 2", + "cpu MHz\t\t: 2200.000", + "cache size\t: 1024 KB", + "physical id\t: 1", + "siblings\t: 2", + "core id\t\t: 1", + "cpu cores\t: 2", + "fpu\t\t: yes", + "fpu_exception\t: yes", + "cpuid level\t: 1", + "wp\t\t: yes", + "flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr " + + "pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall " + + "nx mmxext fxsr_opt rdtscp lm 3dnowext 3dnow pni cx16 lahf_lm " + + "cmp_legacy svm cr8_legacy", + "bogomips\t: 4460.39", + "TLB size\t: 1024 4K pages", + "clflush size\t: 64", + "cache_alignment\t: 64", + "address sizes\t: 40 bits physical, 48 bits virtual", + "power management: ts fid vid ttp tm stc", + "" + ); + String cpuinfoFile = + scratch.file("test_cpuinfo_amd", cpuinfoContent).getPathString(); + String meminfoContent = StringUtilities.joinLines( + "MemTotal: 8223956 kB", + "MemFree: 3670396 kB", + "Buffers: 374068 kB", + "Cached: 3366980 kB", + "SwapCached: 0 kB", + "Active: 3275860 kB", + "Inactive: 737816 kB", + "HighTotal: 0 kB", + "HighFree: 0 kB", + "LowTotal: 8223956 kB", + "LowFree: 3670396 kB", + "SwapTotal: 6024332 kB", + "SwapFree: 6024332 kB", + "Dirty: 84 kB", + "Writeback: 0 kB", + "AnonPages: 272308 kB", + "Mapped: 62604 kB", + "Slab: 506140 kB", + "PageTables: 4608 kB", + "NFS_Unstable: 0 kB", + "Bounce: 0 kB", + "CommitLimit: 10136308 kB", + "Committed_AS: 600672 kB", + "VmallocTotal: 34359738367 kB", + "VmallocUsed: 299068 kB", + "VmallocChunk: 34359438843 kB", + "HugePages_Total: 0", + "HugePages_Free: 0", + "HugePages_Rsvd: 0", + "Hugepagesize: 2048 kB"); + String meminfoFile = + scratch.file("test_meminfo_amd", meminfoContent).getPathString(); + assertEquals(4, LocalHostCapacity.getLogicalCpuCount(cpuinfoContent)); + assertEquals(2, LocalHostCapacity.getPhysicalCpuCount(cpuinfoContent, 4)); + assertEquals(2, LocalHostCapacity.getCoresPerCpu(cpuinfoContent)); + ResourceSet capacity = + LocalHostCapacity.getLocalHostCapacity(cpuinfoFile, meminfoFile); + assertEquals(capacity.getCpuUsage(), 4.0, 0.01); + assertEquals(8223.956, capacity.getMemoryMb(), 0.1); // +/- 0.1MB + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/MapBasedActionGraphTest.java b/src/test/java/com/google/devtools/build/lib/actions/MapBasedActionGraphTest.java new file mode 100644 index 0000000000..cde9aed35c --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/MapBasedActionGraphTest.java @@ -0,0 +1,147 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil.UncheckedActionConflictException; +import com.google.devtools.build.lib.actions.util.TestAction; +import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Tests for {@link MapBasedActionGraph}. + */ +@RunWith(JUnit4.class) +public class MapBasedActionGraphTest { + @Test + public void testSmoke() throws Exception { + MutableActionGraph actionGraph = new MapBasedActionGraph(); + FileSystem fileSystem = new InMemoryFileSystem(BlazeClock.instance()); + Path path = fileSystem.getPath("/root/foo"); + Artifact output = new Artifact(path, Root.asDerivedRoot(path)); + Action action = new TestAction(TestAction.NO_EFFECT, + ImmutableSet.<Artifact>of(), ImmutableSet.of(output)); + actionGraph.registerAction(action); + actionGraph.unregisterAction(action); + path = fileSystem.getPath("/root/bar"); + output = new Artifact(path, Root.asDerivedRoot(path)); + Action action2 = new TestAction(TestAction.NO_EFFECT, + ImmutableSet.<Artifact>of(), ImmutableSet.of(output)); + actionGraph.registerAction(action); + actionGraph.registerAction(action2); + actionGraph.unregisterAction(action); + } + + @Test + public void testNoActionConflictWhenUnregisteringSharedAction() throws Exception { + MutableActionGraph actionGraph = new MapBasedActionGraph(); + FileSystem fileSystem = new InMemoryFileSystem(BlazeClock.instance()); + Path path = fileSystem.getPath("/root/foo"); + Artifact output = new Artifact(path, Root.asDerivedRoot(path)); + Action action = new TestAction(TestAction.NO_EFFECT, + ImmutableSet.<Artifact>of(), ImmutableSet.of(output)); + actionGraph.registerAction(action); + Action otherAction = new TestAction(TestAction.NO_EFFECT, + ImmutableSet.<Artifact>of(), ImmutableSet.of(output)); + actionGraph.registerAction(otherAction); + actionGraph.unregisterAction(action); + } + + private class ActionRegisterer extends AbstractQueueVisitor { + private final MutableActionGraph graph = new MapBasedActionGraph(); + private final Artifact output; + // Just to occasionally add actions that were already present. + private final Set<Action> allActions = Sets.newConcurrentHashSet(); + private final AtomicInteger actionCount = new AtomicInteger(0); + + private ActionRegisterer() { + super(/*concurrent=*/true, 200, 200, 1, TimeUnit.SECONDS, + /*failFastOnException=*/true, /*failFastOnInterrupt=*/true, "action-graph-test"); + FileSystem fileSystem = new InMemoryFileSystem(BlazeClock.instance()); + Path path = fileSystem.getPath("/root/foo"); + output = new Artifact(path, Root.asDerivedRoot(path)); + allActions.add(new TestAction(TestAction.NO_EFFECT, + ImmutableSet.<Artifact>of(), ImmutableSet.of(output))); + } + + private void registerAction(final Action action) { + enqueue(new Runnable() { + @Override + public void run() { + try { + graph.registerAction(action); + } catch (ActionConflictException e) { + throw new UncheckedActionConflictException(e); + } + doRandom(); + } + }); + } + + private void unregisterAction(final Action action) { + enqueue(new Runnable() { + @Override + public void run() { + graph.unregisterAction(action); + doRandom(); + } + }); + } + + private void doRandom() { + if (actionCount.incrementAndGet() > 10000) { + return; + } + Action action = null; + if (Math.random() < 0.5) { + action = Iterables.getFirst(allActions, null); + } else { + action = new TestAction(TestAction.NO_EFFECT, + ImmutableSet.<Artifact>of(), ImmutableSet.of(output)); + allActions.add(action); + } + if (Math.random() < 0.5) { + registerAction(action); + } else { + unregisterAction(action); + } + } + + private void work() throws InterruptedException { + work(/*failFastOnInterrupt=*/true); + } + } + + @Test + public void testSharedActionStressTest() throws Exception { + ActionRegisterer actionRegisterer = new ActionRegisterer(); + actionRegisterer.doRandom(); + actionRegisterer.work(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java new file mode 100644 index 0000000000..4955288aa7 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/ResourceManagerTest.java @@ -0,0 +1,400 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableSet; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.common.options.OptionsParsingException; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.annotation.Nullable; + +/** + * + * Tests for @{link ResourceManager}. + */ +@RunWith(JUnit4.class) +public class ResourceManagerTest { + + private final ActionMetadata resourceOwner = new ResourceOwnerStub(); + private final ResourceManager rm = ResourceManager.instanceForTestingOnly(); + private AtomicInteger counter; + CyclicBarrier sync; + CyclicBarrier sync2; + + @Before + public void setUp() throws Exception { + rm.setRamUtilizationPercentage(100); + rm.setAvailableResources(new ResourceSet(1000, 1, 1)); + rm.setEventBus(new EventBus()); + counter = new AtomicInteger(0); + sync = new CyclicBarrier(2); + sync2 = new CyclicBarrier(2); + rm.resetResourceUsage(); + } + + private void acquire(double ram, double cpu, double io) throws InterruptedException { + rm.acquireResources(resourceOwner, new ResourceSet(ram, cpu, io)); + } + + private boolean acquireNonblocking(double ram, double cpu, double io) { + return rm.tryAcquire(resourceOwner, new ResourceSet(ram, cpu, io)); + } + + private void release(double ram, double cpu, double io) { + rm.releaseResources(resourceOwner, new ResourceSet(ram, cpu, io)); + } + + private void validate (int count) { + assertEquals(count, counter.incrementAndGet()); + } + + @Test + public void testIndependentLargeRequests() throws Exception { + // Available: 1000 RAM and 1 CPU. + assertFalse(rm.inUse()); + acquire(10000, 0, 0); // Available: 0 RAM 1 CPU 1 IO. + acquire(0, 100, 0); // Available: 0 RAM 0 CPU 1 IO. + acquire(0, 0, 1); // Available: 0 RAM 0 CPU 0 IO. + assertTrue(rm.inUse()); + release(9500, 0, 0); // Available: 500 RAM 0 CPU 0 IO. + acquire(400, 0, 0); // Available: 100 RAM 0 CPU 0 IO. + release(0, 99.5, 0.6); // Available: 100 RAM 0.5 CPU 0.4 IO. + acquire(100, 0.5, 0.4); // Available: 0 RAM 0 CPU 0 IO. + release(1000, 1, 1); // Available: 1000 RAM 1 CPU 1 IO. + assertFalse(rm.inUse()); + } + + @Test + public void testOverallocation() throws Exception { + // Since ResourceManager.MIN_NECESSARY_RAM_RATIO = 1.0, overallocation is + // enabled only for the CPU resource. + assertFalse(rm.inUse()); + acquire(900, 0.5, 0.1); // Available: 100 RAM 0.5 CPU 0.9 IO. + acquire(100, 0.6, 0.9); // Available: 0 RAM 0 CPU 0 IO. + release(100, 0.6, 0.9); // Available: 100 RAM 0.5 CPU 0.9 IO. + acquire(100, 0.1, 0.1); // Available: 0 RAM 0.4 CPU 0.8 IO. + acquire(0, 0.5, 0.8); // Available: 0 RAM 0 CPU 0.8 IO. + release(1020, 1.1, 1.05); // Available: 1000 RAM 1 CPU 1 IO. + assertFalse(rm.inUse()); + } + + @Test + public void testNonblocking() throws Exception { + assertFalse(rm.inUse()); + assertTrue(acquireNonblocking(900, 0.5, 0)); // Available: 100 RAM 0.5 CPU 1 IO. + assertTrue(acquireNonblocking(100, 0.5, 0.2)); // Available: 0 RAM 0 CPU 0.8 IO. + assertFalse(acquireNonblocking(.1, .01, 0.0)); + assertFalse(acquireNonblocking(0, 0, 0.9)); + assertTrue(acquireNonblocking(0, 0, 0.8)); // Available: 0 RAM 0 CPU 0 IO. + release(100, 0.5, 0.1); // Available: 100 RAM 0.5 CPU 0.1 IO. + assertTrue(acquireNonblocking(100, 0.1, 0.1)); // Available: 0 RAM 0.4 CPU 0 IO. + assertFalse(acquireNonblocking(5, .5, 0)); + assertFalse(acquireNonblocking(0, .5, 0.1)); + assertTrue(acquireNonblocking(0, 0.4, 0)); // Available: 0 RAM 0 CPU 0 IO. + release(1000, 1, 1); // Available: 1000 RAM 1 CPU 1 IO. + assertFalse(rm.inUse()); + } + + @Test + public void testHasResources() throws Exception { + assertFalse(rm.inUse()); + assertFalse(rm.threadHasResources()); + acquire(1, .1, .1); + assertTrue(rm.threadHasResources()); + + // We have resources in this thread - make sure other threads + // are not affected. + TestThread thread1 = new TestThread () { + @Override public void runTest() throws Exception { + assertFalse(rm.threadHasResources()); + acquire(1, 0, 0); + assertTrue(rm.threadHasResources()); + release(1, 0, 0); + assertFalse(rm.threadHasResources()); + acquire(0, 0.1, 0); + assertTrue(rm.threadHasResources()); + release(0, 0.1, 0); + assertFalse(rm.threadHasResources()); + acquire(0, 0, 0.1); + assertTrue(rm.threadHasResources()); + release(0, 0, 0.1); + assertFalse(rm.threadHasResources()); + } + }; + thread1.start(); + thread1.joinAndAssertState(10000); + + release(1, .1, .1); + assertFalse(rm.threadHasResources()); + assertFalse(rm.inUse()); + } + + @Test + public void testConcurrentLargeRequests() throws Exception { + assertFalse(rm.inUse()); + TestThread thread1 = new TestThread () { + @Override public void runTest() throws Exception { + acquire(2000, 2, 0); + sync.await(); + validate(1); + sync.await(); + // Wait till other thread will be locked. + while (rm.getWaitCount() == 0) { + Thread.yield(); + } + release(2000, 2, 0); + assertEquals(0, rm.getWaitCount()); + acquire(2000, 2, 0); // Will be blocked by the thread2. + validate(3); + release(2000, 2, 0); + } + }; + TestThread thread2 = new TestThread () { + @Override public void runTest() throws Exception { + sync2.await(); + assertFalse(rm.isAvailable(2000, 2, 0)); + acquire(2000, 2, 0); // Will be blocked by the thread1. + validate(2); + sync2.await(); + // Wait till other thread will be locked. + while (rm.getWaitCount() == 0) { + Thread.yield(); + } + release(2000, 2, 0); + } + }; + + thread1.start(); + thread2.start(); + sync.await(1, TimeUnit.SECONDS); + assertTrue(rm.inUse()); + assertEquals(0, rm.getWaitCount()); + sync2.await(1, TimeUnit.SECONDS); + sync.await(1, TimeUnit.SECONDS); + sync2.await(1, TimeUnit.SECONDS); + thread1.joinAndAssertState(1000); + thread2.joinAndAssertState(1000); + assertFalse(rm.inUse()); + } + + @Test + public void testOutOfOrderAllocation() throws Exception { + assertFalse(rm.inUse()); + TestThread thread1 = new TestThread () { + @Override public void runTest() throws Exception { + sync.await(); + acquire(900, 0.5, 0); // Will be blocked by the main thread. + validate(5); + release(900, 0.5, 0); + sync.await(); + } + }; + TestThread thread2 = new TestThread() { + @Override public void runTest() throws Exception { + // Wait till other thread will be locked + while (rm.getWaitCount() == 0) { + Thread.yield(); + } + acquire(100, 0.1, 0); + validate(2); + release(100, 0.1, 0); + sync2.await(); + acquire(200, 0.5, 0); + validate(4); + sync2.await(); + release(200, 0.5, 0); + } + }; + acquire(900, 0.9, 0); + validate(1); + thread1.start(); + sync.await(1, TimeUnit.SECONDS); + thread2.start(); + sync2.await(1, TimeUnit.SECONDS); + //Waiting till both threads are locked. + while (rm.getWaitCount() < 2) { + Thread.yield(); + } + validate(3); // Thread1 is now first in the queue and Thread2 is second. + release(100, 0.4, 0); // This allows Thread2 to continue out of order. + sync2.await(1, TimeUnit.SECONDS); + release(750, 0.3, 0); // At this point thread1 will finally acquire resources. + sync.await(1, TimeUnit.SECONDS); + release(50, 0.2, 0); + thread1.join(); + thread2.join(); + assertFalse(rm.inUse()); + } + + @Test + public void testSingleton() throws Exception { + ResourceManager.instance(); + } + + /** + * Checks that that resource manager + * can recover from LocalHostCapacity.getFreeResources() failure. + */ + @Test + public void testAutoSenseFailure() throws Exception { + boolean isDisabled = LocalHostCapacity.isDisabled; + assertFalse(rm.inUse()); + try { + rm.setAutoSensing(true); + // Resource manager autosense state should be enabled now if + // LocalHostCapacity class supports it. + assertEquals(rm.isAutoSensingEnabled(), !LocalHostCapacity.isDisabled); + rm.setAutoSensing(false); + assertFalse(rm.isAutoSensingEnabled()); + + // Emulate failure to parse /proc/* filesystem. + LocalHostCapacity.isDisabled = true; + rm.setAutoSensing(true); + assertFalse(rm.isAutoSensingEnabled()); + rm.setAutoSensing(false); + assertFalse(rm.isAutoSensingEnabled()); + } finally { + LocalHostCapacity.isDisabled = isDisabled; + rm.setAutoSensing(false); + } + assertFalse(rm.inUse()); + } + + @Test + public void testResourceSetConverter() throws Exception { + ResourceSet.ResourceSetConverter converter = new ResourceSet.ResourceSetConverter(); + + ResourceSet resources = converter.convert("1,0.5,2"); + assertEquals(1.0, resources.getMemoryMb(), 0.01); + assertEquals(0.5, resources.getCpuUsage(), 0.01); + assertEquals(2.0, resources.getIoUsage(), 0.01); + + try { + converter.convert("0,0,"); + fail(); + } catch (OptionsParsingException ope) { + // expected + } + + try { + converter.convert("0,0,0,0"); + fail(); + } catch (OptionsParsingException ope) { + // expected + } + + try { + converter.convert("-1,0,0"); + fail(); + } catch (OptionsParsingException ope) { + // expected + } + } + + private static class ResourceOwnerStub implements ActionMetadata { + + @Override + @Nullable + public String getProgressMessage() { + throw new IllegalStateException(); + } + + @Override + public ActionOwner getOwner() { + throw new IllegalStateException(); + } + + @Override + public String prettyPrint() { + throw new IllegalStateException(); + } + + @Override + public String getMnemonic() { + throw new IllegalStateException(); + } + + @Override + public String describeStrategy(Executor executor) { + throw new IllegalStateException(); + } + + @Override + public boolean inputsKnown() { + throw new IllegalStateException(); + } + + @Override + public boolean discoversInputs() { + throw new IllegalStateException(); + } + + @Override + public Iterable<Artifact> getInputs() { + throw new IllegalStateException(); + } + + @Override + public int getInputCount() { + throw new IllegalStateException(); + } + + @Override + public ImmutableSet<Artifact> getOutputs() { + throw new IllegalStateException(); + } + + @Override + public Artifact getPrimaryInput() { + throw new IllegalStateException(); + } + + @Override + public Artifact getPrimaryOutput() { + throw new IllegalStateException(); + } + + @Override + public Iterable<Artifact> getMandatoryInputs() { + throw new IllegalStateException(); + } + + @Override + public String getKey() { + throw new IllegalStateException(); + } + + @Override + @Nullable + public String describeKey() { + throw new IllegalStateException(); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/RootTest.java b/src/test/java/com/google/devtools/build/lib/actions/RootTest.java new file mode 100644 index 0000000000..c8fc14bf82 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/RootTest.java @@ -0,0 +1,132 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; + +/** + * Tests for {@link Root}. + */ +@RunWith(JUnit4.class) +public class RootTest { + private Scratch scratch = new Scratch(); + + @Test + public void testAsSourceRoot() throws IOException { + Path sourceDir = scratch.dir("/source"); + Root root = Root.asSourceRoot(sourceDir); + assertTrue(root.isSourceRoot()); + assertEquals(PathFragment.EMPTY_FRAGMENT, root.getExecPath()); + assertEquals(sourceDir, root.getPath()); + assertEquals("/source[source]", root.toString()); + } + + @Test + public void testBadAsSourceRoot() { + try { + Root.asSourceRoot(null); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void testAsDerivedRoot() throws IOException { + Path execRoot = scratch.dir("/exec"); + Path rootDir = scratch.dir("/exec/root"); + Root root = Root.asDerivedRoot(execRoot, rootDir); + assertFalse(root.isSourceRoot()); + assertEquals(new PathFragment("root"), root.getExecPath()); + assertEquals(rootDir, root.getPath()); + assertEquals("/exec/root[derived]", root.toString()); + } + + @Test + public void testBadAsDerivedRoot() throws IOException { + try { + Path execRoot = scratch.dir("/exec"); + Path outsideDir = scratch.dir("/not_exec"); + Root.asDerivedRoot(execRoot, outsideDir); + fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testBadAsDerivedRootSameForBoth() throws IOException { + try { + Path execRoot = scratch.dir("/exec"); + Root.asDerivedRoot(execRoot, execRoot); + fail(); + } catch (IllegalArgumentException expected) { + } + } + + @Test + public void testBadAsDerivedRootNullDir() throws IOException { + try { + Path execRoot = scratch.dir("/exec"); + Root.asDerivedRoot(execRoot, null); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void testBadAsDerivedRootNullExecRoot() throws IOException { + try { + Path execRoot = scratch.dir("/exec"); + Root.asDerivedRoot(null, execRoot); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void testEquals() throws IOException { + Path execRoot = scratch.dir("/exec"); + Path rootDir = scratch.dir("/exec/root"); + Path otherRootDir = scratch.dir("/"); + Path sourceDir = scratch.dir("/source"); + Root rootA = Root.asDerivedRoot(execRoot, rootDir); + assertEqualsAndHashCode(true, rootA, Root.asDerivedRoot(execRoot, rootDir)); + assertEqualsAndHashCode(false, rootA, Root.asSourceRoot(sourceDir)); + assertEqualsAndHashCode(false, rootA, Root.asSourceRoot(rootDir)); + assertEqualsAndHashCode(false, rootA, Root.asDerivedRoot(otherRootDir, rootDir)); + } + + public void assertEqualsAndHashCode(boolean expected, Object a, Object b) { + if (expected) { + assertTrue(a.equals(b)); + assertTrue(a.hashCode() == b.hashCode()); + } else { + assertFalse(a.equals(b)); + assertFalse(a.hashCode() == b.hashCode()); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCacheTest.java b/src/test/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCacheTest.java new file mode 100644 index 0000000000..5fc1f4f169 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCacheTest.java @@ -0,0 +1,190 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; + +/** + * Test for the CompactPersistentActionCache class. + */ +@RunWith(JUnit4.class) +public class CompactPersistentActionCacheTest { + + private static class ManualClock implements Clock { + private long currentTime = 0L; + + ManualClock() { } + + @Override public long currentTimeMillis() { + return currentTime; + } + + @Override public long nanoTime() { + return 0; + } + } + + private FsApparatus scratch = FsApparatus.newInMemory(); + private Path dataRoot; + private Path mapFile; + private Path journalFile; + private ManualClock clock = new ManualClock(); + private CompactPersistentActionCache cache; + + @Before + public void setUp() throws Exception { + dataRoot = scratch.path("/cache/test.dat"); + cache = new CompactPersistentActionCache(dataRoot, clock); + mapFile = CompactPersistentActionCache.cacheFile(dataRoot); + journalFile = CompactPersistentActionCache.journalFile(dataRoot); + } + + @Test + public void testGetInvalidKey() { + assertNull(cache.get("key")); + } + + @Test + public void testPutAndGet() { + String key = "key"; + putKey(key); + ActionCache.Entry readentry = cache.get(key); + assertTrue(readentry != null); + assertEquals(cache.get(key).toString(), readentry.toString()); + assertFalse(mapFile.exists()); + } + + @Test + public void testPutAndRemove() { + String key = "key"; + putKey(key); + cache.remove(key); + assertNull(cache.get(key)); + assertFalse(mapFile.exists()); + } + + @Test + public void testSave() throws IOException { + String key = "key"; + putKey(key); + cache.save(); + assertTrue(mapFile.exists()); + assertFalse(journalFile.exists()); + + CompactPersistentActionCache newcache = + new CompactPersistentActionCache(dataRoot, clock); + ActionCache.Entry readentry = newcache.get(key); + assertTrue(readentry != null); + assertEquals(cache.get(key).toString(), readentry.toString()); + } + + @Test + public void testIncrementalSave() throws IOException { + for (int i = 0; i < 300; i++) { + putKey(Integer.toString(i)); + } + assertFullSave(); + + // Add 2 entries to 300. Might as well just leave them in the journal. + putKey("abc"); + putKey("123"); + assertIncrementalSave(cache); + + // Make sure we have all the entries, including those in the journal, + // after deserializing into a new cache. + CompactPersistentActionCache newcache = + new CompactPersistentActionCache(dataRoot, clock); + for (int i = 0; i < 100; i++) { + assertKeyEquals(cache, newcache, Integer.toString(i)); + } + assertKeyEquals(cache, newcache, "abc"); + assertKeyEquals(cache, newcache, "123"); + putKey("xyz", newcache); + assertIncrementalSave(newcache); + + // Make sure we can see previous journal values after a second incremental save. + CompactPersistentActionCache newerCache = + new CompactPersistentActionCache(dataRoot, clock); + for (int i = 0; i < 100; i++) { + assertKeyEquals(cache, newerCache, Integer.toString(i)); + } + assertKeyEquals(cache, newerCache, "abc"); + assertKeyEquals(cache, newerCache, "123"); + assertNotNull(newerCache.get("xyz")); + assertNull(newerCache.get("not_a_key")); + + // Add another 10 entries. This should not be incremental. + for (int i = 300; i < 310; i++) { + putKey(Integer.toString(i)); + } + assertFullSave(); + } + + // Regression test to check that CompactActionCacheEntry.toString does not mutate the object. + // Mutations may result in IllegalStateException. + @Test + public void testEntryToStringIsIdempotent() throws Exception { + ActionCache.Entry entry = new ActionCache.Entry("actionKey"); + entry.toString(); + entry.addFile(new PathFragment("foo/bar"), Metadata.CONSTANT_METADATA); + entry.toString(); + entry.getFileDigest(); + entry.toString(); + } + + private static void assertKeyEquals(ActionCache cache1, ActionCache cache2, String key) { + Object entry = cache1.get(key); + assertNotNull(entry); + assertEquals(entry.toString(), cache2.get(key).toString()); + } + + private void assertFullSave() throws IOException { + cache.save(); + assertTrue(mapFile.exists()); + assertFalse(journalFile.exists()); + } + + private void assertIncrementalSave(ActionCache ac) throws IOException { + ac.save(); + assertTrue(mapFile.exists()); + assertTrue(journalFile.exists()); + } + + private void putKey(String key) { + putKey(key, cache); + } + + private void putKey(String key, ActionCache ac) { + ActionCache.Entry entry = ac.createEntry(key); + entry.getFileDigest(); + ac.put(key, entry); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/cache/MetadataTest.java b/src/test/java/com/google/devtools/build/lib/actions/cache/MetadataTest.java new file mode 100644 index 0000000000..fe37af2595 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/cache/MetadataTest.java @@ -0,0 +1,45 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + + +import com.google.common.io.BaseEncoding; +import com.google.common.testing.EqualsTester; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class MetadataTest { + + private static byte[] toBytes(String hex) { + return BaseEncoding.base16().upperCase().decode(hex); + } + + @Test + public void testEqualsAndHashCode() throws Exception { + // Each "equality group" is checked for equality within itself (including hashCode equality) + // and inequality with members of other equality groups. + new EqualsTester() + .addEqualityGroup(new Metadata(toBytes("00112233445566778899AABBCCDDEEFF")), + new Metadata(toBytes("00112233445566778899AABBCCDDEEFF"))) + .addEqualityGroup(new Metadata(1)) + .addEqualityGroup(new Metadata(toBytes("FFFFFF00000000000000000000000000"))) + .addEqualityGroup(new Metadata(2), + new Metadata(2)) + .addEqualityGroup("a string") + .testEquals(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexerTest.java b/src/test/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexerTest.java new file mode 100644 index 0000000000..7fc0cb1327 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/cache/PersistentStringIndexerTest.java @@ -0,0 +1,387 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.cache; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.EOFException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; + +/** + * Test for the PersistentStringIndexer class. + */ +@RunWith(JUnit4.class) +public class PersistentStringIndexerTest { + + private static class ManualClock implements Clock { + private long currentTime = 0L; + + ManualClock() { } + + @Override public long currentTimeMillis() { + throw new AssertionError("unexpected method call"); + } + + @Override public long nanoTime() { + return currentTime; + } + + void advance(long time) { + currentTime += time; + } + } + + private PersistentStringIndexer psi; + private Map<Integer, String> mappings = new ConcurrentHashMap<>(); + private FsApparatus scratch = FsApparatus.newInMemory(); + private ManualClock clock = new ManualClock(); + private Path dataPath; + private Path journalPath; + + + @Before + public void setUp() throws Exception { + dataPath = scratch.path("/cache/test.dat"); + journalPath = scratch.path("/cache/test.journal"); + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + } + + private void assertSize(int expected) { + assertEquals(expected, psi.size()); + } + + private void assertIndex(int expected, String s) { + int index = psi.getOrCreateIndex(s); + assertEquals(expected, index); + mappings.put(expected, s); + } + + private void assertContent() { + for (int i = 0; i < psi.size(); i++) { + if(mappings.get(i) != null) { + assertEquals(mappings.get(i), psi.getStringForIndex(i)); + } + } + } + + + private void setupTestContent() { + assertSize(0); + assertIndex(0, "abcdefghi"); // Create leafs + assertIndex(1, "abcdefjkl"); + assertIndex(2, "abcdefmno"); + assertIndex(3, "abcdefjklpr"); + assertIndex(3, "abcdefjklpr"); + assertIndex(4, "abcdstr"); + assertIndex(5, "012345"); + assertSize(6); + assertIndex(6, "abcdef"); // Validate inner nodes + assertIndex(7, "abcd"); + assertIndex(8, ""); + assertSize(9); + assertContent(); + } + + /** + * Writes lots of entries with labels "fooconcurrent[int]" at the same time. + * The set of labels written is deterministic, but the label:index mapping is + * not. + */ + private void writeLotsOfEntriesConcurrently(final int numToWrite) throws InterruptedException { + final int NUM_THREADS = 10; + final CountDownLatch synchronizerLatch = new CountDownLatch(NUM_THREADS); + + class IndexAdder extends TestThread { + @Override + public void runTest() throws Exception { + for (int i = 0; i < numToWrite; i++) { + synchronizerLatch.countDown(); + synchronizerLatch.await(); + + String value = "fooconcurrent" + i; + mappings.put(psi.getOrCreateIndex(value), value); + } + } + } + + Collection<TestThread> threads = new ArrayList<>(); + for (int i = 0; i < NUM_THREADS; i++) { + TestThread thread = new IndexAdder(); + thread.start(); + threads.add(thread); + } + + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + } + + @Test + public void testNormalOperation() throws Exception { + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + setupTestContent(); + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + + clock.advance(4); + assertIndex(9, "xyzqwerty"); // This should flush journal to disk. + assertFalse(dataPath.exists()); + assertTrue(journalPath.exists()); + + psi.save(); // Successful save will remove journal file. + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + + // Now restore data from file and verify it. + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + assertFalse(journalPath.exists()); + clock.advance(4); + assertSize(10); + assertContent(); + assertFalse(journalPath.exists()); + } + + @Test + public void testJournalRecoveryWithoutMainDataFile() throws Exception { + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + setupTestContent(); + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + + clock.advance(4); + assertIndex(9, "abc1234"); // This should flush journal to disk. + assertFalse(dataPath.exists()); + assertTrue(journalPath.exists()); + + // Now restore data from file and verify it. All data should be restored from journal; + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + clock.advance(4); + assertSize(10); + assertContent(); + assertFalse(journalPath.exists()); + } + + @Test + public void testJournalRecovery() throws Exception { + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + setupTestContent(); + psi.save(); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + long oldDataFileLen = dataPath.getFileSize(); + + clock.advance(4); + assertIndex(9, "another record"); // This should flush journal to disk. + assertSize(10); + assertTrue(dataPath.exists()); + assertTrue(journalPath.exists()); + + // Now restore data from file and verify it. All data should be restored from journal; + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + assertTrue(dataPath.getFileSize() > oldDataFileLen); // data file should have been updated + clock.advance(4); + assertSize(10); + assertContent(); + assertFalse(journalPath.exists()); + } + + @Test + public void testConcurrentWritesJournalRecovery() throws Exception { + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + setupTestContent(); + psi.save(); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + long oldDataFileLen = dataPath.getFileSize(); + + int size = psi.size(); + int numToWrite = 50000; + writeLotsOfEntriesConcurrently(numToWrite); + assertFalse(journalPath.exists()); + clock.advance(4); + assertIndex(size + numToWrite, "another record"); // This should flush journal to disk. + assertSize(size + numToWrite + 1); + assertTrue(dataPath.exists()); + assertTrue(journalPath.exists()); + + // Now restore data from file and verify it. All data should be restored from journal; + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + assertTrue(dataPath.getFileSize() > oldDataFileLen); // data file should have been updated + clock.advance(4); + assertSize(size + numToWrite + 1); + assertContent(); + assertFalse(journalPath.exists()); + } + + @Test + public void testCorruptedJournal() throws Exception { + FileSystemUtils.createDirectoryAndParents(journalPath.getParentDirectory()); + FileSystemUtils.writeContentAsLatin1(journalPath, "bogus content"); + try { + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + fail(); + } catch (IOException e) { + assertThat(e.getMessage()).contains("too short: Only 13 bytes"); + } + + journalPath.delete(); + setupTestContent(); + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + + clock.advance(4); + assertIndex(9, "abc1234"); // This should flush journal to disk. + assertFalse(dataPath.exists()); + assertTrue(journalPath.exists()); + + byte[] journalContent = FileSystemUtils.readContent(journalPath); + + // Now restore data from file and verify it. All data should be restored from journal; + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + + // Now put back truncated journal. We should get an error. + assertTrue(dataPath.delete()); + FileSystemUtils.writeContent(journalPath, + Arrays.copyOf(journalContent, journalContent.length - 1)); + try { + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + fail(); + } catch (EOFException e) { + // Expected. + } + + // Corrupt the journal with a negative size value. + byte[] journalCopy = Arrays.copyOf(journalContent, journalContent.length); + // Flip this bit to make the key size negative. + journalCopy[95] = -2; + FileSystemUtils.writeContent(journalPath, journalCopy); + try { + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + fail(); + } catch (IOException e) { + // Expected. + assertThat(e.getMessage()).contains("corrupt key length"); + } + + // Now put back corrupted journal. We should get an error. + journalContent[journalContent.length - 13] = 100; + FileSystemUtils.writeContent(journalPath, journalContent); + try { + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + fail(); + } catch (IOException e) { + // Expected. + } + } + + @Test + public void testDupeIndexCorruption() throws Exception { + setupTestContent(); + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + + assertIndex(9, "abc1234"); // This should flush journal to disk. + psi.save(); + assertTrue(dataPath.exists()); + assertFalse(journalPath.exists()); + + byte[] content = FileSystemUtils.readContent(dataPath); + + // We remove the data file, and instead create a corrupt journal. + // + // The journal has a header followed by a sequence of (String, int) pairs, where each int is a + // unique value. The String is encoded by the length (as an int), and the int is simply encoded + // as an int. Note that the DataOutputStream class uses big endian by default, so the low-order + // bits are at the end. + // + // For the purpose of this test, we want to make the journal contain two entries with the same + // index (which is illegal). The PersistentStringIndexer assigns int values in the usual order, + // starting with zero, and it now contains 9 entries. We simply change the last entry to an + // index that is guaranteed to already exist. If it is the index 1, we change it to 2, otherwise + // we change it to 1 - in both cases, the code currently guarantees that the duplicate comes + // earlier in the stream. + assertTrue(dataPath.delete()); + content[content.length - 1] = content[content.length - 1] == 1 ? (byte) 2 : (byte) 1; + FileSystemUtils.writeContent(journalPath, content); + + try { + psi = PersistentStringIndexer.newPersistentStringIndexer(dataPath, clock); + fail(); + } catch (IOException e) { + // Expected. + assertThat(e.getMessage()).contains("Corrupted filename index has duplicate entry"); + } + } + + @Test + public void testDeferredIOFailure() throws Exception { + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + setupTestContent(); + assertFalse(dataPath.exists()); + assertFalse(journalPath.exists()); + + // Ensure that journal cannot be saved. + FileSystemUtils.createDirectoryAndParents(journalPath); + + clock.advance(4); + assertIndex(9, "abc1234"); // This should flush journal to disk (and fail at that). + assertFalse(dataPath.exists()); + + // Subsequent updates should succeed even though journaling is disabled at this point. + clock.advance(4); + assertIndex(10, "another record"); + try { + // Save should actually save main data file but then return us deferred IO failure + // from failed journal write. + psi.save(); + fail(); + } catch(IOException e) { + assertThat(e.getMessage()).contains(journalPath.getPathString() + " (Is a directory)"); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java new file mode 100644 index 0000000000..1db024937d --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionCacheTestHelper.java @@ -0,0 +1,42 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.util; + +import com.google.devtools.build.lib.actions.cache.ActionCache; + +import java.io.PrintStream; + +/** + * Utilities for tests that use the action cache. + */ +public class ActionCacheTestHelper { + private ActionCacheTestHelper() {} + + /** A cache which does not remember anything. Causes perpetual rebuilds! */ + public static final ActionCache AMNESIAC_CACHE = + new ActionCache() { + @Override + public void put(String fingerprint, Entry entry) {} + @Override + public Entry get(String fingerprint) { return null; } + @Override + public void remove(String key) {} + @Override + public Entry createEntry(String key) { return new ActionCache.Entry(key); } + @Override + public long save() { return -1; } + @Override + public void dump(PrintStream out) { } + }; +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java new file mode 100644 index 0000000000..2f601d4fde --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/util/ActionsTestUtil.java @@ -0,0 +1,440 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.util; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.AbstractActionOwner; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.MutableActionGraph; +import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.actions.cache.MetadataHandler; +import com.google.devtools.build.lib.exec.SingleBuildFileCache; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A bunch of utilities that are useful for test concerning actions, artifacts, + * etc. + */ +public final class ActionsTestUtil { + + private final ActionGraph actionGraph; + + public ActionsTestUtil(ActionGraph actionGraph) { + this.actionGraph = actionGraph; + } + + private static final Label NULL_LABEL = Label.parseAbsoluteUnchecked("//null/action:owner"); + + public static ActionExecutionContext createContext(Executor executor, FileOutErr fileOutErr, + Path execRoot, MetadataHandler metadataHandler, @Nullable ActionGraph actionGraph) { + return new ActionExecutionContext( + executor, + new SingleBuildFileCache(execRoot.getPathString(), execRoot.getFileSystem()), + metadataHandler, fileOutErr, + actionGraph == null + ? null + : ActionInputHelper.actionGraphMiddlemanExpander(actionGraph)); + } + + /** + * A dummy ActionOwner implementation for use in tests. + */ + public static class NullActionOwner extends AbstractActionOwner { + @Override + public Label getLabel() { + return NULL_LABEL; + } + + @Override + public final String getConfigurationName() { + return "dummy-configuration"; + } + + @Override + public String getConfigurationMnemonic() { + return "dummy-configuration-mnemonic"; + } + + @Override + public final String getConfigurationShortCacheKey() { + return "dummy-configuration"; + } + } + + public static final Artifact DUMMY_ARTIFACT = new Artifact( + new PathFragment("dummy"), + Root.asSourceRoot(new InMemoryFileSystem().getRootDirectory())); + + public static final ActionOwner NULL_ACTION_OWNER = new NullActionOwner(); + + public static final ArtifactOwner NULL_ARTIFACT_OWNER = + new ArtifactOwner() { + @Override + public Label getLabel() { + return NULL_LABEL; + } + }; + + public static class UncheckedActionConflictException extends RuntimeException { + public UncheckedActionConflictException(ActionConflictException e) { + super(e); + } + } + + /** + * A dummy Action class for use in tests. + */ + public static class NullAction extends AbstractAction { + + public NullAction() { + super(NULL_ACTION_OWNER, Artifact.NO_ARTIFACTS, ImmutableList.of(DUMMY_ARTIFACT)); + } + + public NullAction(ActionOwner owner, Artifact... outputs) { + super(owner, Artifact.NO_ARTIFACTS, ImmutableList.copyOf(outputs)); + } + + public NullAction(Artifact... outputs) { + super(NULL_ACTION_OWNER, Artifact.NO_ARTIFACTS, ImmutableList.copyOf(outputs)); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) { + } + + @Override protected String computeKey() { return "action"; } + @Override public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + @Override + public String getMnemonic() { + return "Null"; + } + } + + /** + * For a bunch of actions, gets the basenames of the paths and accumulates + * them in a space separated string, like <code>foo.o bar.o baz.a</code>. + */ + public static String baseNamesOf(Iterable<Artifact> artifacts) { + List<String> baseNames = baseArtifactNames(artifacts); + return Joiner.on(' ').join(baseNames); + } + + /** + * For a bunch of actions, gets the basenames of the paths, sorts them in alphabetical + * order and accumulates them in a space separated string, for example + * <code>bar.o baz.a foo.o</code>. + */ + public static String sortedBaseNamesOf(Iterable<Artifact> artifacts) { + List<String> baseNames = baseArtifactNames(artifacts); + Collections.sort(baseNames); + return Joiner.on(' ').join(baseNames); + } + + /** + * For a bunch of artifacts, gets the basenames and accumulates them in a + * List. + */ + public static List<String> baseArtifactNames(Iterable<Artifact> artifacts) { + List<String> baseNames = new ArrayList<>(); + for (Artifact artifact : artifacts) { + baseNames.add(artifact.getExecPath().getBaseName()); + } + return baseNames; + } + + /** + * For a bunch of artifacts, gets the exec paths and accumulates them in a + * List. + */ + public static List<String> execPaths(Iterable<Artifact> artifacts) { + List<String> names = new ArrayList<>(); + for (Artifact artifact : artifacts) { + names.add(artifact.getExecPathString()); + } + return names; + } + + /** + * For a bunch of artifacts, gets the pretty printed names and accumulates them in a List. Note + * that this returns the root-relative paths, not the exec paths. + */ + public static List<String> prettyArtifactNames(Iterable<Artifact> artifacts) { + List<String> result = new ArrayList<>(); + for (Artifact artifact : artifacts) { + result.add(artifact.prettyPrint()); + } + return result; + } + + public static List<String> prettyJarNames(Iterable<Artifact> jars) { + List<String> result = new ArrayList<>(); + for (Artifact jar : jars) { + result.add(jar.prettyPrint()); + } + return result; + } + + /** + * Returns the closure of the predecessors of any of the given types, joining the basenames of the + * artifacts into a space-separated string like "libfoo.a libbar.a libbaz.a". + */ + public String predecessorClosureOf(Artifact artifact, FileType... types) { + return predecessorClosureOf(Collections.singleton(artifact), types); + } + + /** + * Returns the closure of the predecessors of any of the given types. + */ + public Collection<String> predecessorClosureAsCollection(Artifact artifact, FileType... types) { + return predecessorClosureAsCollection(Collections.singleton(artifact), types); + } + + /** + * Returns the closure of the predecessors of any of the given types, joining the basenames of the + * artifacts into a space-separated string like "libfoo.a libbar.a libbaz.a". + */ + public String predecessorClosureOf(Iterable<Artifact> artifacts, FileType... types) { + Set<Artifact> visited = artifactClosureOf(artifacts); + return baseNamesOf(FileType.filter(visited, types)); + } + + /** + * Returns the closure of the predecessors of any of the given types. + */ + public Collection<String> predecessorClosureAsCollection(Iterable<Artifact> artifacts, + FileType... types) { + return baseArtifactNames(FileType.filter(artifactClosureOf(artifacts), types)); + } + + public String predecessorClosureOfJars(Iterable<Artifact> artifacts, FileType... types) { + return baseNamesOf(FileType.filter(artifactClosureOf(artifacts), types)); + } + + public Collection<String> predecessorClosureJarsAsCollection(Iterable<Artifact> artifacts, + FileType... types) { + Set<Artifact> visited = artifactClosureOf(artifacts); + return baseArtifactNames(FileType.filter(visited, types)); + } + + /** + * Returns the closure over the input files of an action. + */ + public Set<Artifact> inputClosureOf(Action action) { + return artifactClosureOf(action.getInputs()); + } + + /** + * Returns the closure over the input files of an artifact. + */ + public Set<Artifact> artifactClosureOf(Artifact artifact) { + return artifactClosureOf(Collections.singleton(artifact)); + } + + /** + * Returns the closure over the input files of an artifact, filtered by the given matcher. + */ + public Set<Artifact> filteredArtifactClosureOf(Artifact artifact, Predicate<Artifact> matcher) { + return ImmutableSet.copyOf(Iterables.filter(artifactClosureOf(artifact), matcher)); + } + + /** + * Returns the closure over the input files of a set of artifacts. + */ + public Set<Artifact> artifactClosureOf(Iterable<Artifact> artifacts) { + Set<Artifact> visited = new LinkedHashSet<>(); + List<Artifact> toVisit = Lists.newArrayList(artifacts); + while (!toVisit.isEmpty()) { + Artifact current = toVisit.remove(0); + if (!visited.add(current)) { + continue; + } + Action generatingAction = actionGraph.getGeneratingAction(current); + if (generatingAction != null) { + Iterables.addAll(toVisit, generatingAction.getInputs()); + } + } + return visited; + } + + /** + * Returns the closure over the input files of a set of artifacts, filtered by the given matcher. + */ + public Set<Artifact> filteredArtifactClosureOf(Iterable<Artifact> artifacts, + Predicate<Artifact> matcher) { + return ImmutableSet.copyOf(Iterables.filter(artifactClosureOf(artifacts), matcher)); + } + + /** + * Returns a predicate to match {@link Artifact}s with the given root-relative path suffix. + */ + public static Predicate<Artifact> getArtifactSuffixMatcher(final String suffix) { + return new Predicate<Artifact>() { + @Override + public boolean apply(Artifact input) { + return input.getRootRelativePath().getPathString().endsWith(suffix); + } + }; + } + + /** + * Finds all the actions that are instances of <code>actionClass</code> + * in the transitive closure of prerequisites. + */ + public <A extends Action> List<A> findTransitivePrerequisitesOf(Artifact artifact, + Class<A> actionClass, Predicate<Artifact> allowedArtifacts) { + List<A> actions = new ArrayList<>(); + Set<Artifact> visited = new LinkedHashSet<>(); + List<Artifact> toVisit = new LinkedList<>(); + toVisit.add(artifact); + while (!toVisit.isEmpty()) { + Artifact current = toVisit.remove(0); + if (!visited.add(current)) { + continue; + } + Action generatingAction = actionGraph.getGeneratingAction(current); + if (generatingAction != null) { + Iterables.addAll(toVisit, Iterables.filter(generatingAction.getInputs(), allowedArtifacts)); + if (actionClass.isInstance(generatingAction)) { + actions.add(actionClass.cast(generatingAction)); + } + } + } + return actions; + } + + public <A extends Action> List<A> findTransitivePrerequisitesOf( + Artifact artifact, Class<A> actionClass) { + return findTransitivePrerequisitesOf(artifact, actionClass, Predicates.<Artifact>alwaysTrue()); + } + + /** + * Looks in the given artifacts Iterable for the first Artifact whose path ends with the given + * suffix and returns its generating Action. + */ + public Action getActionForArtifactEndingWith(Iterable<Artifact> artifacts, String suffix) { + Artifact a = getFirstArtifactEndingWith(artifacts, suffix); + return a != null ? actionGraph.getGeneratingAction(a) : null; + } + + /** + * Looks in the given artifacts Iterable for the first Artifact whose path ends with the given + * suffix and returns the Artifact. + */ + public static Artifact getFirstArtifactEndingWith( + Iterable<Artifact> artifacts, String suffix) { + for (Artifact a : artifacts) { + if (a.getExecPath().getPathString().endsWith(suffix)) { + return a; + } + } + return null; + } + + /** + * Returns the first artifact which is an input to "action" and has the + * specified basename. An assertion error is raised if none is found. + */ + public static Artifact getInput(Action action, String basename) { + for (Artifact artifact : action.getInputs()) { + if (artifact.getExecPath().getBaseName().equals(basename)) { + return artifact; + } + } + throw new AssertionError("No input with basename '" + basename + "' in action " + action); + } + + /** + * Returns true if an artifact that is an input to "action" with the specific + * basename exists. + */ + public static boolean hasInput(Action action, String basename) { + try { + getInput(action, basename); + return true; + } catch (AssertionError e) { + return false; + } + } + + /** + * Assert that an artifact is the primary output of its generating action. + */ + public void assertPrimaryInputAndOutputArtifacts(Artifact input, Artifact output) { + Action generatingAction = actionGraph.getGeneratingAction(output); + assertThat(generatingAction).isNotNull(); + assertThat(generatingAction.getPrimaryOutput()).isEqualTo(output); + assertThat(generatingAction.getPrimaryInput()).isEqualTo(input); + } + + /** + * Returns the first artifact which is an output of "action" and has the + * specified basename. An assertion error is raised if none is found. + */ + public static Artifact getOutput(Action action, String basename) { + for (Artifact artifact : action.getOutputs()) { + if (artifact.getExecPath().getBaseName().equals(basename)) { + return artifact; + } + } + throw new AssertionError("No output with basename '" + basename + "' in action " + action); + } + + public static void registerActionWith(Action action, MutableActionGraph actionGraph) { + try { + actionGraph.registerAction(action); + } catch (ActionConflictException e) { + throw new UncheckedActionConflictException(e); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/DummyExecutor.java b/src/test/java/com/google/devtools/build/lib/actions/util/DummyExecutor.java new file mode 100644 index 0000000000..409227ddc8 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/util/DummyExecutor.java @@ -0,0 +1,86 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.util; + +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsClassProvider; + +/** + * A dummy implementation of Executor. + */ +public final class DummyExecutor implements Executor { + private final Path inputDir; + + /** + * @param inputDir + */ + public DummyExecutor(Path inputDir) { + this.inputDir = inputDir; + } + + @Override + public Path getExecRoot() { + return inputDir; + } + + @Override + public Clock getClock() { + throw new UnsupportedOperationException(); + } + + @Override + public EventBus getEventBus() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getVerboseFailures() { + throw new UnsupportedOperationException(); + } + + @Override + public EventHandler getEventHandler() { + throw new UnsupportedOperationException(); + } + + @Override + public <T extends ActionContext> T getContext(Class<? extends T> type) { + throw new UnsupportedOperationException(); + } + + @Override + public SpawnActionContext getSpawnActionContext(String mnemonic) { + throw new UnsupportedOperationException(); + } + + @Override + public OptionsClassProvider getOptions() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean reportsSubcommands() { + throw new UnsupportedOperationException(); + } + + @Override + public void reportSubcommand(String reason, String message) { + throw new UnsupportedOperationException(); + } +}
\ No newline at end of file diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/LabelArtifactOwner.java b/src/test/java/com/google/devtools/build/lib/actions/util/LabelArtifactOwner.java new file mode 100644 index 0000000000..8e200e7370 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/util/LabelArtifactOwner.java @@ -0,0 +1,49 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.syntax.Label; + +import java.util.Objects; + +/** ArtifactOwner wrapper for Labels, for use in tests. */ +@VisibleForTesting +public class LabelArtifactOwner implements ArtifactOwner { + private final Label label; + + @VisibleForTesting + public LabelArtifactOwner(Label label) { + this.label = label; + } + + @Override + public Label getLabel() { + return label; + } + + @Override + public int hashCode() { + return label == null ? super.hashCode() : label.hashCode(); + } + + @Override + public boolean equals(Object that) { + if (!(that instanceof LabelArtifactOwner)) { + return false; + } + return Objects.equals(this.label, ((LabelArtifactOwner) that).label); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/actions/util/TestAction.java b/src/test/java/com/google/devtools/build/lib/actions/util/TestAction.java new file mode 100644 index 0000000000..086138440e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/actions/util/TestAction.java @@ -0,0 +1,176 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions.util; + +import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.NULL_ACTION_OWNER; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; + +/** + * A dummy action for testing. Its execution runs the specified + * Runnable or Callable, which is defined by the test case, + * and touches all the output files. + */ +public class TestAction extends AbstractAction { + + public static final Runnable NO_EFFECT = new Runnable() { @Override public void run() {} }; + + private static final ResourceSet RESOURCES = + new ResourceSet(/*memoryMb=*/1.0, /*cpu=*/0.1, /*io=*/0.0); + + private final Callable<Void> effect; + + /** Use this constructor if the effect can't throw exceptions. */ + public TestAction(Runnable effect, + Collection<Artifact> inputs, + Collection<Artifact> outputs) { + super(NULL_ACTION_OWNER, inputs, outputs); + this.effect = Executors.callable(effect, null); + } + + /** + * Use this constructor if the effect can throw exceptions. + * Any checked exception thrown will be repackaged as an + * ActionExecutionException. + */ + public TestAction(Callable<Void> effect, + Collection<Artifact> inputs, + Collection<Artifact> outputs) { + super(NULL_ACTION_OWNER, inputs, outputs); + this.effect = effect; + } + + @Override + public Collection<Artifact> getMandatoryInputs() { + List<Artifact> mandatoryInputs = new ArrayList<>(); + for (Artifact input : getInputs()) { + if (!input.getExecPath().getBaseName().endsWith(".optional")) { + mandatoryInputs.add(input); + } + } + return mandatoryInputs; + } + + @Override + public boolean discoversInputs() { + for (Artifact input : getInputs()) { + if (!input.getExecPath().getBaseName().endsWith(".optional")) { + return true; + } + } + return false; + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) { + Preconditions.checkState(discoversInputs(), this); + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException { + for (Artifact artifact : getInputs()) { + // Do not check *.optional artifacts - artifacts with such extension are + // used by tests to specify artifacts that may or may not be missing. + // This is used, e.g., to test Blaze behavior when action has missing + // input artifacts but still is successfully executed. + if (!artifact.getPath().exists() && + !artifact.getExecPath().getBaseName().endsWith(".optional")) { + throw new IllegalStateException("action's input file does not exist: " + + artifact.getPath()); + } + } + + try { + effect.call(); + } catch (RuntimeException | Error e) { + throw e; + } catch (Exception e) { + throw new ActionExecutionException("TestAction failed due to exception", + e, this, false); + } + + try { + for (Artifact artifact: getOutputs()) { + FileSystemUtils.touchFile(artifact.getPath()); + } + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + protected String computeKey() { + List<String> outputsList = new ArrayList<>(); + for (Artifact output : getOutputs()) { + outputsList.add(output.getPath().getPathString()); + } + // This could use a functional iterable and avoid creating a list + return "test " + StringUtilities.combineKeys(outputsList); + } + + @Override + public String getMnemonic() { return "Test"; } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return RESOURCES; + } + + + /** No-op action that has exactly one output, and can be a middleman action. */ + public static class DummyAction extends TestAction { + private static final Runnable NOOP = new Runnable() { + @Override + public void run() {} + }; + + private final MiddlemanType type; + + public DummyAction(Collection<Artifact> inputs, Artifact output, MiddlemanType type) { + super(NOOP, inputs, ImmutableList.of(output)); + this.type = type; + } + + public DummyAction(Collection<Artifact> inputs, Artifact output) { + this(inputs, output, MiddlemanType.NORMAL); + } + + @Override + public MiddlemanType getActionType() { + return type; + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/CollectionUtilsTest.java b/src/test/java/com/google/devtools/build/lib/collect/CollectionUtilsTest.java new file mode 100644 index 0000000000..2505501371 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/CollectionUtilsTest.java @@ -0,0 +1,175 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Tests for {@link CollectionUtils}. + */ + +@RunWith(JUnit4.class) +public class CollectionUtilsTest { + + @Test + public void testDuplicatedElementsOf() { + assertDups(ImmutableList.<Integer>of(), ImmutableSet.<Integer>of()); + assertDups(ImmutableList.of(0), ImmutableSet.<Integer>of()); + assertDups(ImmutableList.of(0, 0, 0), ImmutableSet.of(0)); + assertDups(ImmutableList.of(1, 2, 3, 1, 2, 3), ImmutableSet.of(1, 2, 3)); + assertDups(ImmutableList.of(1, 2, 3, 1, 2, 3, 4), ImmutableSet.of(1, 2, 3)); + assertDups(ImmutableList.of(1, 2, 3, 4), ImmutableSet.<Integer>of()); + } + + private static void assertDups(List<Integer> collection, Set<Integer> dups) { + assertEquals(dups, CollectionUtils.duplicatedElementsOf(collection)); + } + + @Test + public void testIsImmutable() throws Exception { + assertTrue(CollectionUtils.isImmutable(ImmutableList.of(1, 2, 3))); + assertTrue(CollectionUtils.isImmutable(ImmutableSet.of(1, 2, 3))); + + NestedSet<Integer> ns = NestedSetBuilder.<Integer>compileOrder() + .add(1).add(2).add(3).build(); + assertTrue(CollectionUtils.isImmutable(ns)); + + NestedSet<Integer> ns2 = NestedSetBuilder.<Integer>linkOrder().add(1).add(2).add(3).build(); + assertTrue(CollectionUtils.isImmutable(ns2)); + + IterablesChain<Integer> chain = IterablesChain.<Integer>builder().addElement(1).build(); + + assertTrue(CollectionUtils.isImmutable(chain)); + + assertFalse(CollectionUtils.isImmutable(Lists.newArrayList())); + assertFalse(CollectionUtils.isImmutable(Lists.newLinkedList())); + assertFalse(CollectionUtils.isImmutable(Sets.newHashSet())); + assertFalse(CollectionUtils.isImmutable(Sets.newLinkedHashSet())); + + // The result of Iterables.concat() actually is immutable, but we have no way of checking if + // a given Iterable comes from concat(). + assertFalse(CollectionUtils.isImmutable(Iterables.concat(ns, ns2))); + + // We can override the check by using the ImmutableIterable wrapper. + assertTrue(CollectionUtils.isImmutable( + ImmutableIterable.from(Iterables.concat(ns, ns2)))); + } + + @Test + public void testCheckImmutable() throws Exception { + CollectionUtils.checkImmutable(ImmutableList.of(1, 2, 3)); + CollectionUtils.checkImmutable(ImmutableSet.of(1, 2, 3)); + + try { + CollectionUtils.checkImmutable(Lists.newArrayList(1, 2, 3)); + } catch (IllegalStateException e) { + return; + } + fail(); + } + + @Test + public void testMakeImmutable() throws Exception { + Iterable<Integer> immutableList = ImmutableList.of(1, 2, 3); + assertSame(immutableList, CollectionUtils.makeImmutable(immutableList)); + + Iterable<Integer> mutableList = Lists.newArrayList(1, 2, 3); + Iterable<Integer> converted = CollectionUtils.makeImmutable(mutableList); + assertNotSame(mutableList, converted); + assertEquals(mutableList, ImmutableList.copyOf(converted)); + } + + private static enum Small { ALPHA, BRAVO } + private static enum Large { + L0, L1, L2, L3, L4, L5, L6, L7, L8, L9, + L10, L11, L12, L13, L14, L15, L16, L17, L18, L19, + L20, L21, L22, L23, L24, L25, L26, L27, L28, L29, + L30, L31, + } + + private static enum TooLarge { + T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, + T10, T11, T12, T13, T14, T15, T16, T17, T18, T19, + T20, T21, T22, T23, T24, T25, T26, T27, T28, T29, + T30, T31, T32, + } + + private static enum Medium { + ONE, TWO, THREE, FOUR, FIVE, SIX, SEVEN, EIGHT, + } + + private <T extends Enum<T>> void assertAllDifferent(Class<T> clazz) throws Exception { + Set<EnumSet<T>> allSets = new HashSet<>(); + + int maxBits = 1 << clazz.getEnumConstants().length; + for (int i = 0; i < maxBits; i++) { + EnumSet<T> set = CollectionUtils.fromBits(i, clazz); + int back = CollectionUtils.toBits(set); + assertEquals(back, i); // Assert that a roundtrip is idempotent + allSets.add(set); + } + + assertEquals(maxBits, allSets.size()); // Assert that every decoded value is different + } + + @Test + public void testEnumBitfields() throws Exception { + assertEquals(0, CollectionUtils.<Small>toBits()); + assertEquals(EnumSet.noneOf(Small.class), CollectionUtils.fromBits(0, Small.class)); + assertEquals(3, CollectionUtils.toBits(Small.ALPHA, Small.BRAVO)); + assertEquals(10, CollectionUtils.toBits(Medium.TWO, Medium.FOUR)); + assertEquals(EnumSet.of(Medium.SEVEN, Medium.EIGHT), + CollectionUtils.fromBits(192, Medium.class)); + + assertAllDifferent(Small.class); + assertAllDifferent(Medium.class); + assertAllDifferent(Large.class); + + try { + CollectionUtils.toBits(TooLarge.T32); + fail(); + } catch (IllegalArgumentException e) { + // good + } + + try { + CollectionUtils.fromBits(0, TooLarge.class); + fail(); + } catch (IllegalArgumentException e) { + // good + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimapTest.java b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimapTest.java new file mode 100644 index 0000000000..1249f6db14 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyListMultimapTest.java @@ -0,0 +1,352 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.HashMultiset; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.testing.google.UnmodifiableCollectionTests; +import com.google.common.testing.EqualsTester; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * A test for {@link ImmutableSortedKeyListMultimap}. Started out as a copy of + * ImmutableListMultimapTest. + */ +@RunWith(JUnit4.class) +public class ImmutableSortedKeyListMultimapTest { + + @Test + public void builderPutAllIterable() { + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll("foo", Arrays.asList(1, 2, 3)); + builder.putAll("bar", Arrays.asList(4, 5)); + builder.putAll("foo", Arrays.asList(6, 7)); + Multimap<String, Integer> multimap = builder.build(); + assertEquals(Arrays.asList(1, 2, 3, 6, 7), multimap.get("foo")); + assertEquals(Arrays.asList(4, 5), multimap.get("bar")); + assertEquals(7, multimap.size()); + } + + @Test + public void builderPutAllVarargs() { + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll("foo", 1, 2, 3); + builder.putAll("bar", 4, 5); + builder.putAll("foo", 6, 7); + Multimap<String, Integer> multimap = builder.build(); + assertEquals(Arrays.asList(1, 2, 3, 6, 7), multimap.get("foo")); + assertEquals(Arrays.asList(4, 5), multimap.get("bar")); + assertEquals(7, multimap.size()); + } + + @Test + public void builderPutAllMultimap() { + Multimap<String, Integer> toPut = LinkedListMultimap.create(); + toPut.put("foo", 1); + toPut.put("bar", 4); + toPut.put("foo", 2); + toPut.put("foo", 3); + Multimap<String, Integer> moreToPut = LinkedListMultimap.create(); + moreToPut.put("foo", 6); + moreToPut.put("bar", 5); + moreToPut.put("foo", 7); + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll(toPut); + builder.putAll(moreToPut); + Multimap<String, Integer> multimap = builder.build(); + assertEquals(Arrays.asList(1, 2, 3, 6, 7), multimap.get("foo")); + assertEquals(Arrays.asList(4, 5), multimap.get("bar")); + assertEquals(7, multimap.size()); + } + + @Test + public void builderPutAllWithDuplicates() { + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll("foo", 1, 2, 3); + builder.putAll("bar", 4, 5); + builder.putAll("foo", 1, 6, 7); + ImmutableSortedKeyListMultimap<String, Integer> multimap = builder.build(); + assertEquals(Arrays.asList(1, 2, 3, 1, 6, 7), multimap.get("foo")); + assertEquals(Arrays.asList(4, 5), multimap.get("bar")); + assertEquals(8, multimap.size()); + } + + @Test + public void builderPutWithDuplicates() { + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll("foo", 1, 2, 3); + builder.putAll("bar", 4, 5); + builder.put("foo", 1); + ImmutableSortedKeyListMultimap<String, Integer> multimap = builder.build(); + assertEquals(Arrays.asList(1, 2, 3, 1), multimap.get("foo")); + assertEquals(Arrays.asList(4, 5), multimap.get("bar")); + assertEquals(6, multimap.size()); + } + + @Test + public void builderPutAllMultimapWithDuplicates() { + Multimap<String, Integer> toPut = LinkedListMultimap.create(); + toPut.put("foo", 1); + toPut.put("bar", 4); + toPut.put("foo", 2); + toPut.put("foo", 1); + toPut.put("bar", 5); + Multimap<String, Integer> moreToPut = LinkedListMultimap.create(); + moreToPut.put("foo", 6); + moreToPut.put("bar", 4); + moreToPut.put("foo", 7); + moreToPut.put("foo", 2); + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll(toPut); + builder.putAll(moreToPut); + Multimap<String, Integer> multimap = builder.build(); + assertEquals(Arrays.asList(1, 2, 1, 6, 7, 2), multimap.get("foo")); + assertEquals(Arrays.asList(4, 5, 4), multimap.get("bar")); + assertEquals(9, multimap.size()); + } + + @Test + public void builderPutNullKey() { + Multimap<String, Integer> toPut = LinkedListMultimap.create(); + toPut.put("foo", null); + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + try { + builder.put(null, 1); + fail(); + } catch (NullPointerException expected) {} + try { + builder.putAll(null, Arrays.asList(1, 2, 3)); + fail(); + } catch (NullPointerException expected) {} + try { + builder.putAll(null, 1, 2, 3); + fail(); + } catch (NullPointerException expected) {} + try { + builder.putAll(toPut); + fail(); + } catch (NullPointerException expected) {} + } + + @Test + public void builderPutNullValue() { + Multimap<String, Integer> toPut = LinkedListMultimap.create(); + toPut.put(null, 1); + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + try { + builder.put("foo", null); + fail(); + } catch (NullPointerException expected) {} + try { + builder.putAll("foo", Arrays.asList(1, null, 3)); + fail(); + } catch (NullPointerException expected) {} + try { + builder.putAll("foo", 1, null, 3); + fail(); + } catch (NullPointerException expected) {} + try { + builder.putAll(toPut); + fail(); + } catch (NullPointerException expected) {} + } + + @Test + public void copyOf() { + ArrayListMultimap<String, Integer> input = ArrayListMultimap.create(); + input.put("foo", 1); + input.put("bar", 2); + input.put("foo", 3); + Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.copyOf(input); + assertEquals(multimap, input); + assertEquals(input, multimap); + } + + @Test + public void copyOfWithDuplicates() { + ArrayListMultimap<String, Integer> input = ArrayListMultimap.create(); + input.put("foo", 1); + input.put("bar", 2); + input.put("foo", 3); + input.put("foo", 1); + Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.copyOf(input); + assertEquals(multimap, input); + assertEquals(input, multimap); + } + + @Test + public void copyOfEmpty() { + ArrayListMultimap<String, Integer> input = ArrayListMultimap.create(); + Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.copyOf(input); + assertEquals(multimap, input); + assertEquals(input, multimap); + } + + @Test + public void copyOfImmutableListMultimap() { + Multimap<String, Integer> multimap = createMultimap(); + assertSame(multimap, ImmutableSortedKeyListMultimap.copyOf(multimap)); + } + + @Test + public void copyOfNullKey() { + ArrayListMultimap<String, Integer> input = ArrayListMultimap.create(); + input.put(null, 1); + try { + ImmutableSortedKeyListMultimap.copyOf(input); + fail(); + } catch (NullPointerException expected) {} + } + + @Test + public void copyOfNullValue() { + ArrayListMultimap<String, Integer> input = ArrayListMultimap.create(); + input.putAll("foo", Arrays.asList(1, null, 3)); + try { + ImmutableSortedKeyListMultimap.copyOf(input); + fail(); + } catch (NullPointerException expected) {} + } + + @Test + public void emptyMultimapReads() { + Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.of(); + assertFalse(multimap.containsKey("foo")); + assertFalse(multimap.containsValue(1)); + assertFalse(multimap.containsEntry("foo", 1)); + assertTrue(multimap.entries().isEmpty()); + assertTrue(multimap.equals(ArrayListMultimap.create())); + assertEquals(Collections.emptyList(), multimap.get("foo")); + assertEquals(0, multimap.hashCode()); + assertTrue(multimap.isEmpty()); + assertEquals(HashMultiset.create(), multimap.keys()); + assertEquals(Collections.emptySet(), multimap.keySet()); + assertEquals(0, multimap.size()); + assertTrue(multimap.values().isEmpty()); + assertEquals("{}", multimap.toString()); + } + + @Test + public void emptyMultimapWrites() { + Multimap<String, Integer> multimap = ImmutableSortedKeyListMultimap.of(); + UnmodifiableCollectionTests.assertMultimapIsUnmodifiable( + multimap, "foo", 1); + } + + private Multimap<String, Integer> createMultimap() { + return ImmutableSortedKeyListMultimap.<String, Integer>builder() + .put("foo", 1).put("bar", 2).put("foo", 3).build(); + } + + @Test + public void multimapReads() { + Multimap<String, Integer> multimap = createMultimap(); + assertTrue(multimap.containsKey("foo")); + assertFalse(multimap.containsKey("cat")); + assertTrue(multimap.containsValue(1)); + assertFalse(multimap.containsValue(5)); + assertTrue(multimap.containsEntry("foo", 1)); + assertFalse(multimap.containsEntry("cat", 1)); + assertFalse(multimap.containsEntry("foo", 5)); + assertFalse(multimap.entries().isEmpty()); + assertEquals(3, multimap.size()); + assertFalse(multimap.isEmpty()); + assertEquals("{bar=[2], foo=[1, 3]}", multimap.toString()); + } + + @Test + public void multimapWrites() { + Multimap<String, Integer> multimap = createMultimap(); + UnmodifiableCollectionTests.assertMultimapIsUnmodifiable( + multimap, "bar", 2); + } + + @Test + public void multimapEquals() { + Multimap<String, Integer> multimap = createMultimap(); + Multimap<String, Integer> arrayListMultimap + = ArrayListMultimap.create(); + arrayListMultimap.putAll("foo", Arrays.asList(1, 3)); + arrayListMultimap.put("bar", 2); + + new EqualsTester() + .addEqualityGroup(multimap, createMultimap(), arrayListMultimap, + ImmutableSortedKeyListMultimap.<String, Integer>builder() + .put("bar", 2).put("foo", 1).put("foo", 3).build()) + .addEqualityGroup(ImmutableSortedKeyListMultimap.<String, Integer>builder() + .put("bar", 2).put("foo", 3).put("foo", 1).build()) + .addEqualityGroup(ImmutableSortedKeyListMultimap.<String, Integer>builder() + .put("foo", 2).put("foo", 3).put("foo", 1).build()) + .addEqualityGroup(ImmutableSortedKeyListMultimap.<String, Integer>builder() + .put("bar", 2).put("foo", 3).build()) + .testEquals(); + } + + @Test + public void asMap() { + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll("foo", Arrays.asList(1, 2, 3)); + builder.putAll("bar", Arrays.asList(4, 5)); + Map<String, Collection<Integer>> map = builder.build().asMap(); + assertEquals(Arrays.asList(1, 2, 3), map.get("foo")); + assertEquals(Arrays.asList(4, 5), map.get("bar")); + assertEquals(2, map.size()); + assertTrue(map.containsKey("foo")); + assertTrue(map.containsKey("bar")); + assertFalse(map.containsKey("notfoo")); + } + + @Test + public void asMapEntries() { + ImmutableSortedKeyListMultimap.Builder<String, Integer> builder + = ImmutableSortedKeyListMultimap.builder(); + builder.putAll("foo", Arrays.asList(1, 2, 3)); + builder.putAll("bar", Arrays.asList(4, 5)); + Set<Map.Entry<String, Collection<Integer>>> set = builder.build().asMap().entrySet(); + Set<Map.Entry<String, Collection<Integer>>> other = + ImmutableSet.<Map.Entry<String, Collection<Integer>>>builder() + .add(new SimpleImmutableEntry<String, Collection<Integer>>("foo", Arrays.asList(1, 2, 3))) + .add(new SimpleImmutableEntry<String, Collection<Integer>>("bar", Arrays.asList(4, 5))) + .build(); + assertEquals(other, set); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMapTest.java b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMapTest.java new file mode 100644 index 0000000000..c712695308 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/ImmutableSortedKeyMapTest.java @@ -0,0 +1,288 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Maps; +import com.google.common.testing.NullPointerTester; +import com.google.devtools.build.lib.collect.ImmutableSortedKeyMap.Builder; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Map.Entry; + +/** + * A test for {@link ImmutableSortedKeyListMultimap}. Started out as a blatant copy of + * ImmutableListMapTest. + */ +@RunWith(JUnit4.class) +public class ImmutableSortedKeyMapTest { + + @Test + public void emptyBuilder() { + ImmutableSortedKeyMap<String, Integer> map + = ImmutableSortedKeyMap.<String, Integer>builder().build(); + assertEquals(Collections.<String, Integer>emptyMap(), map); + } + + @Test + public void singletonBuilder() { + ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder() + .put("one", 1) + .build(); + assertMapEquals(map, "one", 1); + } + + @Test + public void builder() { + ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder() + .put("one", 1) + .put("two", 2) + .put("three", 3) + .put("four", 4) + .put("five", 5) + .build(); + assertMapEquals(map, + "five", 5, "four", 4, "one", 1, "three", 3, "two", 2); + } + + @Test + public void builderPutAllWithEmptyMap() { + ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder() + .putAll(Collections.<String, Integer>emptyMap()) + .build(); + assertEquals(Collections.<String, Integer>emptyMap(), map); + } + + @Test + public void builderPutAll() { + Map<String, Integer> toPut = new LinkedHashMap<>(); + toPut.put("one", 1); + toPut.put("two", 2); + toPut.put("three", 3); + Map<String, Integer> moreToPut = new LinkedHashMap<>(); + moreToPut.put("four", 4); + moreToPut.put("five", 5); + + ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.<String, Integer>builder() + .putAll(toPut) + .putAll(moreToPut) + .build(); + assertMapEquals(map, + "five", 5, "four", 4, "one", 1, "three", 3, "two", 2); + } + + @Test + public void builderReuse() { + ImmutableSortedKeyMap.Builder<String, Integer> builder = + ImmutableSortedKeyMap.<String, Integer>builder(); + ImmutableSortedKeyMap<String, Integer> mapOne = builder + .put("one", 1) + .put("two", 2) + .build(); + ImmutableSortedKeyMap<String, Integer> mapTwo = builder + .put("three", 3) + .put("four", 4) + .build(); + + assertMapEquals(mapOne, "one", 1, "two", 2); + assertMapEquals(mapTwo, "four", 4, "one", 1, "three", 3, "two", 2); + } + + @Test + public void builderPutNullKey() { + Builder<String, Integer> builder = new Builder<>(); + try { + builder.put(null, 1); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void builderPutNullValue() { + Builder<String, Integer> builder = new Builder<>(); + try { + builder.put("one", null); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void builderPutNullKeyViaPutAll() { + Builder<String, Integer> builder = new Builder<>(); + try { + builder.putAll(Collections.<String, Integer>singletonMap(null, 1)); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void builderPutNullValueViaPutAll() { + Builder<String, Integer> builder = new Builder<>(); + try { + builder.putAll(Collections.<String, Integer>singletonMap("one", null)); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void of() { + assertMapEquals( + ImmutableSortedKeyMap.of("one", 1), + "one", 1); + assertMapEquals( + ImmutableSortedKeyMap.of("one", 1, "two", 2), + "one", 1, "two", 2); + } + + @Test + public void ofNullKey() { + try { + ImmutableSortedKeyMap.of((String) null, 1); + fail(); + } catch (NullPointerException expected) { + } + + try { + ImmutableSortedKeyMap.of("one", 1, null, 2); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void ofNullValue() { + try { + ImmutableSortedKeyMap.of("one", null); + fail(); + } catch (NullPointerException expected) { + } + + try { + ImmutableSortedKeyMap.of("one", 1, "two", null); + fail(); + } catch (NullPointerException expected) { + } + } + + @Test + public void copyOfEmptyMap() { + ImmutableSortedKeyMap<String, Integer> copy + = ImmutableSortedKeyMap.copyOf(Collections.<String, Integer>emptyMap()); + assertEquals(Collections.<String, Integer>emptyMap(), copy); + assertSame(copy, ImmutableSortedKeyMap.copyOf(copy)); + } + + @Test + public void copyOfSingletonMap() { + ImmutableSortedKeyMap<String, Integer> copy + = ImmutableSortedKeyMap.copyOf(Collections.singletonMap("one", 1)); + assertMapEquals(copy, "one", 1); + assertSame(copy, ImmutableSortedKeyMap.copyOf(copy)); + } + + @Test + public void copyOf() { + Map<String, Integer> original = new LinkedHashMap<>(); + original.put("one", 1); + original.put("two", 2); + original.put("three", 3); + + ImmutableSortedKeyMap<String, Integer> copy = ImmutableSortedKeyMap.copyOf(original); + assertMapEquals(copy, "one", 1, "three", 3, "two", 2); + assertSame(copy, ImmutableSortedKeyMap.copyOf(copy)); + } + + @Test + public void nullGet() { + ImmutableSortedKeyMap<String, Integer> map = ImmutableSortedKeyMap.of("one", 1); + assertNull(map.get(null)); + } + + @Test + public void nullPointers() { + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicStaticMethods(ImmutableSortedKeyMap.class); + tester.testAllPublicInstanceMethods( + new ImmutableSortedKeyMap.Builder<String, Object>()); + tester.testAllPublicInstanceMethods(ImmutableSortedKeyMap.<String, Integer>of()); + tester.testAllPublicInstanceMethods(ImmutableSortedKeyMap.of("one", 1)); + tester.testAllPublicInstanceMethods( + ImmutableSortedKeyMap.of("one", 1, "two", 2)); + } + + private static <K, V> void assertMapEquals(Map<K, V> map, + Object... alternatingKeysAndValues) { + assertEquals(map.size(), alternatingKeysAndValues.length / 2); + int i = 0; + for (Entry<K, V> entry : map.entrySet()) { + assertEquals(alternatingKeysAndValues[i++], entry.getKey()); + assertEquals(alternatingKeysAndValues[i++], entry.getValue()); + } + } + + private static class IntHolder implements Serializable { + public int value; + + public IntHolder(int value) { + this.value = value; + } + + @Override public boolean equals(Object o) { + return (o instanceof IntHolder) && ((IntHolder) o).value == value; + } + + @Override public int hashCode() { + return value; + } + + private static final long serialVersionUID = 5; + } + + @Test + public void mutableValues() { + IntHolder holderA = new IntHolder(1); + IntHolder holderB = new IntHolder(2); + Map<String, IntHolder> map = ImmutableSortedKeyMap.of("a", holderA, "b", holderB); + holderA.value = 3; + assertTrue(map.entrySet().contains( + Maps.immutableEntry("a", new IntHolder(3)))); + Map<String, Integer> intMap = ImmutableSortedKeyMap.of("a", 3, "b", 2); + assertEquals(intMap.hashCode(), map.entrySet().hashCode()); + assertEquals(intMap.hashCode(), map.hashCode()); + } + + @Test + public void toStringTest() { + Map<String, Integer> map = ImmutableSortedKeyMap.of("a", 1, "b", 2); + assertEquals("{a=1, b=2}", map.toString()); + map = ImmutableSortedKeyMap.of(); + assertEquals("{}", map.toString()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/IterablesChainTest.java b/src/test/java/com/google/devtools/build/lib/collect/IterablesChainTest.java new file mode 100644 index 0000000000..734d801b98 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/IterablesChainTest.java @@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; + +/** + * A test for {@link IterablesChain}. + */ +@RunWith(JUnit4.class) +public class IterablesChainTest { + + @Test + public void addElement() { + IterablesChain.Builder<String> builder = IterablesChain.builder(); + builder.addElement("a"); + builder.addElement("b"); + assertEquals(Arrays.asList("a", "b"), ImmutableList.copyOf(builder.build())); + } + + @Test + public void add() { + IterablesChain.Builder<String> builder = IterablesChain.builder(); + builder.add(ImmutableList.of("a", "b")); + assertEquals(Arrays.asList("a", "b"), ImmutableList.copyOf(builder.build())); + } + + @Test + public void isEmpty() { + IterablesChain.Builder<String> builder = IterablesChain.builder(); + assertTrue(builder.isEmpty()); + builder.addElement("a"); + assertFalse(builder.isEmpty()); + builder = IterablesChain.builder(); + assertTrue(builder.isEmpty()); + builder.add(ImmutableList.of("a")); + assertFalse(builder.isEmpty()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpanderTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpanderTest.java new file mode 100644 index 0000000000..926ce2d77d --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/CompileOrderExpanderTest.java @@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + + +import com.google.common.collect.ImmutableList; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; + +/** + * Tests for {@link CompileOrderExpander}. + */ +@RunWith(JUnit4.class) +public class CompileOrderExpanderTest extends ExpanderTestBase { + + @Override + protected Order expanderOrder() { + return Order.COMPILE_ORDER; + } + + @Override + protected List<String> nestedResult() { + return ImmutableList.of("c", "a", "e", "b", "d"); + } + + @Override + protected List<String> nestedDuplicatesResult() { + return ImmutableList.of("c", "a", "e", "b", "d"); + } + + @Override + protected List<String> chainResult() { + return ImmutableList.of("c", "b", "a"); + } + + @Override + protected List<String> diamondResult() { + return ImmutableList.of("d", "b", "c", "a"); + } + + @Override + protected List<String> extendedDiamondResult() { + return ImmutableList.of("d", "e", "b", "c", "a"); + } + + @Override + protected List<String> extendedDiamondRightArmResult() { + return ImmutableList.of("d", "e", "b", "c2", "c", "a"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java new file mode 100644 index 0000000000..25448c6434 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/ExpanderTestBase.java @@ -0,0 +1,330 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; + +import junit.framework.TestCase; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Base class for tests of {@link NestedSetExpander} implementations. + * + * <p>This class provides test cases for representative nested set structures; the expected + * results must be provided by overriding the corresponding methods. + */ +public abstract class ExpanderTestBase extends TestCase { + + /** + * Returns the type of the expander under test. + */ + protected abstract Order expanderOrder(); + + @Test + public void simple() { + NestedSet<String> s = prepareBuilder("c", "a", "b").build(); + + assertTrue(Arrays.equals(simpleResult().toArray(), s.directMembers())); + assertSetContents(simpleResult(), s); + } + + @Test + public void simpleNoDuplicates() { + NestedSet<String> s = prepareBuilder("c", "a", "a", "a", "b").build(); + + assertTrue(Arrays.equals(simpleResult().toArray(), s.directMembers())); + assertSetContents(simpleResult(), s); + } + + @Test + public void nesting() { + NestedSet<String> subset = prepareBuilder("c", "a", "e").build(); + NestedSet<String> s = prepareBuilder("b", "d").addTransitive(subset).build(); + + assertSetContents(nestedResult(), s); + } + + @Test + public void builderReuse() { + NestedSetBuilder<String> builder = prepareBuilder(); + assertSetContents(Collections.<String>emptyList(), builder.build()); + + builder.add("b"); + assertSetContents(ImmutableList.of("b"), builder.build()); + + builder.addAll(ImmutableList.of("d")); + Collection<String> expected = ImmutableList.copyOf(prepareBuilder("b", "d").build()); + assertSetContents(expected, builder.build()); + + NestedSet<String> child = prepareBuilder("c", "a", "e").build(); + builder.addTransitive(child); + assertSetContents(nestedResult(), builder.build()); + } + + @Test + public void builderChaining() { + NestedSet<String> s = prepareBuilder().add("b").addAll(ImmutableList.of("d")) + .addTransitive(prepareBuilder("c", "a", "e").build()).build(); + assertSetContents(nestedResult(), s); + } + + @Test + public void addAllOrdering() { + NestedSet<String> s1 = prepareBuilder().add("a").add("c").add("b").build(); + NestedSet<String> s2 = prepareBuilder().addAll(ImmutableList.of("a", "c", "b")).build(); + + assertTrue(Arrays.equals(s1.directMembers(), s2.directMembers())); + assertCollectionsEqual(s1.toCollection(), s2.toCollection()); + assertCollectionsEqual(s1.toList(), s2.toList()); + assertCollectionsEqual(Lists.newArrayList(s1), Lists.newArrayList(s2)); + } + + @Test + public void mixedAddAllOrdering() { + NestedSet<String> s1 = prepareBuilder().add("a").add("b").add("c").add("d").build(); + NestedSet<String> s2 = prepareBuilder().add("a").addAll(ImmutableList.of("b", "c")).add("d") + .build(); + + assertTrue(Arrays.equals(s1.directMembers(), s2.directMembers())); + assertCollectionsEqual(s1.toCollection(), s2.toCollection()); + assertCollectionsEqual(s1.toList(), s2.toList()); + assertCollectionsEqual(Lists.newArrayList(s1), Lists.newArrayList(s2)); + } + + @Test + public void transitiveDepsHandledSeparately() { + NestedSet<String> subset = prepareBuilder("c", "a", "e").build(); + NestedSetBuilder<String> b = prepareBuilder(); + // The fact that we add the transitive subset between the add("b") and add("d") calls should + // not change the result. + b.add("b"); + b.addTransitive(subset); + b.add("d"); + NestedSet<String> s = b.build(); + + assertSetContents(nestedResult(), s); + } + + @Test + public void nestingNoDuplicates() { + NestedSet<String> subset = prepareBuilder("c", "a", "e").build(); + NestedSet<String> s = prepareBuilder("b", "d", "e").addTransitive(subset).build(); + + assertSetContents(nestedDuplicatesResult(), s); + } + + @Test + public void chain() { + NestedSet<String> c = prepareBuilder("c").build(); + NestedSet<String> b = prepareBuilder("b").addTransitive(c).build(); + NestedSet<String> a = prepareBuilder("a").addTransitive(b).build(); + + assertTrue(Arrays.equals(new String[]{"a"}, a.directMembers())); + assertSetContents(chainResult(), a); + } + + @Test + public void diamond() { + NestedSet<String> d = prepareBuilder("d").build(); + NestedSet<String> c = prepareBuilder("c").addTransitive(d).build(); + NestedSet<String> b = prepareBuilder("b").addTransitive(d).build(); + NestedSet<String> a = prepareBuilder("a").addTransitive(b).addTransitive(c).build(); + + assertTrue(Arrays.equals(new String[]{"a"}, a.directMembers())); + assertSetContents(diamondResult(), a); + } + + @Test + public void extendedDiamond() { + NestedSet<String> d = prepareBuilder("d").build(); + NestedSet<String> e = prepareBuilder("e").build(); + NestedSet<String> b = prepareBuilder("b").addTransitive(d).addTransitive(e).build(); + NestedSet<String> c = prepareBuilder("c").addTransitive(e).addTransitive(d).build(); + NestedSet<String> a = prepareBuilder("a").addTransitive(b).addTransitive(c).build(); + assertSetContents(extendedDiamondResult(), a); + } + + @Test + public void extendedDiamondRightArm() { + NestedSet<String> d = prepareBuilder("d").build(); + NestedSet<String> e = prepareBuilder("e").build(); + NestedSet<String> b = prepareBuilder("b").addTransitive(d).addTransitive(e).build(); + NestedSet<String> c2 = prepareBuilder("c2").addTransitive(e).addTransitive(d).build(); + NestedSet<String> c = prepareBuilder("c").addTransitive(c2).build(); + NestedSet<String> a = prepareBuilder("a").addTransitive(b).addTransitive(c).build(); + assertSetContents(extendedDiamondRightArmResult(), a); + } + + @Test + public void orderConflict() { + NestedSet<String> child1 = prepareBuilder("a", "b").build(); + NestedSet<String> child2 = prepareBuilder("b", "a").build(); + NestedSet<String> parent = prepareBuilder().addTransitive(child1).addTransitive(child2).build(); + assertSetContents(orderConflictResult(), parent); + } + + @Test + public void orderConflictNested() { + NestedSet<String> a = prepareBuilder("a").build(); + NestedSet<String> b = prepareBuilder("b").build(); + NestedSet<String> child1 = prepareBuilder().addTransitive(a).addTransitive(b).build(); + NestedSet<String> child2 = prepareBuilder().addTransitive(b).addTransitive(a).build(); + NestedSet<String> parent = prepareBuilder().addTransitive(child1).addTransitive(child2).build(); + assertSetContents(orderConflictResult(), parent); + } + + @Test + public void getOrderingEmpty() { + NestedSet<String> s = prepareBuilder().build(); + assertTrue(s.isEmpty()); + assertEquals(expanderOrder(), s.getOrder()); + } + + @Test + public void getOrdering() { + NestedSet<String> s = prepareBuilder("a", "b").build(); + assertTrue(!s.isEmpty()); + assertEquals(expanderOrder(), s.getOrder()); + } + + /** + * In case we have inner NestedSets with different order (allowed by the builder). We should + * maintain the order of the top-level NestedSet. + */ + @Test + public void regressionOnOneTransitiveDep() { + NestedSet<String> subsub = NestedSetBuilder.<String>stableOrder().add("c").add("a").add("e") + .build(); + NestedSet<String> sub = NestedSetBuilder.<String>stableOrder().add("b").add("d") + .addTransitive(subsub).build(); + NestedSet<String> top = prepareBuilder().addTransitive(sub).build(); + assertSetContents(nestedResult(), top); + } + + @Test + public void nestingValidation() { + for (Order ordering : Order.values()) { + NestedSet<String> a = prepareBuilder("a", "b").build(); + NestedSetBuilder<String> b = new NestedSetBuilder<>(ordering); + try { + b.addTransitive(a); + if (ordering != expanderOrder() && ordering != Order.STABLE_ORDER) { + fail(); // An exception was expected. + } + } catch (IllegalStateException e) { + if (ordering == expanderOrder() || ordering == Order.STABLE_ORDER) { + fail(); // No exception was expected. + } + } + } + } + + private NestedSetBuilder<String> prepareBuilder(String... directMembers) { + NestedSetBuilder<String> builder = new NestedSetBuilder<>(expanderOrder()); + builder.addAll(Lists.newArrayList(directMembers)); + return builder; + } + + protected final void assertSetContents(Collection<String> expected, NestedSet<String> set) { + assertEquals(expected, Lists.newArrayList(set)); + assertEquals(expected, Lists.newArrayList(set.toCollection())); + assertEquals(expected, Lists.newArrayList(set.toList())); + assertEquals(expected, Lists.newArrayList(set.toSet())); + } + + protected final void assertCollectionsEqual( + Collection<String> expected, Collection<String> actual) { + assertEquals(Lists.newArrayList(expected), Lists.newArrayList(actual)); + } + + /** + * Returns the enumeration of the nested set {"c", "a", "b"} in the + * implementation's enumeration order. + * + * @see #testSimple() + * @see #testSimpleNoDuplicates() + */ + protected List<String> simpleResult() { + return ImmutableList.of("c", "a", "b"); + } + + /** + * Returns the enumeration of the nested set {"b", "d", {"c", "a", "e"}} in + * the implementation's enumeration order. + * + * @see #testNesting() + */ + protected abstract List<String> nestedResult(); + + /** + * Returns the enumeration of the nested set {"b", "d", "e", {"c", "a", "e"}} in + * the implementation's enumeration order. + * + * @see #testNestingNoDuplicates() + */ + protected abstract List<String> nestedDuplicatesResult(); + + /** + * Returns the enumeration of nested set {"a", {"b", {"c"}}} in the + * implementation's enumeration order. + * + * @see #testChain() + */ + protected abstract List<String> chainResult(); + + /** + * Returns the enumeration of the nested set {"a", {"b", D}, {"c", D}}, where + * D is {"d"}, in the implementation's enumeration order. + * + * @see #testDiamond() + */ + protected abstract List<String> diamondResult(); + + /** + * Returns the enumeration of the nested set {"a", {"b", E, D}, {"c", D, E}}, where + * D is {"d"} and E is {"e"}, in the implementation's enumeration order. + * + * @see #testExtendedDiamond() + */ + protected abstract List<String> extendedDiamondResult(); + + /** + * Returns the enumeration of the nested set {"a", {"b", E, D}, {"c", C2}}, where + * D is {"d"}, E is {"e"} and C2 is {"c2", D, E}, in the implementation's enumeration order. + * + * @see #testExtendedDiamondRightArm() + */ + protected abstract List<String> extendedDiamondRightArmResult(); + + /** + * Returns the enumeration of the nested set {{"a", "b"}, {"b", "a"}}. + * + * @see #testOrderConflict() + * @see #testOrderConflictNested() + */ + protected List<String> orderConflictResult() { + return ImmutableList.of("a", "b"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpanderTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpanderTest.java new file mode 100644 index 0000000000..5d6988270a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/LinkOrderExpanderTest.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; + +/** + * Tests for {@link LinkOrderExpander}. + */ +@RunWith(JUnit4.class) +public class LinkOrderExpanderTest extends ExpanderTestBase { + + @Override + protected Order expanderOrder() { + return Order.LINK_ORDER; + } + + @Override + protected List<String> nestedResult() { + return ImmutableList.of("b", "d", "c", "a", "e"); + } + + @Override + protected List<String> nestedDuplicatesResult() { + return ImmutableList.of("b", "d", "c", "a", "e"); + } + + @Override + protected List<String> chainResult() { + return ImmutableList.of("a", "b", "c"); + } + + @Override + protected List<String> diamondResult() { + return ImmutableList.of("a", "b", "c", "d"); + } + + @Override + protected List<String> orderConflictResult() { + // Rightmost branch determines the order. + return ImmutableList.of("b", "a"); + } + + @Override + protected List<String> extendedDiamondResult() { + return ImmutableList.of("a", "b", "c", "e", "d"); + } + + @Override + protected List<String> extendedDiamondRightArmResult() { + return ImmutableList.of("a", "b", "c", "c2", "e", "d"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpanderTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpanderTest.java new file mode 100644 index 0000000000..80faf7a392 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NaiveLinkOrderExpanderTest.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import com.google.common.collect.ImmutableList; + +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; + +/** + * Tests for {@link NaiveLinkOrderExpander}. + */ +@RunWith(JUnit4.class) +public class NaiveLinkOrderExpanderTest extends ExpanderTestBase { + + @Override + protected Order expanderOrder() { + return Order.NAIVE_LINK_ORDER; + } + + @Override + protected List<String> nestedResult() { + return ImmutableList.of("b", "d", "c", "a", "e"); + } + + @Override + protected List<String> nestedDuplicatesResult() { + return ImmutableList.of("b", "d", "e", "c", "a"); + } + + @Override + protected List<String> chainResult() { + return ImmutableList.of("a", "b", "c"); + } + + @Override + protected List<String> diamondResult() { + // This case illustrates why this implementation is called "naive". + return ImmutableList.of("a", "b", "d", "c"); + } + + @Override + protected List<String> orderConflictResult() { + // Leftmost branch determines the order. + return ImmutableList.of("a", "b"); + } + + @Override + protected List<String> extendedDiamondResult() { + return ImmutableList.of("a", "b", "d", "e", "c"); + } + + @Override + protected List<String> extendedDiamondRightArmResult() { + return ImmutableList.of("a", "b", "d", "e", "c", "c2"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetImplTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetImplTest.java new file mode 100644 index 0000000000..e5cfae33cf --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/NestedSetImplTest.java @@ -0,0 +1,245 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.testing.EqualsTester; + +import junit.framework.TestCase; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; + +/** + * Tests for {@link com.google.devtools.build.lib.collect.nestedset.NestedSet}. + */ +@RunWith(JUnit4.class) +public class NestedSetImplTest extends TestCase { + @SafeVarargs + private static NestedSetBuilder<String> nestedSetBuilder(String... directMembers) { + NestedSetBuilder<String> builder = NestedSetBuilder.stableOrder(); + builder.addAll(Lists.newArrayList(directMembers)); + return builder; + } + + @Test + public void simple() { + NestedSet<String> set = nestedSetBuilder("a").build(); + + assertTrue(Arrays.equals(new String[]{"a"}, set.directMembers())); + assertEquals(0, set.transitiveSets().length); + assertEquals(false, set.isEmpty()); + } + + @Test + public void flatToString() { + assertEquals("{}", nestedSetBuilder().build().toString()); + assertEquals("{a}", nestedSetBuilder("a").build().toString()); + assertEquals("{a, b}", nestedSetBuilder("a", "b").build().toString()); + } + + @Test + public void nestedToString() { + NestedSet<String> b = nestedSetBuilder("b").build(); + NestedSet<String> c = nestedSetBuilder("c").build(); + + assertEquals("{a, {b}}", + nestedSetBuilder("a").addTransitive(b).build().toString()); + assertEquals("{a, {b}, {c}}", + nestedSetBuilder("a").addTransitive(b).addTransitive(c).build().toString()); + + assertEquals("{b}", nestedSetBuilder().addTransitive(b).build().toString()); + } + + @Test + public void isEmpty() { + NestedSet<String> triviallyEmpty = nestedSetBuilder().build(); + assertTrue(triviallyEmpty.isEmpty()); + + NestedSet<String> emptyLevel1 = nestedSetBuilder().addTransitive(triviallyEmpty).build(); + assertTrue(emptyLevel1.isEmpty()); + + NestedSet<String> emptyLevel2 = nestedSetBuilder().addTransitive(emptyLevel1).build(); + assertTrue(emptyLevel2.isEmpty()); + + NestedSet<String> triviallyNonEmpty = nestedSetBuilder("mango").build(); + assertFalse(triviallyNonEmpty.isEmpty()); + + NestedSet<String> nonEmptyLevel1 = nestedSetBuilder().addTransitive(triviallyNonEmpty).build(); + assertFalse(nonEmptyLevel1.isEmpty()); + + NestedSet<String> nonEmptyLevel2 = nestedSetBuilder().addTransitive(nonEmptyLevel1).build(); + assertFalse(nonEmptyLevel2.isEmpty()); + } + + @Test + public void canIncludeAnyOrderInStableOrderAndViceVersa() { + NestedSetBuilder.stableOrder() + .addTransitive(NestedSetBuilder.compileOrder() + .addTransitive(NestedSetBuilder.stableOrder().build()).build()) + .addTransitive(NestedSetBuilder.linkOrder() + .addTransitive(NestedSetBuilder.stableOrder().build()).build()) + .addTransitive(NestedSetBuilder.naiveLinkOrder() + .addTransitive(NestedSetBuilder.stableOrder().build()).build()).build(); + try { + NestedSetBuilder.compileOrder().addTransitive(NestedSetBuilder.linkOrder().build()).build(); + fail("Shouldn't be able to include a non-stable order inside a different non-stable order!"); + } catch (IllegalStateException e) { + // Expected. + } + } + + /** + * A handy wrapper that allows us to use EqualsTester to test shallowEquals and shallowHashCode. + */ + private static class SetWrapper<E> { + NestedSet<E> set; + + SetWrapper(NestedSet<E> wrapped) { + set = wrapped; + } + + @Override + public int hashCode() { + return set.shallowHashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SetWrapper)) { + return false; + } + try { + @SuppressWarnings("unchecked") + SetWrapper<E> other = (SetWrapper<E>) o; + return set.shallowEquals(other.set); + } catch (ClassCastException e) { + return false; + } + } + } + + @SafeVarargs + private static <E> SetWrapper<E> flat(E... directMembers) { + NestedSetBuilder<E> builder = NestedSetBuilder.stableOrder(); + builder.addAll(Lists.newArrayList(directMembers)); + return new SetWrapper<E>(builder.build()); + } + + // Same as flat(), but allows duplicate elements. + @SafeVarargs + private static <E> SetWrapper<E> flatWithDuplicates(E... directMembers) { + return new SetWrapper<E>( + NestedSetBuilder.wrap(Order.STABLE_ORDER, ImmutableList.copyOf(directMembers))); + } + + @SafeVarargs + private static <E> SetWrapper<E> nest(SetWrapper<E>... nested) { + NestedSetBuilder<E> builder = NestedSetBuilder.stableOrder(); + for (SetWrapper<E> wrap : nested) { + builder.addTransitive(wrap.set); + } + return new SetWrapper<E>(builder.build()); + } + + @SafeVarargs + // Restricted to <Integer> to avoid ambiguity with the other nest() function. + private static SetWrapper<Integer> nest(Integer elem, SetWrapper<Integer>... nested) { + NestedSetBuilder<Integer> builder = NestedSetBuilder.stableOrder(); + builder.add(elem); + for (SetWrapper<Integer> wrap : nested) { + builder.addTransitive(wrap.set); + } + return new SetWrapper<Integer>(builder.build()); + } + + @Test + public void shallowEquality() { + // Used below to check that inner nested sets can be compared by reference equality. + SetWrapper<Integer> myRef = nest(nest(flat(7, 8)), flat(9)); + + // Each "equality group" contains elements that are equal to one another + // (according to equals() and hashCode()), yet distinct from all elements + // of all other equality groups. + new EqualsTester() + .addEqualityGroup(flat(), + flat(), + nest(flat())) // Empty set elision. + .addEqualityGroup(NestedSetBuilder.<Integer>linkOrder().build()) + .addEqualityGroup(flat(3), + flat(3), + flat(3, 3)) // Element de-duplication. + .addEqualityGroup(flatWithDuplicates(3, 3)) + .addEqualityGroup(flat(4), + nest(flat(4))) // Automatic elision of one-element nested sets. + .addEqualityGroup(NestedSetBuilder.<Integer>linkOrder().add(4).build()) + .addEqualityGroup(nestedSetBuilder("4").build()) // Like flat("4"). + .addEqualityGroup(flat(3, 4), + flat(3, 4)) + // Shallow equality means that {{3},{5}} != {{3},{5}}. + .addEqualityGroup(nest(flat(3), flat(5))) + .addEqualityGroup(nest(flat(3), flat(5))) + .addEqualityGroup(nest(myRef), + nest(myRef), + nest(myRef, myRef)) // Set de-duplication. + .addEqualityGroup(nest(3, myRef)) + .addEqualityGroup(nest(4, myRef)) + .testEquals(); + + // Some things that are not tested by the above: + // - ordering among direct members + // - ordering among transitive sets + } + + /** Checks that the builder always return a nested set with the correct order. */ + @Test + public void correctOrder() { + for (Order order : Order.values()) { + for (int numDirects = 0; numDirects < 3; numDirects++) { + for (int numTransitives = 0; numTransitives < 3; numTransitives++) { + assertEquals(order, createNestedSet(order, numDirects, numTransitives, order).getOrder()); + // We allow mixing orders if one of them is stable. This tests that the top level order is + // the correct one. + assertEquals(order, + createNestedSet(order, numDirects, numTransitives, Order.STABLE_ORDER).getOrder()); + } + } + } + } + + private NestedSet<Integer> createNestedSet(Order order, int numDirects, int numTransitives, + Order transitiveOrder) { + NestedSetBuilder<Integer> builder = new NestedSetBuilder<>(order); + + for (int direct = 0; direct < numDirects; direct++) { + builder.add(direct); + } + for (int transitive = 0; transitive < numTransitives; transitive++) { + builder.addTransitive(new NestedSetBuilder<Integer>(transitiveOrder).add(transitive).build()); + } + return builder.build(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifierTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifierTest.java new file mode 100644 index 0000000000..9764f4df53 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/RecordingUniqueifierTest.java @@ -0,0 +1,134 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.collect.nestedset; + +import static org.junit.Assert.assertEquals; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import junit.framework.TestCase; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Random; + +/** + * Tests for {@link RecordingUniqueifier}. + */ +@RunWith(JUnit4.class) +public class RecordingUniqueifierTest extends TestCase { + + private static final Random RANDOM = new Random(); + + private static final int VERY_SMALL = 3; // one byte + private static final int SMALL = 11; // two bytes + private static final int MEDIUM = 18; // three bytes -- unmemoed + // For this one, the "* 8" is a bytes to bits (1 memo is 1 bit) + private static final int LARGE = (RecordingUniqueifier.LENGTH_THRESHOLD * 8) + 3; + + private static final int[] SIZES = new int[] {VERY_SMALL, SMALL, MEDIUM, LARGE}; + + private void doTest(int uniqueInputs, int deterministicHeadSize) throws Exception { + Preconditions.checkArgument(deterministicHeadSize <= uniqueInputs, + "deterministicHeadSize must be smaller than uniqueInputs"); + + // Setup + + List<Integer> inputList = new ArrayList<>(uniqueInputs); + Collection<Integer> inputsDeduped = new LinkedHashSet<>(uniqueInputs); + + for (int i = 0; i < deterministicHeadSize; i++) { // deterministic head + inputList.add(i); + inputsDeduped.add(i); + } + + while (inputsDeduped.size() < uniqueInputs) { // random selectees + Integer i = RANDOM.nextInt(uniqueInputs); + inputList.add(i); + inputsDeduped.add(i); + } + + // Unmemoed run + + List<Integer> firstList = new ArrayList<>(uniqueInputs); + RecordingUniqueifier recordingUniqueifier = new RecordingUniqueifier(); + for (Integer i : inputList) { + if (recordingUniqueifier.isUnique(i)) { + firstList.add(i); + } + } + + // Potentially memo'ed run + + List<Integer> secondList = new ArrayList<>(uniqueInputs); + Object memo = recordingUniqueifier.getMemo(); + Uniqueifier uniqueifier = RecordingUniqueifier.createReplayUniqueifier(memo); + for (Integer i : inputList) { + if (uniqueifier.isUnique(i)) { + secondList.add(i); + } + } + + // Evaluate results + + inputsDeduped = ImmutableList.copyOf(inputsDeduped); + assertEquals("Unmemo'ed run has unexpected contents", inputsDeduped, firstList); + assertEquals("Memo'ed run has unexpected contents", inputsDeduped, secondList); + } + + private void doTestWithLucidException(int uniqueInputs, int deterministicHeadSize) + throws Exception { + try { + doTest(uniqueInputs, deterministicHeadSize); + } catch (Exception e) { + throw new Exception("Failure in size: " + uniqueInputs, e); + } + } + + @Test + public void noInputs() throws Exception { + doTestWithLucidException(0, 0); + } + + @Test + public void allUnique() throws Exception { + for (int size : SIZES) { + doTestWithLucidException(size, size); + } + } + + @Test + public void fuzzedWithDeterministic2() throws Exception { + // The way that it is used, we know that the first two additions are not equal. + // Optimizations were made for this case in small memos. + for (int size : SIZES) { + doTestWithLucidException(size, 2); + } + } + + @Test + public void fuzzedWithDeterministic2_otherSizes() throws Exception { + for (int i = 0; i < 100; i++) { + int size = RANDOM.nextInt(10000) + 2; + doTestWithLucidException(size, 2); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitorTest.java b/src/test/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitorTest.java new file mode 100644 index 0000000000..8a6485c12c --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/concurrent/AbstractQueueVisitorTest.java @@ -0,0 +1,493 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.testutil.TestUtils; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Tests for AbstractQueueVisitor. + */ +@RunWith(JUnit4.class) +public class AbstractQueueVisitorTest { + + private static final RuntimeException THROWABLE = new RuntimeException(); + + @Test + public void simpleCounter() throws Exception { + CountingQueueVisitor counter = new CountingQueueVisitor(); + counter.enqueue(); + counter.work(false); + assertSame(10, counter.getCount()); + } + + @Test + public void callerOwnedPool() throws Exception { + ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 5, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()); + assertSame(0, executor.getActiveCount()); + + CountingQueueVisitor counter = new CountingQueueVisitor(executor); + counter.enqueue(); + counter.work(false); + assertSame(10, counter.getCount()); + + executor.shutdown(); + assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + + @Test + public void doubleCounter() throws Exception { + CountingQueueVisitor counter = new CountingQueueVisitor(); + counter.enqueue(); + counter.enqueue(); + counter.work(false); + assertSame(10, counter.getCount()); + } + + @Test + public void exceptionFromWorkerThread() { + final RuntimeException myException = new IllegalStateException(); + ConcreteQueueVisitor visitor = new ConcreteQueueVisitor(); + visitor.enqueue(new Runnable() { + @Override + public void run() { + throw myException; + } + }); + + try { + // The exception from the worker thread should be + // re-thrown from the main thread. + visitor.work(false); + fail(); + } catch (Exception e) { + assertSame(myException, e); + } + } + + // Regression test for "AbstractQueueVisitor loses track of jobs if thread allocation fails". + @Test + public void threadPoolThrowsSometimes() throws Exception { + // In certain cases (for example, if the address space is almost entirely consumed by a huge + // JVM heap), thread allocation can fail with an OutOfMemoryError. If the queue visitor + // does not handle this gracefully, we lose track of tasks and hang the visitor indefinitely. + + ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()) { + private final AtomicLong count = new AtomicLong(); + + @Override + public void execute(Runnable command) { + long count = this.count.incrementAndGet(); + if (count == 6) { + throw new Error("Could not create thread (fakeout)"); + } + super.execute(command); + } + }; + + CountingQueueVisitor counter = new CountingQueueVisitor(executor); + counter.enqueue(); + try { + counter.work(false); + fail(); + } catch (Error expected) { + assertEquals("Could not create thread (fakeout)", expected.getMessage()); + } + assertSame(5, counter.getCount()); + + executor.shutdown(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + + // Regression test to make sure that AbstractQueueVisitor doesn't swallow unchecked exceptions if + // it is interrupted concurrently with the unchecked exception being thrown. + @Test + public void interruptAndThrownIsInterruptedAndThrown() throws Exception { + final ConcreteQueueVisitor visitor = new ConcreteQueueVisitor(); + // Use a latch to make sure the thread gets a chance to start. + final CountDownLatch threadStarted = new CountDownLatch(1); + visitor.enqueue(new Runnable() { + @Override + public void run() { + threadStarted.countDown(); + assertTrue(Uninterruptibles.awaitUninterruptibly( + visitor.getInterruptionLatchForTestingOnly(), 2, TimeUnit.SECONDS)); + throw THROWABLE; + } + }); + assertTrue(threadStarted.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + // Interrupt will not be processed until work starts. + Thread.currentThread().interrupt(); + try { + visitor.work(/*interruptWorkers=*/true); + fail(); + } catch (Exception e) { + assertEquals(THROWABLE, e); + assertTrue(Thread.interrupted()); + } + } + + @Test + public void interruptionWithoutInterruptingWorkers() throws Exception { + final Thread mainThread = Thread.currentThread(); + final CountDownLatch latch1 = new CountDownLatch(1); + final CountDownLatch latch2 = new CountDownLatch(1); + final boolean[] workerThreadCompleted = { false }; + final ConcreteQueueVisitor visitor = new ConcreteQueueVisitor(); + + visitor.enqueue(new Runnable() { + @Override + public void run() { + try { + latch1.countDown(); + latch2.await(); + workerThreadCompleted[0] = true; + } catch (InterruptedException e) { + // Do not set workerThreadCompleted to true + } + } + }); + + TestThread interrupterThread = new TestThread() { + @Override + public void runTest() throws Exception { + latch1.await(); + mainThread.interrupt(); + assertTrue(visitor.awaitInterruptionForTestingOnly(TestUtils.WAIT_TIMEOUT_MILLISECONDS, + TimeUnit.MILLISECONDS)); + latch2.countDown(); + } + }; + + interrupterThread.start(); + + try { + visitor.work(false); + fail(); + } catch (InterruptedException e) { + // Expected. + } + + interrupterThread.joinAndAssertState(400); + assertTrue(workerThreadCompleted[0]); + } + + @Test + public void interruptionWithInterruptingWorkers() throws Exception { + assertInterruptWorkers(null); + + ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()); + assertInterruptWorkers(executor); + executor.shutdown(); + executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + private void assertInterruptWorkers(ThreadPoolExecutor executor) throws Exception { + final CountDownLatch latch1 = new CountDownLatch(1); + final CountDownLatch latch2 = new CountDownLatch(1); + final boolean[] workerThreadInterrupted = { false }; + ConcreteQueueVisitor visitor = (executor == null) + ? new ConcreteQueueVisitor() + : new ConcreteQueueVisitor(executor, true); + + visitor.enqueue(new Runnable() { + @Override + public void run() { + try { + latch1.countDown(); + latch2.await(); + } catch (InterruptedException e) { + workerThreadInterrupted[0] = true; + } + } + }); + + latch1.await(); + Thread.currentThread().interrupt(); + + try { + visitor.work(true); + fail(); + } catch (InterruptedException e) { + // Expected. + } + + assertTrue(workerThreadInterrupted[0]); + } + + @Test + public void failFast() throws Exception { + // In failFast mode, we only run actions queued before the exception. + assertFailFast(null, true, false, false, "a", "b"); + + // In !failFast mode, we complete all queued actions. + assertFailFast(null, false, false, false, "a", "b", "1", "2"); + + // Now check fail-fast on interrupt: + assertFailFast(null, false, true, true, "a", "b"); + assertFailFast(null, false, false, true, "a", "b", "1", "2"); + } + + @Test + public void failFastNoShutdown() throws Exception { + ThreadPoolExecutor executor = new ThreadPoolExecutor(5, 5, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()); + // In failFast mode, we only run actions queued before the exception. + assertFailFast(executor, true, false, false, "a", "b"); + + // In !failFast mode, we complete all queued actions. + assertFailFast(executor, false, false, false, "a", "b", "1", "2"); + + // Now check fail-fast on interrupt: + assertFailFast(executor, false, true, true, "a", "b"); + assertFailFast(executor, false, false, true, "a", "b", "1", "2"); + + executor.shutdown(); + assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + + private void assertFailFast(ThreadPoolExecutor executor, + boolean failFastOnException, boolean failFastOnInterrupt, + boolean interrupt, String... expectedVisited) throws Exception { + assertTrue(executor == null || !executor.isShutdown()); + AbstractQueueVisitor visitor = (executor == null) + ? new ConcreteQueueVisitor(failFastOnException, failFastOnInterrupt) + : new ConcreteQueueVisitor(executor, failFastOnException, failFastOnInterrupt); + + List<String> visitedList = Collections.synchronizedList(Lists.<String>newArrayList()); + + // Runnable "ra" will await the uncaught exception from + // "throwingRunnable", then add "a" to the list and + // enqueue "r1". Runnable "r1" should be + // executed iff !failFast. + + CountDownLatch latchA = new CountDownLatch(1); + CountDownLatch latchB = new CountDownLatch(1); + + Runnable r1 = awaitAddAndEnqueueRunnable(interrupt, visitor, null, visitedList, "1", null); + Runnable r2 = awaitAddAndEnqueueRunnable(interrupt, visitor, null, visitedList, "2", null); + Runnable ra = awaitAddAndEnqueueRunnable(interrupt, visitor, latchA, visitedList, "a", r1); + Runnable rb = awaitAddAndEnqueueRunnable(interrupt, visitor, latchB, visitedList, "b", r2); + + visitor.enqueue(ra); + visitor.enqueue(rb); + latchA.await(); + latchB.await(); + visitor.enqueue(interrupt ? interruptingRunnable(Thread.currentThread()) : throwingRunnable()); + + try { + visitor.work(false); + fail(); + } catch (Exception e) { + if (interrupt) { + assertTrue(e instanceof InterruptedException); + } else { + assertSame(THROWABLE, e); + } + } + assertTrue( + "got: " + visitedList + "\nwant: " + Arrays.toString(expectedVisited), + Sets.newHashSet(visitedList).equals(Sets.newHashSet(expectedVisited))); + + if (executor != null) { + assertFalse(executor.isShutdown()); + assertEquals(0, visitor.getTaskCount()); + } + } + + @Test + public void jobIsInterruptedWhenOtherFails() throws Exception { + ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>()); + + final QueueVisitorWithCriticalError visitor = new QueueVisitorWithCriticalError(executor); + final CountDownLatch latch1 = new CountDownLatch(1); + final AtomicBoolean wasInterrupted = new AtomicBoolean(false); + + Runnable r1 = new Runnable() { + + @Override + public void run() { + latch1.countDown(); + try { + // Interruption is expected during a sleep. There is no sense in fail or assert call + // because exception is going to be swallowed inside AbstractQueueVisitior. + // We are using wasInterrupted flag to assert in the end of test. + Thread.sleep(1000); + } catch (InterruptedException e) { + wasInterrupted.set(true); + } + } + }; + + visitor.enqueue(r1); + latch1.await(); + visitor.enqueue(throwingRunnable()); + + try { + visitor.work(true); + fail(); + } catch (Exception e) { + assertSame(THROWABLE, e); + } + + assertTrue(wasInterrupted.get()); + assertTrue(executor.isShutdown()); + } + + private Runnable throwingRunnable() { + return new Runnable() { + @Override + public void run() { + throw THROWABLE; + } + }; + } + + private Runnable interruptingRunnable(final Thread thread) { + return new Runnable() { + @Override + public void run() { + thread.interrupt(); + } + }; + } + + private static Runnable awaitAddAndEnqueueRunnable(final boolean interrupt, + final AbstractQueueVisitor visitor, + final CountDownLatch started, + final List<String> list, + final String toAdd, + final Runnable toEnqueue) { + return new Runnable() { + @Override + public void run() { + if (started != null) { + started.countDown(); + } + + try { + assertTrue(interrupt + ? visitor.awaitInterruptionForTestingOnly(1, TimeUnit.MINUTES) + : visitor.getExceptionLatchForTestingOnly().await(1, TimeUnit.MINUTES)); + } catch (InterruptedException e) { + // Unexpected. + throw new RuntimeException(e); + } + list.add(toAdd); + if (toEnqueue != null) { + visitor.enqueue(toEnqueue); + } + } + }; + } + + private static class CountingQueueVisitor extends AbstractQueueVisitor { + + private final static String THREAD_NAME = "BlazeTest CountingQueueVisitor"; + + private int theInt = 0; + private final Object lock = new Object(); + + public CountingQueueVisitor() { + super(5, 5, 3L, TimeUnit.SECONDS, THREAD_NAME); + } + + public CountingQueueVisitor(ThreadPoolExecutor executor) { + super(executor, false, true, true); + } + + public void enqueue() { + super.enqueue(new Runnable() { + @Override + public void run() { + synchronized (lock) { + if (theInt < 10) { + theInt++; + enqueue(); + } + } + } + }); + } + + public int getCount() { + return theInt; + } + } + + private static class ConcreteQueueVisitor extends AbstractQueueVisitor { + + private final static String THREAD_NAME = "BlazeTest ConcreteQueueVisitor"; + + public ConcreteQueueVisitor() { + super(5, 5, 3L, TimeUnit.SECONDS, THREAD_NAME); + } + + public ConcreteQueueVisitor(boolean failFast) { + super(true, 5, 5, 3L, TimeUnit.SECONDS, failFast, THREAD_NAME); + } + + public ConcreteQueueVisitor(boolean failFast, boolean failFastOnInterrupt) { + super(true, 5, 5, 3L, TimeUnit.SECONDS, failFast, failFastOnInterrupt, THREAD_NAME); + } + + public ConcreteQueueVisitor(ThreadPoolExecutor executor, boolean failFast, + boolean failFastOnInterrupt) { + super(executor, /*shutdownOnCompletion=*/false, failFast, failFastOnInterrupt); + } + + public ConcreteQueueVisitor(ThreadPoolExecutor executor, boolean failFast) { + super(executor, /*shutdownOnCompletion=*/false, failFast, true); + } + } + + private static class QueueVisitorWithCriticalError extends AbstractQueueVisitor { + + public QueueVisitorWithCriticalError(ThreadPoolExecutor executor) { + super(executor, false); + } + + @Override + protected boolean isCriticalError(Throwable e) { + return true; + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/concurrent/MoreFuturesTest.java b/src/test/java/com/google/devtools/build/lib/concurrent/MoreFuturesTest.java new file mode 100644 index 0000000000..60f29ac910 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/concurrent/MoreFuturesTest.java @@ -0,0 +1,151 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.util.concurrent.AbstractFuture; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.testutil.TestUtils; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * Tests for MoreFutures + */ +@RunWith(JUnit4.class) +public class MoreFuturesTest { + + private ExecutorService executorService; + + @Before + public void setUp() throws Exception { + executorService = Executors.newFixedThreadPool(5); + } + + @After + public void tearDown() throws Exception { + MoreExecutors.shutdownAndAwaitTermination(executorService, TestUtils.WAIT_TIMEOUT_SECONDS, + TimeUnit.SECONDS); + + } + + /** Test the normal path where everything is successful. */ + @Test + public void allAsListOrCancelAllHappy() throws ExecutionException, InterruptedException { + final List<DelayedFuture> futureList = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + DelayedFuture future = new DelayedFuture(i); + executorService.execute(future); + futureList.add(future); + } + ListenableFuture<List<Object>> list = MoreFutures.allAsListOrCancelAll(futureList); + List<Object> result = list.get(); + assertEquals(futureList.size(), result.size()); + for (DelayedFuture delayedFuture : futureList) { + assertFalse(delayedFuture.wasCanceled); + assertFalse(delayedFuture.wasInterrupted); + assertNotNull(delayedFuture.get()); + assertTrue(result.contains(delayedFuture.get())); + } + } + + /** Test that if any of the futures in the list fails, we cancel all the futures immediately. */ + @Test + public void allAsListOrCancelAllCancellation() throws InterruptedException { + final List<DelayedFuture> futureList = new ArrayList<>(); + for (int i = 1; i < 6; i++) { + DelayedFuture future = new DelayedFuture(i * 1000); + executorService.execute(future); + futureList.add(future); + } + DelayedFuture toFail = new DelayedFuture(1000); + futureList.add(toFail); + toFail.makeItFail(); + ListenableFuture<List<Object>> list = MoreFutures.allAsListOrCancelAll(futureList); + + try { + list.get(); + fail("This should fail"); + } catch (InterruptedException | ExecutionException ignored) { + } + Thread.sleep(100); + for (DelayedFuture delayedFuture : futureList) { + assertTrue(delayedFuture.wasCanceled || delayedFuture == toFail); + assertFalse(delayedFuture.wasInterrupted); + } + } + + /** + * A future that (if added to an executor) waits {@code delay} milliseconds before setting a + * response. + */ + private static class DelayedFuture extends AbstractFuture<Object> implements Runnable { + + private final int delay; + private final CountDownLatch latch = new CountDownLatch(1); + private boolean wasCanceled; + private boolean wasInterrupted; + + public DelayedFuture(int delay) { + this.delay = delay; + } + + @Override + public void run() { + try { + wasCanceled = latch.await(delay, TimeUnit.MILLISECONDS); + // Not canceled and not done (makeItFail sets the value, so in that case is done). + if (!wasCanceled && !isDone()) { + set(new Object()); + } + } catch (InterruptedException e) { + wasInterrupted = true; + } + } + + public void makeItFail() { + setException(new RuntimeException("I like to fail!!")); + latch.countDown(); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return super.cancel(mayInterruptIfRunning); + } + + @Override + protected void interruptTask() { + latch.countDown(); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/concurrent/ThreadSafetyTest.java b/src/test/java/com/google/devtools/build/lib/concurrent/ThreadSafetyTest.java new file mode 100644 index 0000000000..8532baee01 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/concurrent/ThreadSafetyTest.java @@ -0,0 +1,313 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.concurrent; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This file just contains some examples of the use of + * annotations for different categories of thread safety: + * ThreadSafe + * ThreadCompatible + * ThreadHostile + * Immutable ThreadSafe + * Immutable ThreadHostile + * + * It doesn't really test much -- just that this code + * using those annotations compiles and runs. + * + * The main class here is annotated as being both ConditionallyThreadSafe + * and ConditionallyThreadCompatible, and accordingly we document here the + * conditions under which it is thread-safe and thread-compatible: + * - it is thread-safe if you only use the testThreadSafety() method, + * the ThreadSafeCounter class, and/or ImmutableThreadSafeCounter class; + * - it is thread-compatible if you use only those and/or the + * ThreadCompatibleCounter and/or ImmutableThreadCompatibleCounter class; + * - it is thread-hostile otherwise. + */ +@ConditionallyThreadSafe @ConditionallyThreadCompatible +@RunWith(JUnit4.class) +public class ThreadSafetyTest { + + @ThreadSafe + public static final class ThreadSafeCounter { + + // A ThreadSafe class can have public mutable fields, + // provided they are atomic or volatile. + + public volatile boolean myBool; + public AtomicInteger myInt; + + // A ThreadSafe class can have private mutable fields, + // provided that access to them is synchronized. + private int value; + public ThreadSafeCounter(int value) { + synchronized (this) { // is this needed? + this.value = value; + } + } + public synchronized int getValue() { + return value; + } + public synchronized void increment() { + value++; + } + + // A ThreadSafe class can have private mutable members + // provided that the methods of the class synchronize access + // to them. + // These members could be static... + private static int numFoos = 0; + public static synchronized void foo() { + numFoos++; + } + public static synchronized int getNumFoos() { + return numFoos; + } + // ... or non-static. + private int numBars = 0; + public synchronized void bar() { + numBars++; + } + public synchronized int getNumBars() { + return numBars; + } + } + + @ThreadCompatible + public static final class ThreadCompatibleCounter { + + // A ThreadCompatible class can have public mutable fields. + public int value; + public ThreadCompatibleCounter(int value) { + this.value = value; + } + public int getValue() { + return value; + } + public void increment() { + value++; + } + + // A ThreadCompatible class can have mutable static members + // provided that the methods of the class synchronize access + // to them. + private static int numFoos = 0; + public static synchronized void foo() { + numFoos++; + } + public static synchronized int getNumFoos() { + return numFoos; + } + } + + @ThreadHostile + public static final class ThreadHostileCounter { + + // A ThreadHostile class can have public mutable fields. + public int value; + public ThreadHostileCounter(int value) { + this.value = value; + } + public int getValue() { + return value; + } + public void increment() { + value++; + } + + // A ThreadHostile class can perform unsynchronized access + // to mutable static data. + private static int numFoos = 0; + public static void foo() { + numFoos++; + } + public static int getNumFoos() { + return numFoos; + } + } + + @Immutable @ThreadSafe + public static final class ImmutableThreadSafeCounter { + + // An Immutable ThreadSafe class can have public fields, + // provided they are final and immutable. + public final int value; + public ImmutableThreadSafeCounter(int value) { + this.value = value; + } + public int getValue() { + return value; + } + public ImmutableThreadSafeCounter increment() { + return new ImmutableThreadSafeCounter(value + 1); + } + + // An Immutable ThreadSafe class can have immutable static members. + public static final int NUM_STATIC_CACHE_ENTRIES = 3; + private static final ImmutableThreadSafeCounter[] staticCache = + new ImmutableThreadSafeCounter[] { + new ImmutableThreadSafeCounter(0), + new ImmutableThreadSafeCounter(1), + new ImmutableThreadSafeCounter(2) + }; + public static ImmutableThreadSafeCounter makeUsingStaticCache(int value) { + if (value < NUM_STATIC_CACHE_ENTRIES) { + return staticCache[value]; + } else { + return new ImmutableThreadSafeCounter(value); + } + } + + // An Immutable ThreadSafe class can have private mutable members + // provided that the methods of the class synchronize access + // to them. + // These members could be static... + private static int cachedValue = 0; + private static ImmutableThreadSafeCounter cachedCounter = + new ImmutableThreadSafeCounter(0); + public static synchronized ImmutableThreadSafeCounter + makeUsingDynamicCache(int value) { + if (value != cachedValue) { + cachedValue = value; + cachedCounter = new ImmutableThreadSafeCounter(value); + } + return cachedCounter; + } + // ... or non-static. + private ImmutableThreadSafeCounter incrementCache = null; + public synchronized ImmutableThreadSafeCounter incrementUsingCache() { + if (incrementCache == null) { + incrementCache = new ImmutableThreadSafeCounter(value + 1); + } + return incrementCache; + } + // Methods of an Immutable class need not be deterministic. + private static Random random = new Random(); + public int choose() { + return random.nextInt(value); + } + } + + @Immutable @ThreadHostile + public static final class ImmutableThreadHostileCounter { + + // An Immutable ThreadHostile class can have public fields, + // provided they are final and immutable. + public final int value; + public ImmutableThreadHostileCounter(int value) { + this.value = value; + } + public int getValue() { + return value; + } + public ImmutableThreadHostileCounter increment() { + return new ImmutableThreadHostileCounter(value + 1); + } + + // An Immutable ThreadHostile class can have private mutable members, + // and doesn't need to synchronize access to them. + // These members could be static... + private static int cachedValue = 0; + private static ImmutableThreadHostileCounter cachedCounter = + new ImmutableThreadHostileCounter(0); + public static ImmutableThreadHostileCounter + makeUsingDynamicCache(int value) { + if (value != cachedValue) { + cachedValue = value; + cachedCounter = new ImmutableThreadHostileCounter(value); + } + return cachedCounter; + } + // ... or non-static. + private ImmutableThreadHostileCounter incrementCache = null; + public ImmutableThreadHostileCounter incrementUsingCache() { + if (incrementCache == null) { + incrementCache = new ImmutableThreadHostileCounter(value + 1); + } + return incrementCache; + } + } + + @Test + public void threadSafety() throws InterruptedException { + final ThreadSafeCounter threadSafeCounterArray[] = + new ThreadSafeCounter[] { + new ThreadSafeCounter(1), + new ThreadSafeCounter(2), + new ThreadSafeCounter(3) + }; + final ThreadCompatibleCounter threadCompatibleCounterArray[] = + new ThreadCompatibleCounter[] { + new ThreadCompatibleCounter(1), + new ThreadCompatibleCounter(2), + new ThreadCompatibleCounter(3) + }; + final ThreadHostileCounter threadHostileCounter = + new ThreadHostileCounter(1); + + class MyThread implements Runnable { + + ThreadCompatibleCounter threadCompatibleCounter = + new ThreadCompatibleCounter(1); + + @Override + public void run() { + + // ThreadSafe objects can be accessed with without synchronization + for (ThreadSafeCounter counter : threadSafeCounterArray) { + counter.increment(); + } + + // ThreadCompatible objects can be accessed with without + // synchronization if they are thread-local + threadCompatibleCounter.increment(); + + // Access to ThreadCompatible objects must be synchronized + // if they could be concurrently accessed by other threads + for (ThreadCompatibleCounter counter : threadCompatibleCounterArray) { + synchronized (counter) { + counter.increment(); + } + } + + // Access to ThreadHostile objects must be synchronized. + synchronized (this.getClass()) { + threadHostileCounter.increment(); + } + + } + } + + Thread thread1 = new Thread(new MyThread()); + Thread thread2 = new Thread(new MyThread()); + thread1.start(); + thread2.start(); + thread1.join(); + thread2.join(); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/AbstractEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/AbstractEventHandlerTest.java new file mode 100644 index 0000000000..7033f17990 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/AbstractEventHandlerTest.java @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Set; + +/** + * Tests {@link AbstractEventHandler}. + */ +@RunWith(JUnit4.class) +public class AbstractEventHandlerTest { + + private static AbstractEventHandler create(Set<EventKind> mask) { + return new AbstractEventHandler(mask) { + @Override + public void handle(Event event) {} + }; + } + + @Test + public void retainsEventMask() { + assertEquals(EventKind.ALL_EVENTS, + create(EventKind.ALL_EVENTS).getEventMask()); + assertEquals(EventKind.ERRORS_AND_WARNINGS, + create(EventKind.ERRORS_AND_WARNINGS).getEventMask()); + assertEquals(EventKind.ERRORS, + create(EventKind.ERRORS).getEventMask()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/EventCollectorTest.java b/src/test/java/com/google/devtools/build/lib/events/EventCollectorTest.java new file mode 100644 index 0000000000..332afacee3 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/EventCollectorTest.java @@ -0,0 +1,67 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; + +import com.google.devtools.build.lib.events.Event; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; + +/** + * Tests the {@link EventCollector} class. + */ +@RunWith(JUnit4.class) +public class EventCollectorTest extends EventTestTemplate { + + @Test + public void usesPassedInCollection() { + Collection<Event> events = new ArrayList<>(); + EventCollector collector = + new EventCollector(EventKind.ERRORS_AND_WARNINGS, events); + collector.handle(event); + Event onlyEvent = events.iterator().next(); + assertEquals(event.getMessage(), onlyEvent.getMessage()); + assertSame(location, onlyEvent.getLocation()); + assertEquals(event.getKind(), onlyEvent.getKind()); + assertEquals(event.getLocation().getStartOffset(), + onlyEvent.getLocation().getStartOffset()); + assertEquals(collector.count(), 1); + assertEquals(events.size(), 1); + } + + @Test + public void collectsEvents() { + EventCollector collector = + new EventCollector(EventKind.ERRORS_AND_WARNINGS); + collector.handle(event); + Iterator<Event> collectedEventIt = collector.iterator(); + Event onlyEvent = collectedEventIt.next(); + assertEquals(event.getMessage(), onlyEvent.getMessage()); + assertSame(location, onlyEvent.getLocation()); + assertEquals(event.getKind(), onlyEvent.getKind()); + assertEquals(event.getLocation().getStartOffset(), + onlyEvent.getLocation().getStartOffset()); + assertFalse(collectedEventIt.hasNext()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/EventSensorTest.java b/src/test/java/com/google/devtools/build/lib/events/EventSensorTest.java new file mode 100644 index 0000000000..3c97b37263 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/EventSensorTest.java @@ -0,0 +1,74 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A test for {@link EventSensor}. + */ +@RunWith(JUnit4.class) +public class EventSensorTest extends EventTestTemplate { + + @Test + public void sensorStartsOutWithFalse() { + assertFalse(new EventSensor(EventKind.ALL_EVENTS).wasTriggered()); + assertFalse(new EventSensor(EventKind.ERRORS).wasTriggered()); + assertFalse(new EventSensor(EventKind.ERRORS_AND_WARNINGS).wasTriggered()); + } + + @Test + public void sensorNoticesEventsInItsMask() { + EventSensor sensor = new EventSensor(EventKind.ERRORS); + Reporter reporter = new Reporter(sensor); + reporter.handle(Event.error(location, "An ERROR event.")); + assertTrue(sensor.wasTriggered()); + } + + @Test + public void sensorNoticesEventsInItsMask2() { + EventSensor sensor = new EventSensor(EventKind.ALL_EVENTS); + Reporter reporter = new Reporter(sensor); + reporter.handle(Event.error(location, "An ERROR event.")); + reporter.handle(Event.warn(location, "A warning event.")); + assertTrue(sensor.wasTriggered()); + } + + @Test + public void sensorIgnoresEventsNotInItsMask() { + EventSensor sensor = new EventSensor(EventKind.ERRORS_AND_WARNINGS); + Reporter reporter = new Reporter(sensor); + reporter.handle(Event.info(location, "An INFO event.")); + assertFalse(sensor.wasTriggered()); + } + + @Test + public void sensorCanCount() { + EventSensor sensor = new EventSensor(EventKind.ERRORS_AND_WARNINGS); + Reporter reporter = new Reporter(sensor); + reporter.handle(Event.error(location, "An ERROR event.")); + reporter.handle(Event.error(location, "Another ERROR event.")); + reporter.handle(Event.warn(location, "A warning event.")); + reporter.handle(Event.info(location, "An info event.")); // not in mask + assertEquals(3, sensor.getTriggerCount()); + assertTrue(sensor.wasTriggered()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/EventTest.java b/src/test/java/com/google/devtools/build/lib/events/EventTest.java new file mode 100644 index 0000000000..50fe88afdd --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/EventTest.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A super simple little test for the {@link Event} class. + */ +@RunWith(JUnit4.class) +public class EventTest extends EventTestTemplate { + + @Test + public void eventRetainsEventKind() { + assertEquals(EventKind.WARNING, event.getKind()); + } + + @Test + public void eventRetainsMessage() { + assertEquals("This is not an error message.", event.getMessage()); + } + + @Test + public void eventRetainsLocation() { + assertEquals(21, event.getLocation().getStartOffset()); + assertEquals(31, event.getLocation().getEndOffset()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/EventTestTemplate.java b/src/test/java/com/google/devtools/build/lib/events/EventTestTemplate.java new file mode 100644 index 0000000000..612cdf08c3 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/EventTestTemplate.java @@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import com.google.devtools.build.lib.events.Location.LineAndColumn; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Before; + +public abstract class EventTestTemplate { + + protected Event event; + protected Path path; + protected Location location; + protected Location locationNoPath; + protected Location locationNoLineInfo; + + private FsApparatus scratch = FsApparatus.newInMemory(); + + @Before + public void setUp() throws Exception { + String message = "This is not an error message."; + path = scratch.path("/my/sample/path.txt"); + + location = Location.fromPathAndStartColumn(path, 21, 31, new LineAndColumn(3, 4)); + + event = new Event(EventKind.WARNING, location, message); + + locationNoPath = Location.fromPathAndStartColumn(null, 21, 31, new LineAndColumn(3, 4)); + + locationNoLineInfo = Location.fromFileAndOffsets(path, 21, 31); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/LocationTest.java b/src/test/java/com/google/devtools/build/lib/events/LocationTest.java new file mode 100644 index 0000000000..a585b0c016 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/LocationTest.java @@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class LocationTest extends EventTestTemplate { + + @Test + public void fromFile() throws Exception { + Location location = Location.fromFile(path); + assertEquals(path.asFragment(), location.getPath()); + assertEquals(0, location.getStartOffset()); + assertEquals(0, location.getEndOffset()); + assertNull(location.getStartLineAndColumn()); + assertNull(location.getEndLineAndColumn()); + assertEquals(path + ":1", location.print()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/events/PrintingEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/PrintingEventHandlerTest.java new file mode 100644 index 0000000000..4cdfcb4567 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/PrintingEventHandlerTest.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; + +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.util.io.RecordingOutErr; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests the {@link PrintingEventHandler}. + */ +@RunWith(JUnit4.class) +public class PrintingEventHandlerTest extends EventTestTemplate { + + @Test + public void collectsEvents() { + RecordingOutErr recordingOutErr = new RecordingOutErr(); + PrintingEventHandler handler = new PrintingEventHandler(EventKind.ERRORS_AND_WARNINGS); + handler.setOutErr(recordingOutErr); + handler.handle(event); + MoreAsserts.assertEqualsUnifyingLineEnds("WARNING: /my/sample/path.txt:3:4: " + + "This is not an error message.\n", + recordingOutErr.errAsLatin1()); + assertEquals("", recordingOutErr.outAsLatin1()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/ReporterStreamTest.java b/src/test/java/com/google/devtools/build/lib/events/ReporterStreamTest.java new file mode 100644 index 0000000000..092d9400cd --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/ReporterStreamTest.java @@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; + +import com.google.devtools.build.lib.testutil.MoreAsserts; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.PrintWriter; + +@RunWith(JUnit4.class) +public class ReporterStreamTest { + + private Reporter reporter; + private StringBuilder out; + private EventHandler outAppender; + + @Before + public void setUp() throws Exception { + reporter = new Reporter(); + out = new StringBuilder(); + outAppender = new EventHandler() { + @Override + public void handle(Event event) { + out.append("[" + event.getKind() + ": " + event.getMessage() + "]\n"); + } + }; + } + + @Test + public void reporterStream() throws Exception { + assertEquals("", out.toString()); + reporter.addHandler(outAppender); + PrintWriter infoWriter = new PrintWriter(new ReporterStream(reporter, EventKind.INFO), true); + PrintWriter warnWriter = new PrintWriter(new ReporterStream(reporter, EventKind.WARNING), true); + try { + infoWriter.println("some info"); + warnWriter.println("a warning"); + } finally { + infoWriter.close(); + warnWriter.close(); + } + reporter.getOutErr().printOutLn("some output"); + reporter.getOutErr().printErrLn("an error"); + MoreAsserts.assertEqualsUnifyingLineEnds( + "[INFO: some info\n]\n" + + "[WARNING: a warning\n]\n" + + "[STDOUT: some output\n]\n" + + "[STDERR: an error\n]\n", + out.toString()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/events/ReporterTest.java b/src/test/java/com/google/devtools/build/lib/events/ReporterTest.java new file mode 100644 index 0000000000..f51451ab3e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/ReporterTest.java @@ -0,0 +1,100 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableList; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests the {@link Reporter} class. + */ +@RunWith(JUnit4.class) +public class ReporterTest extends EventTestTemplate { + + private Reporter reporter; + private StringBuilder out; + private AbstractEventHandler outAppender; + + @Before + public void setUp() throws Exception { + super.setUp(); + reporter = new Reporter(); + out = new StringBuilder(); + outAppender = new AbstractEventHandler(EventKind.ERRORS) { + @Override + public void handle(Event event) { + out.append(event.getMessage()); + } + }; + } + + @Test + public void reporterShowOutput() { + reporter.setOutputFilter(OutputFilter.RegexOutputFilter.forRegex("naughty")); + EventCollector collector = new EventCollector(EventKind.ALL_EVENTS); + reporter.addHandler(collector); + Event interesting = new Event(EventKind.WARNING, null, "show-me", "naughty"); + + reporter.handle(interesting); + reporter.handle(new Event(EventKind.WARNING, null, "ignore-me", "good")); + + assertEquals(ImmutableList.copyOf(collector.iterator()), ImmutableList.of(interesting)); + } + + @Test + public void reporterCollectsEvents() { + ImmutableList<Event> want = ImmutableList.of(Event.warn("xyz"), Event.error("err")); + EventCollector collector = new EventCollector(EventKind.ALL_EVENTS); + reporter.addHandler(collector); + for (Event e : want) { + reporter.handle(e); + } + ImmutableList<Event> got = ImmutableList.copyOf(collector.iterator()); + assertEquals(got, want); + } + + @Test + public void reporterCopyConstructorCopiesHandlersList() { + reporter.addHandler(outAppender); + reporter.addHandler(outAppender); + Reporter copiedReporter = new Reporter(reporter); + copiedReporter.addHandler(outAppender); // Should have 3 handlers now. + reporter.addHandler(outAppender); + reporter.addHandler(outAppender); // Should have 4 handlers now. + copiedReporter.handle(Event.error(location, ".")); + assertEquals("...", out.toString()); // The copied reporter has 3 handlers. + out = new StringBuilder(); + reporter.handle(Event.error(location, ".")); + assertEquals("....", out.toString()); // The old reporter has 4 handlers. + } + + @Test + public void removeHandlerUndoesAddHandler() { + assertEquals("", out.toString()); + reporter.addHandler(outAppender); + reporter.handle(Event.error(location, "Event gets registered.")); + assertEquals("Event gets registered.", out.toString()); + out = new StringBuilder(); + reporter.removeHandler(outAppender); + reporter.handle(Event.error(location, "Event gets ignored.")); + assertEquals("", out.toString()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/SimpleReportersTest.java b/src/test/java/com/google/devtools/build/lib/events/SimpleReportersTest.java new file mode 100644 index 0000000000..deed44f652 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/SimpleReportersTest.java @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A test for {@link Reporter}. + */ +@RunWith(JUnit4.class) +public class SimpleReportersTest extends EventTestTemplate { + + private int handlerCount = 0; + + @Test + public void addsHandlers() { + EventHandler handler = new EventHandler() { + @Override + public void handle(Event event) { + handlerCount++; + } + + }; + + Reporter reporter = new Reporter(handler); + reporter.handle(new Event(EventKind.INFO, location, "Add to handlerCount.")); + reporter.handle(new Event(EventKind.INFO, location, "Add to handlerCount.")); + reporter.handle(new Event(EventKind.INFO, location, "Add to handlerCount.")); + assertEquals(3, handlerCount); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/StoredErrorEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/StoredErrorEventHandlerTest.java new file mode 100644 index 0000000000..d47cf86dd4 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/StoredErrorEventHandlerTest.java @@ -0,0 +1,75 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; + +/** + * Tests the {@link StoredEventHandler} class. + */ +@RunWith(JUnit4.class) +public class StoredErrorEventHandlerTest { + + @Test + public void hasErrors() { + StoredEventHandler eventHandler = new StoredEventHandler(); + assertFalse(eventHandler.hasErrors()); + eventHandler.handle(Event.warn("warning")); + assertFalse(eventHandler.hasErrors()); + eventHandler.handle(Event.info("info")); + assertFalse(eventHandler.hasErrors()); + eventHandler.handle(Event.error("error")); + assertTrue(eventHandler.hasErrors()); + } + + @Test + public void replayOnWithoutEvents() { + StoredEventHandler eventHandler = new StoredEventHandler(); + StoredEventHandler sink = new StoredEventHandler(); + + eventHandler.replayOn(sink); + assertTrue(sink.isEmpty()); + } + + @Test + public void replayOn() { + StoredEventHandler eventHandler = new StoredEventHandler(); + StoredEventHandler sink = new StoredEventHandler(); + + List<Event> events = ImmutableList.of( + Event.warn("a"), + Event.error("b"), + Event.info("c"), + Event.warn("d")); + for (Event e : events) { + eventHandler.handle(e); + } + + eventHandler.replayOn(sink); + assertEquals(events.size(), sink.getEvents().size()); + for (int i = 0; i < events.size(); i++) { + assertEquals(events.get(i), sink.getEvents().get(i)); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandlerTest.java b/src/test/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandlerTest.java new file mode 100644 index 0000000000..ce6b1a97ca --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/WarningsAsErrorsEventHandlerTest.java @@ -0,0 +1,47 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests the {@link StoredEventHandler} class. + */ +@RunWith(JUnit4.class) +public class WarningsAsErrorsEventHandlerTest { + + @Test + public void hasErrors() { + ErrorSensingEventHandler delegate = + new ErrorSensingEventHandler(NullEventHandler.INSTANCE); + WarningsAsErrorsEventHandler eventHandler = + new WarningsAsErrorsEventHandler(delegate); + + eventHandler.handle(Event.info("info")); + assertFalse(delegate.hasErrors()); + + eventHandler.handle(Event.warn("warning")); + assertTrue(delegate.hasErrors()); + + delegate.resetErrors(); + + eventHandler.handle(Event.error("error")); + assertTrue(delegate.hasErrors()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java b/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java new file mode 100644 index 0000000000..1513735b07 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/util/EventCollectionApparatus.java @@ -0,0 +1,149 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events.util; + +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventCollector; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.PrintingEventHandler; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.testutil.JunitTestUtils; +import com.google.devtools.build.lib.util.io.OutErr; + +import java.util.List; +import java.util.Set; + +/** + * An apparatus for reporting / collecting events. + */ +public class EventCollectionApparatus { + + /** + * The fail fast handler, which fails the test fail whenever we encounter + * an error event. + */ + private static final EventHandler FAIL_FAST_HANDLER = new EventHandler() { + @Override + public void handle(Event event) { + assertWithMessage(event.toString()).that(EventKind.ERRORS_AND_WARNINGS) + .doesNotContain(event.getKind()); + } + }; + private Set<EventKind> customMask; + + /* + * Determine which events the {@link #collector()} created by this apparatus + * will collect. Default: {@link EventKind#ERRORS_AND_WARNINGS}. + * + */ + public EventCollectionApparatus(Set<EventKind> mask) { + this.customMask = mask; + + eventCollector = new EventCollector(customMask); + reporter = new Reporter(eventCollector); + printingEventHandler = new PrintingEventHandler(EventKind.ERRORS_AND_WARNINGS_AND_OUTPUT); + reporter.addHandler(printingEventHandler); + + this.setFailFast(true); + } + + public EventCollectionApparatus() { + this(EventKind.ERRORS_AND_WARNINGS); + } + + /* ---- Settings for the apparatus (configuration for creating state) ---- */ + + /* ---------- State that the apparatus initializes / operates on --------- */ + private EventCollector eventCollector; + private Reporter reporter; + private PrintingEventHandler printingEventHandler; + + /** + * Determine whether the {#link reporter()} created by this apparatus will + * fail fast, that is, throw an exception whenever we encounter an event of + * matching {@link EventKind#ERRORS_AND_WARNINGS}. + * Default: {@code true}. + */ + public void setFailFast(boolean failFast) { + if (failFast) { + reporter.addHandler(FAIL_FAST_HANDLER); + } else { + reporter.removeHandler(FAIL_FAST_HANDLER); + } + } + + /** + * Initializes the apparatus (if it's not been initialized yet) and returns + * the reporter created with the settings specified by this apparatus. + */ + public Reporter reporter() { + return reporter; + } + + /** + * Initializes the apparatus (if it's not been initialized yet) and returns + * the collector created with the settings specified by this apparatus. + */ + public EventCollector collector() { + return eventCollector; + } + + /** + * Redirects all output to the specified OutErr stream pair. + * Returns the previous OutErr. + */ + public OutErr setOutErr(OutErr outErr) { + return printingEventHandler.setOutErr(outErr); + } + + /** + * Utility method: Asserts that the {@link #collector()} has not collected + * any events. + * + * @throws IllegalStateException If the apparatus has not yet been + * initialized by calling {@link #reporter()} or {@link #collector()}. + */ + public void assertNoEvents() { + JunitTestUtils.assertNoEvents(eventCollector); + } + + /** + * Utility method: Assert that the {@link #collector()} has received an + * event with the {@code expectedMessage}. + */ + public Event assertContainsEvent(String expectedMessage) { + return JunitTestUtils.assertContainsEvent(eventCollector, + expectedMessage); + } + + public List<Event> assertContainsEventWithFrequency(String expectedMessage, + int expectedFrequency) { + return JunitTestUtils.assertContainsEventWithFrequency(eventCollector, expectedMessage, + expectedFrequency); + } + + /** + * Utility method: Assert that the {@link #collector()} has received an + * event with the {@code expectedMessage} in quotes. + */ + + public Event assertContainsEventWithWordsInQuotes(String... words) { + return JunitTestUtils.assertContainsEventWithWordsInQuotes( + eventCollector, words); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/events/util/LocationTestingUtil.java b/src/test/java/com/google/devtools/build/lib/events/util/LocationTestingUtil.java new file mode 100644 index 0000000000..66aaf30dd6 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/events/util/LocationTestingUtil.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.events.util; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.events.Location; + +/** + * Static utility methods for testing Locations. + */ +public class LocationTestingUtil { + + private LocationTestingUtil() { + } + + public static void assertEqualLocations(Location expected, Location actual) { + assertThat(actual.getStartOffset()).isEqualTo(expected.getStartOffset()); + assertThat(actual.getStartLineAndColumn()).isEqualTo(expected.getStartLineAndColumn()); + assertThat(actual.getEndOffset()).isEqualTo(expected.getEndOffset()); + assertThat(actual.getEndLineAndColumn()).isEqualTo(expected.getEndLineAndColumn()); + assertThat(actual.getPath()).isEqualTo(expected.getPath()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestSuiteBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestSuiteBuilder.java new file mode 100644 index 0000000000..d2f36a6cd5 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestSuiteBuilder.java @@ -0,0 +1,123 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import com.google.common.base.Predicate; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * A base class for constructing test suites by searching the classpath for + * tests, possibly restricted to a predicate. + */ +public class BlazeTestSuiteBuilder { + + /** + * @return a TestSuiteBuilder configured for Blaze. + */ + protected TestSuiteBuilder getBuilder() { + return new TestSuiteBuilder() + .addPackageRecursive("com.google.devtools.build.lib"); + } + + /** A predicate that succeeds only for LARGE tests. */ + public static final Predicate<Class<?>> TEST_IS_LARGE = + hasSize(Suite.LARGE_TESTS); + + /** A predicate that succeeds only for MEDIUM tests. */ + public static final Predicate<Class<?>> TEST_IS_MEDIUM = + hasSize(Suite.MEDIUM_TESTS); + + /** A predicate that succeeds only for SMALL tests. */ + public static final Predicate<Class<?>> TEST_IS_SMALL = + hasSize(Suite.SMALL_TESTS); + + /** A predicate that succeeds only for non-flaky tests. */ + public static final Predicate<Class<?>> TEST_IS_FLAKY = new Predicate<Class<?>>() { + @Override + public boolean apply(Class<?> testClass) { + return Suite.isFlaky(testClass); + } + }; + + private static Predicate<Class<?>> hasSize(final Suite size) { + return new Predicate<Class<?>>() { + @Override + public boolean apply(Class<?> testClass) { + return Suite.getSize(testClass) == size; + } + }; + } + + protected static Predicate<Class<?>> inSuite(final String suiteName) { + return new Predicate<Class<?>>() { + @Override + public boolean apply(Class<?> testClass) { + return Suite.getSuiteName(testClass).equalsIgnoreCase(suiteName); + } + }; + } + + /** + * Given a TestCase subclass, returns its designated suite annotation, if + * any, or the empty string otherwise. + */ + public static String getSuite(Class<?> clazz) { + TestSpec spec = clazz.getAnnotation(TestSpec.class); + return spec == null ? "" : spec.suite(); + } + + /** + * Returns a predicate over TestCases that is true iff the TestCase has a + * TestSpec annotation whose suite="..." value (a comma-separated list of + * tags) matches all of the query operators specified in the system property + * {@code blaze.suite}. The latter is also a comma-separated list, but of + * query operators, each of which is either the name of a tag which must be + * present (e.g. "foo"), or the !-prefixed name of a tag that must be absent + * (e.g. "!foo"). + */ + public static Predicate<Class<?>> matchesSuiteQuery() { + final String suiteProperty = System.getProperty("blaze.suite"); + if (suiteProperty == null) { + throw new IllegalArgumentException("blaze.suite property not found"); + } + final Set<String> queryTokens = splitCommas(suiteProperty); + return new Predicate<Class<?>>() { + @Override + public boolean apply(Class<?> testClass) { + // Return true iff every queryToken is satisfied by suiteTags. + Set<String> suiteTags = splitCommas(getSuite(testClass)); + for (String queryToken : queryTokens) { + if (queryToken.startsWith("!")) { // forbidden tag + if (suiteTags.contains(queryToken.substring(1))) { + return false; + } + } else { // mandatory tag + if (!suiteTags.contains(queryToken)) { + return false; + } + } + } + return true; + } + }; + } + + private static Set<String> splitCommas(String s) { + return new HashSet<>(Arrays.asList(s.split(","))); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestUtils.java b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestUtils.java new file mode 100644 index 0000000000..8588a08256 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/BlazeTestUtils.java @@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +/** + * Some static utility functions for testing Blaze code. In contrast to {@link TestUtils}, these + * functions are Blaze-specific. + */ +public class BlazeTestUtils { + private BlazeTestUtils() {} + + /** + * Populates the _embedded_binaries/ directory, containing all binaries/libraries, by symlinking + * directories#getEmbeddedBinariesRoot() to the test's runfiles tree. + */ + public static BinTools getIntegrationBinTools(BlazeDirectories directories) throws IOException { + Path embeddedDir = directories.getEmbeddedBinariesRoot(); + FileSystemUtils.createDirectoryAndParents(embeddedDir); + + Path runfiles = directories.getFileSystem().getPath(BlazeTestUtils.runfilesDir()); + // Copy over everything in embedded_scripts. + Path embeddedScripts = runfiles.getRelative(TestConstants.EMBEDDED_SCRIPTS_PATH); + Collection<Path> files = new ArrayList<Path>(); + if (embeddedScripts.exists()) { + files.addAll(embeddedScripts.getDirectoryEntries()); + } else { + System.err.println("test does not have " + embeddedScripts); + } + + for (Path fromFile : files) { + try { + embeddedDir.getChild(fromFile.getBaseName()).createSymbolicLink(fromFile); + } catch (IOException e) { + System.err.println("Could not symlink: " + e.getMessage()); + } + } + + return BinTools.forIntegrationTesting( + directories, embeddedDir.toString(), TestConstants.EMBEDDED_TOOLS); + } + + /** + * Writes a FilesetRule to a String array. + * + * @param name the name of the rule. + * @param out the output directory. + * @param entries The FilesetEntry entries. + * @return the String array of the rule. One String for each line. + */ + public static String[] createFilesetRule(String name, String out, String... entries) { + return new String[] { + String.format("Fileset(name = '%s', out = '%s',", name, out), + " entries = [" + Joiner.on(", ").join(entries) + "])" + }; + } + + public static File undeclaredOutputDir() { + String dir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR"); + if (dir != null) { + return new File(dir); + } + + return TestUtils.tmpDirFile(); + } + + public static String runfilesDir() { + String runfilesDirStr = TestUtils.getUserValue("TEST_SRCDIR"); + Preconditions.checkState(runfilesDirStr != null && runfilesDirStr.length() > 0, + "TEST_SRCDIR unset or empty"); + return new File(runfilesDirStr).getAbsolutePath(); + } + + /** Creates an empty file, along with all its parent directories. */ + public static void makeEmptyFile(Path path) throws IOException { + FileSystemUtils.createDirectoryAndParents(path.getParentDirectory()); + FileSystemUtils.createEmptyFile(path); + } + + /** + * Changes the mtime of the file "path", which must exist. No guarantee is + * made about the new mtime except that it is different from the previous one. + * + * @throws IOException if the mtime could not be read or set. + */ + public static void changeModtime(Path path) + throws IOException { + long prevMtime = path.getLastModifiedTime(); + long newMtime = prevMtime; + do { + newMtime += 1000; + path.setLastModifiedTime(newMtime); + } while (path.getLastModifiedTime() == prevMtime); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleBuilder.java new file mode 100644 index 0000000000..9f770c531f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleBuilder.java @@ -0,0 +1,202 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.packages.RuleClass; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Utility for quickly creating BUILD file rules for use in tests. + * + * <p>The use case for this class is writing BUILD files where simple + * readability for the sake of rules' relationship to the test framework + * is more important than detailed semantics and layout. + * + * <p>The behavior provided by this class is not meant to be exhaustive, + * but should handle a majority of simple cases. + * + * <p>Example: + * + * <pre> + * String text = new BuildRuleBuilder("java_library", "MyRule") + .setSources("First.java", "Second.java", "Third.java") + .setDeps(":library", "//java/com/google/common/collect") + .setResources("schema/myschema.xsd") + .build(); + * </pre> + * + */ +public class BuildRuleBuilder { + protected final RuleClass ruleClass; + protected final String ruleName; + private Map<String, List<String>> multiValueAttributes; + private Map<String, Object> singleValueAttributes; + protected Map<String, RuleClass> ruleClassMap; + + /** + * Create a new instance. + * + * @param ruleClass the rule class of the new rule + * @param ruleName the name of the new rule. + */ + public BuildRuleBuilder(String ruleClass, String ruleName) { + this(ruleClass, ruleName, getDefaultRuleClassMap()); + } + + protected static Map<String, RuleClass> getDefaultRuleClassMap() { + return TestRuleClassProvider.getRuleClassProvider().getRuleClassMap(); + } + + public BuildRuleBuilder(String ruleClass, String ruleName, Map<String, RuleClass> ruleClassMap) { + this.ruleClass = ruleClassMap.get(ruleClass); + this.ruleName = ruleName; + this.multiValueAttributes = new HashMap<>(); + this.singleValueAttributes = new HashMap<>(); + this.ruleClassMap = ruleClassMap; + } + + /** + * Sets the value of a single valued attribute + */ + public BuildRuleBuilder setSingleValueAttribute(String attrName, Object value) { + Preconditions.checkState(!singleValueAttributes.containsKey(attrName), + "attribute '" + attrName + "' already set"); + singleValueAttributes.put(attrName, value); + return this; + } + + /** + * Sets the value of a list type attribute + */ + public BuildRuleBuilder setMultiValueAttribute(String attrName, String... value) { + Preconditions.checkState(!multiValueAttributes.containsKey(attrName), + "attribute '" + attrName + "' already set"); + multiValueAttributes.put(attrName, Lists.newArrayList(value)); + return this; + } + + /** + * Set the srcs attribute. + */ + public BuildRuleBuilder setSources(String... sources) { + return setMultiValueAttribute("srcs", sources); + } + + /** + * Set the deps attribute. + */ + public BuildRuleBuilder setDeps(String... deps) { + return setMultiValueAttribute("deps", deps); + } + + /** + * Set the resources attribute. + */ + public BuildRuleBuilder setResources(String... resources) { + return setMultiValueAttribute("resources", resources); + } + + /** + * Set the data attribute. + */ + public BuildRuleBuilder setData(String... data) { + return setMultiValueAttribute("data", data); + } + + /** + * Generate the rule + * + * @return a string representation of the rule. + */ + public String build() { + StringBuilder sb = new StringBuilder(); + sb.append(ruleClass.getName()).append("("); + printNormal(sb, "name", ruleName); + for (Map.Entry<String, List<String>> entry : multiValueAttributes.entrySet()) { + printArray(sb, entry.getKey(), entry.getValue()); + } + for (Map.Entry<String, Object> entry : singleValueAttributes.entrySet()) { + printNormal(sb, entry.getKey(), entry.getValue()); + } + sb.append(")\n"); + return sb.toString(); + } + + private void printArray(StringBuilder sb, String attr, List<String> values) { + if (values == null || values.isEmpty()) { + return; + } + sb.append(" ").append(attr).append(" = "); + printList(sb, values); + sb.append(","); + sb.append("\n"); + } + + private void printNormal(StringBuilder sb, String attr, Object value) { + if (value == null) { + return; + } + sb.append(" ").append(attr).append(" = "); + if (value instanceof Integer) { + sb.append(value); + } else { + sb.append("'").append(value).append("'"); + } + sb.append(","); + sb.append("\n"); + } + + /** + * Turns iterable of {a b c} into string "['a', 'b', 'c']", appends to + * supplied StringBuilder. + */ + private void printList(StringBuilder sb, List<String> elements) { + sb.append("["); + Joiner.on(",").appendTo(sb, + Iterables.transform(elements, new Function<String, String>() { + @Override + public String apply(String from) { + return "'" + from + "'"; + } + })); + sb.append("]"); + } + + /** + * Returns the transitive closure of file names need to be generated in order + * for this rule to build. + */ + public Collection<String> getFilesToGenerate() { + return ImmutableList.of(); + } + + /** + * Returns the transitive closure of BuildRuleBuilders need to be generated in order + * for this rule to build. + */ + public Collection<BuildRuleBuilder> getRulesToGenerate() { + return ImmutableList.of(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleWithDefaultsBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleWithDefaultsBuilder.java new file mode 100644 index 0000000000..4af7ad1d8c --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/BuildRuleWithDefaultsBuilder.java @@ -0,0 +1,231 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Attribute.AllowedValueSet; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * A helper class to generate valid rules with filled attributes if necessary. + */ +public class BuildRuleWithDefaultsBuilder extends BuildRuleBuilder { + + private Set<String> generateFiles; + private Map<String, BuildRuleBuilder> generateRules; + + public BuildRuleWithDefaultsBuilder(String ruleClass, String ruleName) { + super(ruleClass, ruleName); + this.generateFiles = new HashSet<>(); + this.generateRules = new HashMap<>(); + } + + private BuildRuleWithDefaultsBuilder(String ruleClass, String ruleName, + Map<String, RuleClass> ruleClassMap, Set<String> generateFiles, + Map<String, BuildRuleBuilder> generateRules) { + super(ruleClass, ruleName, ruleClassMap); + this.generateFiles = generateFiles; + this.generateRules = generateRules; + } + + /** + * Creates a dummy file with the given extension in the given package and returns a valid Blaze + * label referring to the file. Note, the created label depends on the package of the rule. + */ + private String getDummyFileLabel(String rulePkg, String filePkg, String extension, + Type<?> attrType) { + boolean isInput = (attrType == Type.LABEL || attrType == Type.LABEL_LIST); + String fileName = (isInput ? "dummy_input" : "dummy_output") + extension; + generateFiles.add(filePkg + "/" + fileName); + if (rulePkg.equals(filePkg)) { + return ":" + fileName; + } else { + return filePkg + ":" + fileName; + } + } + + private String getDummyRuleLabel(String rulePkg, RuleClass referencedRuleClass) { + String referencedRuleName = ruleName + "_ref_" + referencedRuleClass.getName() + .replace("$", "").replace(":", ""); + // The new generated rule should have the same generatedFiles and generatedRules + // in order to avoid duplications + BuildRuleWithDefaultsBuilder builder = new BuildRuleWithDefaultsBuilder( + referencedRuleClass.getName(), referencedRuleName, ruleClassMap, generateFiles, + generateRules); + builder.popuplateAttributes(rulePkg, true); + generateRules.put(referencedRuleClass.getName(), builder); + return referencedRuleName; + } + + public BuildRuleWithDefaultsBuilder popuplateLabelAttribute(String pkg, Attribute attribute) { + return popuplateLabelAttribute(pkg, pkg, attribute); + } + + /** + * Populates the label type attribute with generated values. Populates with a file if possible, or + * generates an appropriate rule. Note, that the rules are always generated in the same package. + */ + public BuildRuleWithDefaultsBuilder popuplateLabelAttribute(String rulePkg, String filePkg, + Attribute attribute) { + Type<?> attrType = attribute.getType(); + String label = null; + if (attribute.getAllowedFileTypesPredicate() != FileTypeSet.NO_FILE) { + // Try to populate with files first + String extension = null; + if (attribute.getAllowedFileTypesPredicate() == FileTypeSet.ANY_FILE) { + extension = ".txt"; + } else { + FileTypeSet fileTypes = attribute.getAllowedFileTypesPredicate(); + // This argument should always hold, if not that means a Blaze design/implementation error + Preconditions.checkArgument(fileTypes.getExtensions().size() > 0); + extension = fileTypes.getExtensions().get(0); + } + label = getDummyFileLabel(rulePkg, filePkg, extension, attrType); + } else { + Predicate<RuleClass> allowedRuleClasses = attribute.getAllowedRuleClassesPredicate(); + if (allowedRuleClasses != Predicates.<RuleClass>alwaysFalse()) { + // See if there is an applicable rule among the already enqueued rules + BuildRuleBuilder referencedRuleBuilder = getFirstApplicableRule(allowedRuleClasses); + if (referencedRuleBuilder != null) { + label = ":" + referencedRuleBuilder.ruleName; + } else { + RuleClass referencedRuleClass = getFirstApplicableRuleClass(allowedRuleClasses); + if (referencedRuleClass != null) { + // Generate a rule with the appropriate ruleClass and a label for it in + // the original rule + label = ":" + getDummyRuleLabel(rulePkg, referencedRuleClass); + } + } + } + } + if (label != null) { + if (attrType == Type.LABEL_LIST || attrType == Type.OUTPUT_LIST) { + setMultiValueAttribute(attribute.getName(), label); + } else { + setSingleValueAttribute(attribute.getName(), label); + } + } + return this; + } + + private BuildRuleBuilder getFirstApplicableRule(Predicate<RuleClass> allowedRuleClasses) { + // There is no direct way to get the set of allowedRuleClasses from the Attribute + // The Attribute API probably should not be modified for sole testing purposes + for (Map.Entry<String, BuildRuleBuilder> entry : generateRules.entrySet()) { + if (allowedRuleClasses.apply(ruleClassMap.get(entry.getKey()))) { + return entry.getValue(); + } + } + return null; + } + + private RuleClass getFirstApplicableRuleClass(Predicate<RuleClass> allowedRuleClasses) { + // See comments in getFirstApplicableRule(Predicate<RuleClass> allowedRuleClasses) + for (RuleClass ruleClass : ruleClassMap.values()) { + if (allowedRuleClasses.apply(ruleClass)) { + return ruleClass; + } + } + return null; + } + + public BuildRuleWithDefaultsBuilder popuplateStringListAttribute(Attribute attribute) { + setMultiValueAttribute(attribute.getName(), "x"); + return this; + } + + public BuildRuleWithDefaultsBuilder popuplateStringAttribute(Attribute attribute) { + setSingleValueAttribute(attribute.getName(), "x"); + return this; + } + + public BuildRuleWithDefaultsBuilder popuplateBooleanAttribute(Attribute attribute) { + setSingleValueAttribute(attribute.getName(), "false"); + return this; + } + + public BuildRuleWithDefaultsBuilder popuplateIntegerAttribute(Attribute attribute) { + setSingleValueAttribute(attribute.getName(), 1); + return this; + } + + public BuildRuleWithDefaultsBuilder popuplateAttributes(String rulePkg, boolean heuristics) { + for (Attribute attribute : ruleClass.getAttributes()) { + if (attribute.isMandatory()) { + if (attribute.getType() == Type.LABEL_LIST || attribute.getType() == Type.OUTPUT_LIST) { + if (attribute.isNonEmpty()) { + popuplateLabelAttribute(rulePkg, attribute); + } else { + // TODO(bazel-team): actually here an empty list would be fine, but BuildRuleBuilder + // doesn't support that, and it makes little sense anyway + popuplateLabelAttribute(rulePkg, attribute); + } + } else if (attribute.getType() == Type.LABEL || attribute.getType() == Type.OUTPUT) { + popuplateLabelAttribute(rulePkg, attribute); + } else { + // Non label type attributes + if (attribute.getAllowedValues() instanceof AllowedValueSet) { + Collection<Object> allowedValues = + ((AllowedValueSet) attribute.getAllowedValues()).getAllowedValues(); + setSingleValueAttribute(attribute.getName(), allowedValues.iterator().next()); + } else if (attribute.getType() == Type.STRING) { + popuplateStringAttribute(attribute); + } else if (attribute.getType() == Type.BOOLEAN) { + popuplateBooleanAttribute(attribute); + } else if (attribute.getType() == Type.INTEGER) { + popuplateIntegerAttribute(attribute); + } else if (attribute.getType() == Type.STRING_LIST) { + popuplateStringListAttribute(attribute); + } + } + // TODO(bazel-team): populate for other data types + } else if (heuristics) { + populateAttributesHeuristics(rulePkg, attribute); + } + } + return this; + } + + // Heuristics which might help to generate valid rules. + // This is a bit hackish, but it helps some generated ruleclasses to pass analysis phase. + private void populateAttributesHeuristics(String rulePkg, Attribute attribute) { + if (attribute.getName().equals("srcs") && attribute.getType() == Type.LABEL_LIST) { + // If there is a srcs attribute it might be better to populate it even if it's not mandatory + popuplateLabelAttribute(rulePkg, attribute); + } else if (attribute.getName().equals("main_class") && attribute.getType() == Type.STRING) { + popuplateStringAttribute(attribute); + } + } + + @Override + public Collection<String> getFilesToGenerate() { + return generateFiles; + } + + @Override + public Collection<BuildRuleBuilder> getRulesToGenerate() { + return generateRules.values(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ChattyAssertsTestCase.java b/src/test/java/com/google/devtools/build/lib/testutil/ChattyAssertsTestCase.java new file mode 100644 index 0000000000..f17d81ead9 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/ChattyAssertsTestCase.java @@ -0,0 +1,237 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.ExitCode; + +import junit.framework.TestCase; + +import java.util.Objects; +import java.util.Set; + +/** + * Most of this stuff is copied from junit's {@link junit.framework.Assert} + * class, and then customized to make the error messages a bit more informative. + */ +public abstract class ChattyAssertsTestCase extends TestCase { + private long currentTestStartTime = -1; + + @Override + protected void setUp() throws Exception { + super.setUp(); + currentTestStartTime = BlazeClock.instance().currentTimeMillis(); + } + + @Override + protected void tearDown() throws Exception { + JunitTestUtils.nullifyInstanceFields(this); + assertFalse("tearDown without setUp!", currentTestStartTime == -1); + + super.tearDown(); + } + + /** + * Asserts that two objects are equal. If they are not + * an AssertionFailedError is thrown with the given message. + */ + public static void assertEquals(String message, Object expected, + Object actual) { + if (Objects.equals(expected, actual)) { + return; + } + chattyFailNotEquals(message, expected, actual); + } + + /** + * Asserts that two objects are equal. If they are not + * an AssertionFailedError is thrown. + */ + public static void assertEquals(Object expected, Object actual) { + assertEquals(null, expected, actual); + } + + /** + * Asserts that two Strings are equal. + */ + public static void assertEquals(String message, String expected, String actual) { + assertWithMessage(message).that(actual).isEqualTo(expected); + } + + /** + * Asserts that two Strings are equal. + */ + public static void assertEquals(String expected, String actual) { + assertEquals(null, expected, actual); + } + + /** + * Asserts that two Strings are equal considering the line separator to be \n + * independently of the operating system. + */ + public static void assertEqualsUnifyingLineEnds(String expected, String actual) { + MoreAsserts.assertEqualsUnifyingLineEnds(expected, actual); + } + + private static void chattyFailNotEquals(String message, Object expected, + Object actual) { + fail(MoreAsserts.chattyFormat(message, expected, actual)); + } + + /** + * Asserts that {@code e}'s exception message contains each of {@code strings} + * <b>surrounded by single quotation marks</b>. + */ + public static void assertMessageContainsWordsWithQuotes(Exception e, + String... strings) { + assertContainsWordsWithQuotes(e.getMessage(), strings); + } + + /** + * Asserts that {@code message} contains each of {@code strings} + * <b>surrounded by single quotation marks</b>. + */ + public static void assertContainsWordsWithQuotes(String message, + String... strings) { + MoreAsserts.assertContainsWordsWithQuotes(message, strings); + } + + public static void assertNonZeroExitCode(int exitCode, String stdout, String stderr) { + MoreAsserts.assertNonZeroExitCode(exitCode, stdout, stderr); + } + + public static void assertZeroExitCode(int exitCode, String stdout, String stderr) { + assertExitCode(0, exitCode, stdout, stderr); + } + + public static void assertExitCode(ExitCode expectedExitCode, + int exitCode, String stdout, String stderr) { + int expectedExitCodeValue = expectedExitCode.getNumericExitCode(); + if (exitCode != expectedExitCodeValue) { + fail(String.format("expected exit code '%s' <%d> but exit code was <%d> and stdout was <%s> " + + "and stderr was <%s>", + expectedExitCode.name(), expectedExitCodeValue, exitCode, stdout, stderr)); + } + } + + public static void assertExitCode(int expectedExitCode, + int exitCode, String stdout, String stderr) { + MoreAsserts.assertExitCode(expectedExitCode, exitCode, stdout, stderr); + } + + public static void assertStdoutContainsString(String expected, String stdout, String stderr) { + MoreAsserts.assertStdoutContainsString(expected, stdout, stderr); + } + + public static void assertStderrContainsString(String expected, String stdout, String stderr) { + MoreAsserts.assertStderrContainsString(expected, stdout, stderr); + } + + public static void assertStdoutContainsRegex(String expectedRegex, + String stdout, String stderr) { + MoreAsserts.assertStdoutContainsRegex(expectedRegex, stdout, stderr); + } + + public static void assertStderrContainsRegex(String expectedRegex, + String stdout, String stderr) { + MoreAsserts.assertStderrContainsRegex(expectedRegex, stdout, stderr); + } + + + + /******************************************************************** + * * + * Other testing utilities (unrelated to "chattiness") * + * * + ********************************************************************/ + + /** + * Returns the elements from the given collection in a set. + */ + protected static <T> Set<T> asSet(Iterable<T> collection) { + return Sets.newHashSet(collection); + } + + /** + * Returns the arguments given as varargs as a set. + */ + @SuppressWarnings({"unchecked", "varargs"}) + protected static <T> Set<T> asSet(T... elements) { + return Sets.newHashSet(elements); + } + + /** + * Returns the arguments given as varargs as a set of sorted Strings. + */ + protected static Set<String> asStringSet(Iterable<?> collection) { + return MoreAsserts.asStringSet(collection); + } + + /** + * An equivalence relation for Collection, based on mapping to Set. + * + * Oft-forgotten fact: for all x in Set, y in List, !x.equals(y) even if + * their elements are the same. + */ + protected static <T> void + assertSameContents(Iterable<? extends T> expected, Iterable<? extends T> actual) { + MoreAsserts.assertSameContents(expected, actual); + } + + /** + * Asserts the presence or absence of values in the collection. + */ + protected <T> void assertPresence(Iterable<T> actual, Iterable<Presence<T>> expectedPresences) { + for (Presence<T> expected : expectedPresences) { + if (expected.presence) { + assertThat(actual).contains(expected.value); + } else { + assertThat(actual).doesNotContain(expected.value); + } + } + } + + /** Creates a presence information with expected value. */ + protected static <T> Presence<T> present(T expected) { + return new Presence<>(expected, true); + } + + /** Creates an absence information with expected value. */ + protected static <T> Presence<T> absent(T expected) { + return new Presence<>(expected, false); + } + + /** + * Combines value with the boolean presence flag. + * + * @param <T> value type + */ + protected final static class Presence <T> { + /** wrapped value */ + public final T value; + /** boolean presence flag */ + public final boolean presence; + + /** Creates a tuple of value and a boolean presence flag. */ + Presence(T value, boolean presence) { + this.value = value; + this.presence = presence; + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/Classpath.java b/src/test/java/com/google/devtools/build/lib/testutil/Classpath.java new file mode 100644 index 0000000000..94711acc79 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/Classpath.java @@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import com.google.common.base.Preconditions; + +import java.io.File; +import java.io.IOException; +import java.util.Enumeration; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * A helper class to find all classes on the current classpath. This is used to automatically create + * JUnit 3 and 4 test suites. + */ +final class Classpath { + private static final String CLASS_EXTENSION = ".class"; + + /** + * Finds all classes that live in or below the given package. + */ + static Set<Class<?>> findClasses(String packageName) { + Set<Class<?>> result = new LinkedHashSet<>(); + String pathPrefix = (packageName + '.').replace('.', '/'); + for (String entryName : getClassPath()) { + File classPathEntry = new File(entryName); + if (classPathEntry.exists()) { + try { + Set<String> classNames; + if (classPathEntry.isDirectory()) { + classNames = findClassesInDirectory(classPathEntry, pathPrefix); + } else { + classNames = findClassesInJar(classPathEntry, pathPrefix); + } + for (String className : classNames) { + Class<?> clazz = Class.forName(className); + result.add(clazz); + } + } catch (IOException e) { + throw new AssertionError("Can't read classpath entry " + + entryName + ": " + e.getMessage()); + } catch (ClassNotFoundException e) { + throw new AssertionError("Class not found even though it is on the classpath " + + entryName + ": " + e.getMessage()); + } + } + } + return result; + } + + private static Set<String> findClassesInDirectory(File classPathEntry, String pathPrefix) { + Set<String> result = new TreeSet<>(); + File directory = new File(classPathEntry, pathPrefix); + innerFindClassesInDirectory(result, directory, pathPrefix); + return result; + } + + /** + * Finds all classes and sub packages in the given directory that are below the given package and + * add them to the respective sets. + * + * @param directory Directory to inspect + * @param pathPrefix Prefix for the path to the classes that are requested + * (ex: {@code com/google/foo/bar}) + */ + private static void innerFindClassesInDirectory(Set<String> classNames, File directory, + String pathPrefix) { + Preconditions.checkArgument(pathPrefix.endsWith("/")); + if (directory.exists()) { + for (File f : directory.listFiles()) { + String name = f.getName(); + if (name.endsWith(CLASS_EXTENSION)) { + String clzName = getClassName(pathPrefix + name); + classNames.add(clzName); + } else if (f.isDirectory()) { + findClassesInDirectory(f, pathPrefix + name + "/"); + } + } + } + } + + /** + * Returns a set of all classes in the jar that start with the given prefix. + */ + private static Set<String> findClassesInJar(File jarFile, String pathPrefix) throws IOException { + Set<String> classNames = new TreeSet<>(); + try (ZipFile zipFile = new ZipFile(jarFile)) { + Enumeration<? extends ZipEntry> entries = zipFile.entries(); + while (entries.hasMoreElements()) { + String entryName = entries.nextElement().getName(); + if (entryName.startsWith(pathPrefix) && entryName.endsWith(CLASS_EXTENSION)) { + classNames.add(getClassName(entryName)); + } + } + } + return classNames; + } + + /** + * Given the absolute path of a class file, return the class name. + */ + private static String getClassName(String className) { + int classNameEnd = className.length() - CLASS_EXTENSION.length(); + return className.substring(0, classNameEnd).replace('/', '.'); + } + + /** + * Gets the class path from the System Property "java.class.path" and splits + * it up into the individual elements. + */ + private static String[] getClassPath() { + String classPath = System.getProperty("java.class.path"); + String separator = System.getProperty("path.separator", ":"); + return classPath.split(Pattern.quote(separator)); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ClasspathSuite.java b/src/test/java/com/google/devtools/build/lib/testutil/ClasspathSuite.java new file mode 100644 index 0000000000..ee880fcddc --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/ClasspathSuite.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import org.junit.runners.Suite; +import org.junit.runners.model.RunnerBuilder; + +import java.util.Set; + +/** + * A suite implementation that finds all JUnit 3 and 4 classes on the current classpath in or below + * the package of the annotated class, except classes that are annotated with {@code ClasspathSuite} + * or {@link CustomSuite}. + * + * <p>If you need to specify a custom test class filter or a different package prefix, then use + * {@link CustomSuite} instead. + */ +public final class ClasspathSuite extends Suite { + + /** + * Only called reflectively. Do not use programmatically. + */ + public ClasspathSuite(Class<?> klass, RunnerBuilder builder) throws Throwable { + super(builder, klass, getClasses(klass)); + } + + private static Class<?>[] getClasses(Class<?> klass) { + Set<Class<?>> result = new TestSuiteBuilder().addPackageRecursive(klass.getPackage().getName()) + .create(); + return result.toArray(new Class<?>[result.size()]); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/CustomSuite.java b/src/test/java/com/google/devtools/build/lib/testutil/CustomSuite.java new file mode 100644 index 0000000000..6e3b6c56ab --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/CustomSuite.java @@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import org.junit.runners.Suite; +import org.junit.runners.model.RunnerBuilder; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Set; + +/** + * A JUnit4 suite implementation that delegates the class finding to a {@code suite()} method on the + * annotated class. To be used in combination with {@link TestSuiteBuilder}. + */ +public final class CustomSuite extends Suite { + + /** + * Only called reflectively. Do not use programmatically. + */ + public CustomSuite(Class<?> klass, RunnerBuilder builder) throws Throwable { + super(builder, klass, getClasses(klass)); + } + + private static Class<?>[] getClasses(Class<?> klass) { + Set<Class<?>> result = evalSuite(klass); + return result.toArray(new Class<?>[result.size()]); + } + + @SuppressWarnings("unchecked") // unchecked cast to a generic type + private static Set<Class<?>> evalSuite(Class<?> klass) { + try { + Method m = klass.getMethod("suite"); + if (!Modifier.isStatic(m.getModifiers())) { + throw new IllegalStateException("suite() must be static"); + } + return (Set<Class<?>>) m.invoke(null); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/DebuggingEventHandler.java b/src/test/java/com/google/devtools/build/lib/testutil/DebuggingEventHandler.java new file mode 100644 index 0000000000..59fe5fb574 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/DebuggingEventHandler.java @@ -0,0 +1,41 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; + +import java.io.PrintStream; + +/** + * Prints all errors and warnings to {@link System#out}. + */ +public class DebuggingEventHandler implements EventHandler { + + private PrintStream out; + + public DebuggingEventHandler() { + this.out = System.out; + } + + @Override + public void handle(Event e) { + if (e.getLocation() != null) { + out.println(e.getKind() + " " + e.getLocation() + ": " + e.getMessage()); + } else { + out.println(e.getKind() + " " + e.getMessage()); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/FoundationTestCase.java b/src/test/java/com/google/devtools/build/lib/testutil/FoundationTestCase.java new file mode 100644 index 0000000000..5c04612fc8 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/FoundationTestCase.java @@ -0,0 +1,264 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import com.google.common.io.Files; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventCollector; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Set; + +/** + * This is a specialization of {@link ChattyAssertsTestCase} that's useful for + * implementing tests of the "foundation" library. + */ +public abstract class FoundationTestCase extends ChattyAssertsTestCase { + + protected Path rootDirectory; + + protected Path outputBase; + + protected Path actionOutputBase; + + // May be overridden by subclasses: + protected Reporter reporter; + protected EventCollector eventCollector; + + private Scratch scratch; + + + // Individual tests can opt-out of this handler if they expect an error, by + // calling reporter.removeHandler(failFastHandler). + protected static final EventHandler failFastHandler = new EventHandler() { + @Override + public void handle(Event event) { + if (EventKind.ERRORS.contains(event.getKind())) { + fail(event.toString()); + } + } + }; + + protected static final EventHandler printHandler = new EventHandler() { + @Override + public void handle(Event event) { + System.out.println(event); + } + }; + + @Override + protected void setUp() throws Exception { + super.setUp(); + scratch = new Scratch(createFileSystem()); + outputBase = scratchDir("/usr/local/google/_blaze_jrluser/FAKEMD5/"); + rootDirectory = scratchDir("/" + TestConstants.TEST_WORKSPACE_DIRECTORY); + copySkylarkFilesIfExist(); + actionOutputBase = scratchDir("/usr/local/google/_blaze_jrluser/FAKEMD5/action_out/"); + eventCollector = new EventCollector(EventKind.ERRORS_AND_WARNINGS); + reporter = new Reporter(eventCollector); + reporter.addHandler(failFastHandler); + } + + /* + * Creates the file system; override to inject FS behavior. + */ + protected FileSystem createFileSystem() { + return new InMemoryFileSystem(BlazeClock.instance()); + } + + + private void copySkylarkFilesIfExist() throws IOException { + scratchFile(rootDirectory.getRelative("devtools/blaze/rules/BUILD").getPathString()); + scratchFile(rootDirectory.getRelative("rules/BUILD").getPathString()); + copySkylarkFilesIfExist("devtools/blaze/rules/staging", "devtools/blaze/rules"); + copySkylarkFilesIfExist("devtools/blaze/bazel/base_workspace/tools/build_rules", "rules"); + } + + private void copySkylarkFilesIfExist(String from, String to) throws IOException { + File rulesDir = new File(from); + if (rulesDir.exists() && rulesDir.isDirectory()) { + for (String fileName : rulesDir.list()) { + File file = new File(from + "/" + fileName); + if (file.isFile() && fileName.endsWith(".bzl")) { + String context = loadFile(file); + Path path = rootDirectory.getRelative(to + "/" + fileName); + if (path.exists()) { + overwriteScratchFile(path.getPathString(), context); + } else { + scratchFile(path.getPathString(), context); + } + } + } + } + } + + @Override + protected void tearDown() throws Exception { + Thread.interrupted(); // Clear any interrupt pending against this thread, + // so that we don't cause later tests to fail. + + super.tearDown(); + } + + /** + * A scratch filesystem that is completely in-memory. Since this file system + * is "cached" in a private (but *not* static) field in the test class, + * each testFoo method in junit sees a fresh filesystem. + */ + protected FileSystem scratchFS() { + return scratch.getFileSystem(); + } + + /** + * Create a scratch file in the scratch filesystem, with the given pathName, + * consisting of a set of lines. The method returns a Path instance for the + * scratch file. + */ + protected Path scratchFile(String pathName, String... lines) + throws IOException { + return scratch.file(pathName, lines); + } + + /** + * Like {@code scratchFile}, but the file is first deleted if it already + * exists. + */ + protected Path overwriteScratchFile(String pathName, String... lines) throws IOException { + return scratch.overwriteFile(pathName, lines); + } + + /** + * Deletes the specified scratch file, using the same specification as {@link Path#delete}. + */ + protected boolean deleteScratchFile(String pathName) throws IOException { + return scratch.deleteFile(pathName); + } + + /** + * Create a scratch file in the given filesystem, with the given pathName, + * consisting of a set of lines. The method returns a Path instance for the + * scratch file. + */ + protected Path scratchFile(FileSystem fs, String pathName, String... lines) + throws IOException { + return scratch.file(fs, pathName, lines); + } + + /** + * Create a scratch file in the given filesystem, with the given pathName, + * consisting of a set of lines. The method returns a Path instance for the + * scratch file. + */ + protected Path scratchFile(FileSystem fs, String pathName, byte[] content) + throws IOException { + return scratch.file(fs, pathName, content); + } + + /** + * Create a directory in the scratch filesystem, with the given path name. + */ + public Path scratchDir(String pathName) throws IOException { + return scratch.dir(pathName); + } + + /** + * If "expectedSuffix" is not a suffix of "actual", fails with an informative + * assertion. + */ + protected void assertEndsWith(String expectedSuffix, String actual) { + if (!actual.endsWith(expectedSuffix)) { + fail("\"" + actual + "\" does not end with " + + "\"" + expectedSuffix + "\""); + } + } + + /** + * If "expectedPrefix" is not a prefix of "actual", fails with an informative + * assertion. + */ + protected void assertStartsWith(String expectedPrefix, String actual) { + if (!actual.startsWith(expectedPrefix)) { + fail("\"" + actual + "\" does not start with " + + "\"" + expectedPrefix + "\""); + } + } + + // Mix-in assertions: + + protected void assertNoEvents() { + JunitTestUtils.assertNoEvents(eventCollector); + } + + protected Event assertContainsEvent(String expectedMessage) { + return JunitTestUtils.assertContainsEvent(eventCollector, + expectedMessage); + } + + protected Event assertContainsEvent(String expectedMessage, Set<EventKind> kinds) { + return JunitTestUtils.assertContainsEvent(eventCollector, + expectedMessage, + kinds); + } + + protected void assertContainsEventWithFrequency(String expectedMessage, + int expectedFrequency) { + JunitTestUtils.assertContainsEventWithFrequency(eventCollector, expectedMessage, + expectedFrequency); + } + + protected void assertDoesNotContainEvent(String expectedMessage) { + JunitTestUtils.assertDoesNotContainEvent(eventCollector, + expectedMessage); + } + + protected Event assertContainsEventWithWordsInQuotes(String... words) { + return JunitTestUtils.assertContainsEventWithWordsInQuotes( + eventCollector, words); + } + + protected void assertContainsEventsInOrder(String... expectedMessages) { + JunitTestUtils.assertContainsEventsInOrder(eventCollector, expectedMessages); + } + + @SuppressWarnings({"unchecked", "varargs"}) + protected static <T> void assertContainsSublist(List<T> arguments, + T... expectedSublist) { + JunitTestUtils.assertContainsSublist(arguments, expectedSublist); + } + + @SuppressWarnings({"unchecked", "varargs"}) + protected static <T> void assertDoesNotContainSublist(List<T> arguments, + T... expectedSublist) { + JunitTestUtils.assertDoesNotContainSublist(arguments, expectedSublist); + } + + protected static <T> void assertContainsSubset(Iterable<T> arguments, + Iterable<T> expectedSubset) { + JunitTestUtils.assertContainsSubset(arguments, expectedSubset); + } + + protected String loadFile(File file) throws IOException { + return Files.toString(file, Charset.defaultCharset()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/JunitTestUtils.java b/src/test/java/com/google/devtools/build/lib/testutil/JunitTestUtils.java new file mode 100644 index 0000000000..efe15992d5 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/JunitTestUtils.java @@ -0,0 +1,310 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventCollector; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.util.Pair; + +import junit.framework.TestCase; + +import java.lang.reflect.AccessibleObject; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * This class contains a utility method {@link #nullifyInstanceFields(Object)} + * for setting all fields in an instance to {@code null}. This is needed for + * junit {@code TestCase} instances that keep expensive objects in fields. + * Basically junit holds onto the instances + * even after the test methods have run, and it creates one such instance + * per {@code testFoo} method. + */ +public class JunitTestUtils { + + public static void nullifyInstanceFields(Object instance) + throws IllegalAccessException { + /** + * We're cleaning up this test case instance by assigning null pointers + * to all fields to reduce the memory overhead of test case instances + * staying around after the test methods have been executed. This is a + * bug in junit. + */ + List<Field> instanceFields = new ArrayList<>(); + for (Class<?> clazz = instance.getClass(); + !clazz.equals(TestCase.class) && !clazz.equals(Object.class); + clazz = clazz.getSuperclass()) { + for (Field field : clazz.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers())) { + continue; + } + if (field.getType().isPrimitive()) { + continue; + } + if (Modifier.isFinal(field.getModifiers())) { + String msg = "Please make field \"" + field + "\" non-final, or, if " + + "it's very simple and truly immutable and not too " + + "big, make it static."; + throw new AssertionError(msg); + } + instanceFields.add(field); + } + } + // Run setAccessible for efficiency + AccessibleObject.setAccessible(instanceFields.toArray(new Field[0]), true); + for (Field field : instanceFields) { + field.set(instance, null); + } + } + + /******************************************************************** + * * + * "Mix-in methods" * + * * + ********************************************************************/ + + // Java doesn't support mix-ins, but we need them in our tests so that we can + // inherit a bunch of useful methods, e.g. assertions over an EventCollector. + // We do this by hand, by delegating from instance methods in each TestCase + // to the static methods below. + + /** + * If the specified EventCollector contains any events, an informative + * assertion fails in the context of the specified TestCase. + */ + public static void assertNoEvents(Iterable<Event> eventCollector) { + String eventsString = eventsToString(eventCollector); + assertThat(eventsString).isEmpty(); + } + + /** + * If the specified EventCollector contains an unexpected number of events, an informative + * assertion fails in the context of the specified TestCase. + */ + public static void assertEventCount(int expectedCount, EventCollector eventCollector) { + assertWithMessage(eventsToString(eventCollector)) + .that(eventCollector.count()).isEqualTo(expectedCount); + } + + /** + * If the specified EventCollector does not contain an event which has + * 'expectedEvent' as a substring, an informative assertion fails. Otherwise + * the matching event is returned. + */ + public static Event assertContainsEvent(Iterable<Event> eventCollector, + String expectedEvent) { + return assertContainsEvent(eventCollector, expectedEvent, EventKind.ALL_EVENTS); + } + + /** + * If the specified EventCollector does not contain an event of a kind of 'kinds' which has + * 'expectedEvent' as a substring, an informative assertion fails. Otherwise + * the matching event is returned. + */ + public static Event assertContainsEvent(Iterable<Event> eventCollector, + String expectedEvent, + Set<EventKind> kinds) { + for (Event event : eventCollector) { + if (event.getMessage().contains(expectedEvent) && kinds.contains(event.getKind())) { + return event; + } + } + String eventsString = eventsToString(eventCollector); + assertWithMessage("Event '" + expectedEvent + "' not found" + + (eventsString.length() == 0 ? "" : ("; found these though:" + eventsString))) + .that(false).isTrue(); + return null; // unreachable + } + + /** + * If the specified EventCollector contains an event which has + * 'expectedEvent' as a substring, an informative assertion fails. + */ + public static void assertDoesNotContainEvent(Iterable<Event> eventCollector, + String expectedEvent) { + for (Event event : eventCollector) { + assertWithMessage("Unexpected string '" + expectedEvent + "' matched following event:\n" + + event.getMessage()).that(event.getMessage()).doesNotContain(expectedEvent); + } + } + + /** + * If the specified EventCollector does not contain an event which has + * each of {@code words} surrounded by single quotes as a substring, an + * informative assertion fails. Otherwise the matching event is returned. + */ + public static Event assertContainsEventWithWordsInQuotes( + Iterable<Event> eventCollector, + String... words) { + for (Event event : eventCollector) { + boolean found = true; + for (String word : words) { + if (!event.getMessage().contains("'" + word + "'")) { + found = false; + break; + } + } + if (found) { + return event; + } + } + String eventsString = eventsToString(eventCollector); + assertWithMessage("Event containing words " + Arrays.toString(words) + " in " + + "single quotes not found" + + (eventsString.length() == 0 ? "" : ("; found these though:" + eventsString))) + .that(false).isTrue(); + return null; // unreachable + } + + /** + * Returns a string consisting of each event in the specified collector, + * preceded by a newline. + */ + private static String eventsToString(Iterable<Event> eventCollector) { + StringBuilder buf = new StringBuilder(); + eventLoop: for (Event event : eventCollector) { + for (String ignoredPrefix : TestConstants.IGNORED_MESSAGE_PREFIXES) { + if (event.getMessage().startsWith(ignoredPrefix)) { + continue eventLoop; + } + } + buf.append('\n').append(event); + } + return buf.toString(); + } + + /** + * If "expectedSublist" is not a sublist of "arguments", an informative + * assertion is failed in the context of the specified TestCase. + * + * Argument order mnemonic: assert(X)ContainsSublist(Y). + */ + @SuppressWarnings({"unchecked", "varargs"}) + public static <T> void assertContainsSublist(List<T> arguments, T... expectedSublist) { + List<T> sublist = Arrays.asList(expectedSublist); + try { + assertThat(Collections.indexOfSubList(arguments, sublist)).isNotEqualTo(-1); + } catch (AssertionError e) { + throw new AssertionError("Did not find " + sublist + " as a sublist of " + arguments, e); + } + } + + /** + * If "expectedSublist" is a sublist of "arguments", an informative + * assertion is failed in the context of the specified TestCase. + * + * Argument order mnemonic: assert(X)DoesNotContainSublist(Y). + */ + @SuppressWarnings({"unchecked", "varargs"}) + public static <T> void assertDoesNotContainSublist(List<T> arguments, T... expectedSublist) { + List<T> sublist = Arrays.asList(expectedSublist); + try { + assertThat(Collections.indexOfSubList(arguments, sublist)).isEqualTo(-1); + } catch (AssertionError e) { + throw new AssertionError("Found " + sublist + " as a sublist of " + arguments, e); + } + } + + /** + * If "arguments" does not contain "expectedSubset" as a subset, an + * informative assertion is failed in the context of the specified TestCase. + * + * Argument order mnemonic: assert(X)ContainsSubset(Y). + */ + public static <T> void assertContainsSubset(Iterable<T> arguments, + Iterable<T> expectedSubset) { + Set<T> argumentsSet = arguments instanceof Set<?> + ? (Set<T>) arguments + : Sets.newHashSet(arguments); + + for (T x : expectedSubset) { + assertWithMessage("assertContainsSubset failed: did not find element " + x + + "\nExpected subset = " + expectedSubset + "\nArguments = " + arguments) + .that(argumentsSet).contains(x); + } + } + + /** + * Check to see if each element of expectedMessages is the beginning of a message + * in eventCollector, in order, as in {@link #containsSublistWithGapsAndEqualityChecker}. + * If not, an informative assertion is failed + */ + protected static void assertContainsEventsInOrder(Iterable<Event> eventCollector, + String... expectedMessages) { + String failure = containsSublistWithGapsAndEqualityChecker( + ImmutableList.copyOf(eventCollector), + new Function<Pair<Event, String>, Boolean> () { + @Override + public Boolean apply(Pair<Event, String> pair) { + return pair.first.getMessage().contains(pair.second); + } + }, expectedMessages); + + String eventsString = eventsToString(eventCollector); + assertWithMessage("Event '" + failure + "' not found in proper order" + + (eventsString.length() == 0 ? "" : ("; found these though:" + eventsString))) + .that(failure).isNull(); + } + + /** + * Check to see if each element of expectedSublist is in arguments, according to + * the equalityChecker, in the same order as in expectedSublist (although with + * other interspersed elements in arguments allowed). + * @param equalityChecker function that takes a Pair<S, T> element and returns true + * if the elements of the pair are equal by its lights. + * @return first element not in arguments in order, or null if success. + */ + @SuppressWarnings({"unchecked"}) + protected static <S, T> T containsSublistWithGapsAndEqualityChecker(List<S> arguments, + Function<Pair<S, T>, Boolean> equalityChecker, T... expectedSublist) { + Iterator<S> iter = arguments.iterator(); + outerLoop: + for (T expected : expectedSublist) { + while (iter.hasNext()) { + S actual = iter.next(); + if (equalityChecker.apply(Pair.of(actual, expected))) { + continue outerLoop; + } + } + return expected; + } + return null; + } + + public static List<Event> assertContainsEventWithFrequency(Iterable<Event> events, + String expectedMessage, int expectedFrequency) { + ImmutableList.Builder<Event> builder = ImmutableList.builder(); + for (Event event : events) { + if (event.getMessage().contains(expectedMessage)) { + builder.add(event); + } + } + List<Event> foundEvents = builder.build(); + assertWithMessage(foundEvents.toString()).that(foundEvents).hasSize(expectedFrequency); + return foundEvents; + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java new file mode 100644 index 0000000000..d4f6058169 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.devtools.build.lib.util.Clock; + +import java.util.concurrent.TimeUnit; + +/** + * A fake clock for testing. + */ +public final class ManualClock implements Clock { + private long currentTimeMillis = 0L; + + @Override + public long currentTimeMillis() { + return currentTimeMillis; + } + + @Override + public long nanoTime() { + return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis); + } + + public void advanceMillis(long time) { + currentTimeMillis += time; + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/MoreAsserts.java b/src/test/java/com/google/devtools/build/lib/testutil/MoreAsserts.java new file mode 100644 index 0000000000..9224b8a6ef --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/MoreAsserts.java @@ -0,0 +1,319 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.common.truth.Truth.assert_; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.lang.ref.Reference; +import java.lang.reflect.Field; +import java.util.Comparator; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * A helper class for tests providing a simple interface for asserts. + */ +public class MoreAsserts { + + public static void assertContainsRegex(String regex, String actual) { + assertThat(actual).containsMatch(regex); + } + + public static void assertContainsRegex(String msg, String regex, String actual) { + assertWithMessage(msg).that(actual).containsMatch(regex); + } + + public static void assertNotContainsRegex(String regex, String actual) { + assertThat(actual).doesNotContainMatch(regex); + } + + public static void assertNotContainsRegex(String msg, String regex, String actual) { + assertWithMessage(msg).that(actual).doesNotContainMatch(regex); + } + + public static void assertMatchesRegex(String regex, String actual) { + assertThat(actual).matches(regex); + } + + public static void assertMatchesRegex(String msg, String regex, String actual) { + assertWithMessage(msg).that(actual).matches(regex); + } + + public static void assertNotMatchesRegex(String regex, String actual) { + assertThat(actual).doesNotMatch(regex); + } + + public static <T> void assertEquals(T expected, T actual, Comparator<T> comp) { + assertThat(comp.compare(expected, actual)).isEqualTo(0); + } + + public static <T> void assertContentsAnyOrder( + Iterable<? extends T> expected, Iterable<? extends T> actual, + Comparator<? super T> comp) { + assertThat(actual).hasSize(Iterables.size(expected)); + int i = 0; + for (T e : expected) { + for (T a : actual) { + if (comp.compare(e, a) == 0) { + i++; + } + } + } + assertThat(actual).hasSize(i); + } + + public static void assertGreaterThanOrEqual(long target, long actual) { + assertThat(actual).isAtLeast(target); + } + + public static void assertGreaterThanOrEqual(String msg, long target, long actual) { + assertWithMessage(msg).that(actual).isAtLeast(target); + } + + public static void assertGreaterThan(long target, long actual) { + assertThat(actual).isGreaterThan(target); + } + + public static void assertGreaterThan(String msg, long target, long actual) { + assertWithMessage(msg).that(actual).isGreaterThan(target); + } + + public static void assertLessThanOrEqual(long target, long actual) { + assertThat(actual).isAtMost(target); + } + + public static void assertLessThanOrEqual(String msg, long target, long actual) { + assertWithMessage(msg).that(actual).isAtMost(target); + } + + public static void assertLessThan(long target, long actual) { + assertThat(actual).isLessThan(target); + } + + public static void assertLessThan(String msg, long target, long actual) { + assertWithMessage(msg).that(actual).isLessThan(target); + } + + public static void assertEndsWith(String ending, String actual) { + assertThat(actual).endsWith(ending); + } + + public static void assertStartsWith(String prefix, String actual) { + assertThat(actual).startsWith(prefix); + } + + /** + * Scans if an instance of given class is strongly reachable from a given + * object. + * <p>Runs breadth-first search in object reachability graph to check if + * an instance of <code>clz</code> can be reached. + * <strong>Note:</strong> This method can take a long time if analyzed + * data structure spans across large part of heap and may need a lot of + * memory. + * + * @param start object to start the search from + * @param clazz class to look for + */ + public static void assertInstanceOfNotReachable( + Object start, final Class<?> clazz) { + Predicate<Object> p = new Predicate<Object>() { + @Override + public boolean apply(Object obj) { + return clazz.isAssignableFrom(obj.getClass()); + } + }; + if (isRetained(p, start)) { + assert_().fail("Found an instance of " + clazz.getCanonicalName() + + " reachable from " + start.toString()); + } + } + + private static final Field NON_STRONG_REF; + + static { + try { + NON_STRONG_REF = Reference.class.getDeclaredField("referent"); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + static final Predicate<Field> ALL_STRONG_REFS = new Predicate<Field>() { + @Override + public boolean apply(Field field) { + return NON_STRONG_REF.equals(field); + } + }; + + private static boolean isRetained(Predicate<Object> predicate, Object start) { + Map<Object, Object> visited = Maps.newIdentityHashMap(); + visited.put(start, start); + Queue<Object> toScan = Lists.newLinkedList(); + toScan.add(start); + + while (!toScan.isEmpty()) { + Object current = toScan.poll(); + if (current.getClass().isArray()) { + if (current.getClass().getComponentType().isPrimitive()) { + continue; + } + + for (Object ref : (Object[]) current) { + if (ref != null) { + if (predicate.apply(ref)) { + return true; + } + if (visited.put(ref, ref) == null) { + toScan.add(ref); + } + } + } + } else { + // iterate *all* fields (getFields() returns only accessible ones) + for (Class<?> clazz = current.getClass(); clazz != null; + clazz = clazz.getSuperclass()) { + for (Field f : clazz.getDeclaredFields()) { + if (f.getType().isPrimitive() || ALL_STRONG_REFS.apply(f)) { + continue; + } + + f.setAccessible(true); + try { + Object ref = f.get(current); + if (ref != null) { + if (predicate.apply(ref)) { + return true; + } + if (visited.put(ref, ref) == null) { + toScan.add(ref); + } + } + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Error when scanning the heap", e); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Error when scanning the heap", e); + } + } + } + } + } + return false; + } + + private static String getClassDescription(Object object) { + return object == null + ? "null" + : ("instance of " + object.getClass().getName()); + } + + public static String chattyFormat(String message, Object expected, Object actual) { + String expectedClass = getClassDescription(expected); + String actualClass = getClassDescription(actual); + + return Joiner.on('\n').join((message != null) ? ("\n" + message) : "", + " expected " + expectedClass + ": <" + expected + ">", + " but was " + actualClass + ": <" + actual + ">"); + } + + public static void assertEqualsUnifyingLineEnds(String expected, String actual) { + if (actual != null) { + actual = actual.replaceAll(System.getProperty("line.separator"), "\n"); + } + assertThat(actual).isEqualTo(expected); + } + + public static void assertContainsWordsWithQuotes(String message, + String... strings) { + for (String string : strings) { + assertTrue(message + " should contain '" + string + "' (with quotes)", + message.contains("'" + string + "'")); + } + } + + public static void assertNonZeroExitCode(int exitCode, String stdout, String stderr) { + if (exitCode == 0) { + fail("expected non-zero exit code but exit code was 0 and stdout was <" + + stdout + "> and stderr was <" + stderr + ">"); + } + } + + public static void assertExitCode(int expectedExitCode, + int exitCode, String stdout, String stderr) { + if (exitCode != expectedExitCode) { + fail(String.format("expected exit code <%d> but exit code was <%d> and stdout was <%s> " + + "and stderr was <%s>", expectedExitCode, exitCode, stdout, stderr)); + } + } + + public static void assertStdoutContainsString(String expected, String stdout, String stderr) { + if (!stdout.contains(expected)) { + fail("expected stdout to contain string <" + expected + "> but stdout was <" + + stdout + "> and stderr was <" + stderr + ">"); + } + } + + public static void assertStderrContainsString(String expected, String stdout, String stderr) { + if (!stderr.contains(expected)) { + fail("expected stderr to contain string <" + expected + "> but stdout was <" + + stdout + "> and stderr was <" + stderr + ">"); + } + } + + public static void assertStdoutContainsRegex(String expectedRegex, + String stdout, String stderr) { + if (!Pattern.compile(expectedRegex).matcher(stdout).find()) { + fail("expected stdout to contain regex <" + expectedRegex + "> but stdout was <" + + stdout + "> and stderr was <" + stderr + ">"); + } + } + + public static void assertStderrContainsRegex(String expectedRegex, + String stdout, String stderr) { + if (!Pattern.compile(expectedRegex).matcher(stderr).find()) { + fail("expected stderr to contain regex <" + expectedRegex + "> but stdout was <" + + stdout + "> and stderr was <" + stderr + ">"); + } + } + + public static Set<String> asStringSet(Iterable<?> collection) { + Set<String> set = Sets.newTreeSet(); + for (Object o : collection) { + set.add("\"" + String.valueOf(o) + "\""); + } + return set; + } + + public static <T> void + assertSameContents(Iterable<? extends T> expected, Iterable<? extends T> actual) { + if (!Sets.newHashSet(expected).equals(Sets.newHashSet(actual))) { + fail("got string set: " + asStringSet(actual).toString() + + "\nwant: " + asStringSet(expected).toString()); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/Scratch.java b/src/test/java/com/google/devtools/build/lib/testutil/Scratch.java new file mode 100644 index 0000000000..229d2a7a75 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/Scratch.java @@ -0,0 +1,150 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import java.io.IOException; + +/** + * Allow tests to easily manage scratch files in a FileSystem. + */ +public final class Scratch { + + private final FileSystem fileSystem; + + /** + * Create a new ScratchFileSystem using the {@link InMemoryFileSystem} + */ + public Scratch() { + this(new InMemoryFileSystem(BlazeClock.instance())); + } + + /** + * Create a new ScratchFileSystem using the supplied FileSystem. + */ + public Scratch(FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + /** + * Returns the FileSystem in use. + */ + public FileSystem getFileSystem() { + return fileSystem; + } + + /** + * Create a directory in the scratch filesystem, with the given path name. + */ + public Path dir(String pathName) throws IOException { + Path dir = getFileSystem().getPath(pathName); + if (!dir.exists()) { + FileSystemUtils.createDirectoryAndParents(dir); + } + if (!dir.isDirectory()) { + throw new IOException("Exists, but is not a directory: " + pathName); + } + return dir; + } + + /** + * Create a scratch file in the scratch filesystem, with the given pathName, + * consisting of a set of lines. The method returns a Path instance for the + * scratch file. + */ + public Path file(String pathName, String... lines) + throws IOException { + Path newFile = file(getFileSystem(), pathName, lines); + newFile.setLastModifiedTime(-1L); + return newFile; + } + + /** + * Like {@code scratchFile}, but the file is first deleted if it already + * exists. + */ + public Path overwriteFile(String pathName, String... lines) throws IOException { + Path oldFile = getFileSystem().getPath(pathName); + long newMTime = oldFile.exists() ? oldFile.getLastModifiedTime() + 1 : -1; + oldFile.delete(); + Path newFile = file(getFileSystem(), pathName, lines); + newFile.setLastModifiedTime(newMTime); + return newFile; + } + + /** + * Deletes the specified scratch file, using the same specification as {@link Path#delete}. + */ + public boolean deleteFile(String pathName) throws IOException { + Path file = getFileSystem().getPath(pathName); + return file.delete(); + } + + /** + * Create a scratch file in the given filesystem, with the given pathName, + * consisting of a set of lines. The method returns a Path instance for the + * scratch file. + */ + public Path file(FileSystem fs, String pathName, String... lines) + throws IOException { + Path file = newScratchFile(fs, pathName); + FileSystemUtils.writeContentAsLatin1(file, linesAsString(lines)); + return file; + } + + /** + * Create a scratch file in the given filesystem, with the given pathName, + * consisting of a set of lines. The method returns a Path instance for the + * scratch file. + */ + public Path file(FileSystem fs, String pathName, byte[] content) + throws IOException { + Path file = newScratchFile(fs, pathName); + FileSystemUtils.writeContent(file, content); + return file; + } + + /** Creates a new scratch file, ensuring parents exist. */ + private Path newScratchFile(FileSystem fs, String pathName) throws IOException { + Path file = fs.getPath(pathName); + Path parentDir = file.getParentDirectory(); + if (!parentDir.exists()) { + FileSystemUtils.createDirectoryAndParents(parentDir); + } + if (file.exists()) { + throw new IOException("Could not create scratch file (file exists) " + + pathName); + } + return file; + } + + /** + * Converts the lines into a String with linebreaks. Useful for creating + * in-memory input for a file, for example. + */ + private static String linesAsString(String... lines) { + StringBuilder builder = new StringBuilder(); + for (String line : lines) { + builder.append(line); + builder.append('\n'); + } + return builder.toString(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/Suite.java b/src/test/java/com/google/devtools/build/lib/testutil/Suite.java new file mode 100644 index 0000000000..43590d463c --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/Suite.java @@ -0,0 +1,86 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * Test annotations used to select which tests to run in a given situation. + */ +public enum Suite { + + /** + * It's so blazingly fast and lightweight we run it whenever we make any + * build.lib change. This size is the default. + */ + SMALL_TESTS, + + /** + * It's a bit too slow to run all the time, but it still tests some + * unit of functionality. May run external commands such as gcc, for example. + */ + MEDIUM_TESTS, + + /** + * I don't even want to think about running this one after every edit, + * but I don't mind if the continuous build runs it, and I'm happy to have + * it before making a release. + */ + LARGE_TESTS, + + /** + * These tests take a long time. They should only ever be run manually and probably from their + * own Blaze test target. + */ + ENORMOUS_TESTS; + + /** + * Given a class, determine the test size. + */ + public static Suite getSize(Class<?> clazz) { + return getAnnotationElementOrDefault(clazz, "size"); + } + + /** + * Given a class, determine the suite it belongs to. + */ + public static String getSuiteName(Class<?> clazz) { + return getAnnotationElementOrDefault(clazz, "suite"); + } + + /** + * Given a class, determine if it is flaky. + */ + public static boolean isFlaky(Class<?> clazz) { + return getAnnotationElementOrDefault(clazz, "flaky"); + } + + /** + * Returns the value of the given element in the {@link TestSpec} annotation of the given class, + * or the default value of that element if the class doesn't have a {@link TestSpec} annotation. + */ + @SuppressWarnings("unchecked") + private static <T> T getAnnotationElementOrDefault(Class<?> clazz, String elementName) { + TestSpec spec = clazz.getAnnotation(TestSpec.class); + try { + Method method = TestSpec.class.getMethod(elementName); + return spec != null ? (T) method.invoke(spec) : (T) method.getDefaultValue(); + } catch (NoSuchMethodException e) { + throw new IllegalStateException("no such element " + elementName, e); + } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { + throw new IllegalStateException("can't invoke accessor for element " + elementName, e); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java b/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java new file mode 100644 index 0000000000..d9552ada1f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java @@ -0,0 +1,53 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.common.collect.ImmutableList; + +/** + * Various constants required by the tests. + */ +public class TestConstants { + private TestConstants() { + } + + /** + * A list of all embedded binaries that go into the regular Bazel binary. + */ + public static final ImmutableList<String> EMBEDDED_TOOLS = ImmutableList.of( + "build-runfiles", + "process-wrapper", + "build_interface_so"); + + + /** + * Location in the bazel repo where embedded binaries come from. + */ + public static final String EMBEDDED_SCRIPTS_PATH = "DOES-NOT-WORK-YET"; + + /** + * Directory where we can find bazel's Java tests, relative to a test's runfiles directory. + */ + public static final String JAVATESTS_ROOT = "src/test/java/"; + + /** + * The directory in InMemoryFileSystem where workspaces created during unit tests reside. + */ + public static final String TEST_WORKSPACE_DIRECTORY = "bazel"; + + public static final String TEST_RULE_CLASS_PROVIDER = + "com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider"; + public static final ImmutableList<String> IGNORED_MESSAGE_PREFIXES = ImmutableList.<String>of(); +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestFileOutErr.java b/src/test/java/com/google/devtools/build/lib/testutil/TestFileOutErr.java new file mode 100644 index 0000000000..6f0494fbdd --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestFileOutErr.java @@ -0,0 +1,127 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.util.io.RecordingOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * An implementation of the FileOutErr that doesn't use a file. + * This is useful for tests, as they often test the action directly + * and would otherwise have to create files on the vfs. + */ +public class TestFileOutErr extends FileOutErr { + + RecordingOutErr recorder; + + public TestFileOutErr(TestFileOutErr arg) { + this(arg.getOutputStream(), arg.getErrorStream()); + } + + public TestFileOutErr() { + this(new ByteArrayOutputStream(), new ByteArrayOutputStream()); + } + + public TestFileOutErr(ByteArrayOutputStream stream) { + super(null, null); // This is a pretty brutal overloading - We're just inheriting for the type. + recorder = new RecordingOutErr(stream, stream); + } + + public TestFileOutErr(ByteArrayOutputStream stream1, ByteArrayOutputStream stream2) { + super(null, null); // This is a pretty brutal overloading - We're just inheriting for the type. + recorder = new RecordingOutErr(stream1, stream2); + } + + + @Override + public Path getOutputFile() { + return null; + } + + @Override + public Path getErrorFile() { + return null; + } + + @Override + public ByteArrayOutputStream getOutputStream() { + return recorder.getOutputStream(); + } + + @Override + public ByteArrayOutputStream getErrorStream() { + return recorder.getErrorStream(); + } + + @Override + public void printOut(String s) { + recorder.printOut(s); + } + + @Override + public void printErr(String s) { + recorder.printErr(s); + } + + @Override + public String toString() { + return recorder.toString(); + } + + @Override + public boolean hasRecordedOutput() { + return recorder.hasRecordedOutput(); + } + + @Override + public String outAsLatin1() { + return recorder.outAsLatin1(); + } + + @Override + public String errAsLatin1() { + return recorder.errAsLatin1(); + } + + @Override + public void dumpOutAsLatin1(OutputStream out) { + try { + out.write(recorder.getOutputStream().toByteArray()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void dumpErrAsLatin1(OutputStream out) { + try { + out.write(recorder.getErrorStream().toByteArray()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String getRecordedOutput() { + return recorder.outAsLatin1() + recorder.errAsLatin1(); + } + + public void reset() { + recorder.reset(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestRuleClassProvider.java b/src/test/java/com/google/devtools/build/lib/testutil/TestRuleClassProvider.java new file mode 100644 index 0000000000..752605cbd7 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestRuleClassProvider.java @@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.INTEGER; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.OUTPUT_LIST; +import static com.google.devtools.build.lib.packages.Type.STRING_LIST; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.lang.reflect.Method; + +/** + * Helper class to provide a RuleClassProvider for tests. + */ +public class TestRuleClassProvider { + private static ConfiguredRuleClassProvider ruleProvider = null; + + /** + * Adds all the rule classes supported internally within the build tool to the given builder. + */ + public static void addStandardRules(ConfiguredRuleClassProvider.Builder builder) { + try { + Class<?> providerClass = Class.forName(TestConstants.TEST_RULE_CLASS_PROVIDER); + Method setupMethod = providerClass.getMethod("setup", + ConfiguredRuleClassProvider.Builder.class); + setupMethod.invoke(null, builder); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + /** + * Return a rule class provider. + */ + public static ConfiguredRuleClassProvider getRuleClassProvider() { + if (ruleProvider == null) { + ConfiguredRuleClassProvider.Builder builder = + new ConfiguredRuleClassProvider.Builder(); + addStandardRules(builder); + builder.addRuleDefinition(TestingDummyRule.class); + ruleProvider = builder.build(); + } + return ruleProvider; + } + + @BlazeRule(name = "testing_dummy_rule", + ancestors = { BaseRuleClasses.RuleBase.class }, + // Instantiated only in tests + factoryClass = UnknownRuleConfiguredTarget.class) + public static final class TestingDummyRule implements RuleDefinition { + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setUndocumented() + .add(attr("srcs", LABEL_LIST).allowedFileTypes(FileTypeSet.ANY_FILE)) + .add(attr("outs", OUTPUT_LIST)) + .add(attr("dummystrings", STRING_LIST)) + .add(attr("dummyinteger", INTEGER)) + .build(); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestSpec.java b/src/test/java/com/google/devtools/build/lib/testutil/TestSpec.java new file mode 100644 index 0000000000..316169c10b --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestSpec.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation class which we use to attach a little meta data to test + * classes. For now, we use this to attach a {@link Suite}. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface TestSpec { + + /** + * The size of the specified test, in terms of its resource consumption and + * execution time. + */ + Suite size() default Suite.SMALL_TESTS; + + /** + * The name of the suite to which this test belongs. Useful for creating + * test suites organised by function. + */ + String suite() default ""; + + /** + * If the test will pass consistently without outside changes. + * This should be fixed as soon as possible. + */ + boolean flaky() default false; +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestSuiteBuilder.java b/src/test/java/com/google/devtools/build/lib/testutil/TestSuiteBuilder.java new file mode 100644 index 0000000000..af90c525ed --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestSuiteBuilder.java @@ -0,0 +1,139 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutil; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; + +import junit.framework.TestCase; + +import org.junit.runner.RunWith; + +import java.lang.reflect.Modifier; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A collector for test classes, for both JUnit 3 and 4. To be used in combination with {@link + * CustomSuite}. + */ +public final class TestSuiteBuilder { + + private Set<Class<?>> testClasses = Sets.newTreeSet(new TestClassNameComparator()); + private Predicate<Class<?>> matchClassPredicate = Predicates.alwaysTrue(); + + /** + * Adds the tests found (directly) in class {@code c} to the set of tests + * this builder will search. + */ + public TestSuiteBuilder addTestClass(Class<?> c) { + testClasses.add(c); + return this; + } + + /** + * Adds all the test classes (top-level or nested) found in package + * {@code pkgName} or its subpackages to the set of tests this builder will + * search. + */ + public TestSuiteBuilder addPackageRecursive(String pkgName) { + for (Class<?> c : getClassesRecursive(pkgName)) { + addTestClass(c); + } + return this; + } + + private Set<Class<?>> getClassesRecursive(String pkgName) { + Set<Class<?>> result = new LinkedHashSet<>(); + for (Class<?> clazz : Classpath.findClasses(pkgName)) { + if (isTestClass(clazz)) { + result.add(clazz); + } + } + return result; + } + + /** + * Specifies a predicate returns false for classes we want to exclude. + */ + public TestSuiteBuilder matchClasses(Predicate<Class<?>> predicate) { + matchClassPredicate = predicate; + return this; + } + + /** + * Creates and returns a TestSuite containing the tests from the given + * classes and/or packages which matched the given tags. + */ + public Set<Class<?>> create() { + Set<Class<?>> result = new LinkedHashSet<>(); + // We have some cases where the resulting test suite is empty, which some of our test + // infrastructure treats as an error. + result.add(TautologyTest.class); + for (Class<?> testClass : Iterables.filter(testClasses, matchClassPredicate)) { + result.add(testClass); + } + return result; + } + + /** + * Determines if a given class is a test class. + * + * @param container class to test + * @return <code>true</code> if the test is a test class. + */ + private static boolean isTestClass(Class<?> container) { + return (isJunit4Test(container) || isJunit3Test(container)) + && !isSuite(container) + && Modifier.isPublic(container.getModifiers()) + && !Modifier.isAbstract(container.getModifiers()); + } + + private static boolean isJunit4Test(Class<?> container) { + return container.getAnnotation(RunWith.class) != null; + } + + private static boolean isJunit3Test(Class<?> container) { + return TestCase.class.isAssignableFrom(container); + } + + /** + * Classes that have a {@code RunWith} annotation for {@link ClasspathSuite} or {@link + * CustomSuite} are automatically excluded to avoid picking up the suite class itself. + */ + private static boolean isSuite(Class<?> container) { + RunWith runWith = container.getAnnotation(RunWith.class); + return (runWith != null) + && ((runWith.value() == ClasspathSuite.class) || (runWith.value() == CustomSuite.class)); + } + + private static class TestClassNameComparator implements Comparator<Class<?>> { + @Override + public int compare(Class<?> o1, Class<?> o2) { + return o1.getName().compareTo(o2.getName()); + } + } + + /** + * A test that does nothing and always passes. We have some cases where an empty test suite is + * treated as an error, so we use this test to make sure that the test suite is always non-empty. + */ + public static class TautologyTest extends TestCase { + public void testThatNothingHappens() { + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestThread.java b/src/test/java/com/google/devtools/build/lib/testutil/TestThread.java new file mode 100644 index 0000000000..e04302566e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestThread.java @@ -0,0 +1,66 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import static com.google.common.truth.Truth.assertWithMessage; + +/** + * Test thread implementation that allows the use of assertions within + * spawned threads. + * + * Main test method must call {@link TestThread#joinAndAssertState(long)} + * for each spawned test thread. + */ +public abstract class TestThread extends Thread { + Throwable testException = null; + boolean isSucceeded = false; + + /** + * Specific test thread implementation overrides this method. + */ + abstract public void runTest() throws Exception; + + @Override public final void run() { + try { + runTest(); + isSucceeded = true; + } catch (Exception e) { + testException = e; + } catch (AssertionError e) { + testException = e; + } + } + + /** + * Joins test thread (waiting specified number of ms) and validates that + * it has been completed successfully. + */ + public void joinAndAssertState(long timeout) throws InterruptedException { + join(timeout); + Throwable exception = this.testException; + if (isAlive()) { + exception = new AssertionError ( + "Test thread " + getName() + " is still alive"); + exception.setStackTrace(getStackTrace()); + } + if(exception != null) { + AssertionError error = new AssertionError("Test thread " + getName() + " failed to execute"); + error.initCause(exception); + throw error; + } + assertWithMessage("Test thread " + getName() + " has not run successfully").that(isSucceeded) + .isTrue(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestUtils.java b/src/test/java/com/google/devtools/build/lib/testutil/TestUtils.java new file mode 100644 index 0000000000..2ee59721c8 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestUtils.java @@ -0,0 +1,152 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Some static utility functions for testing. + */ +public class TestUtils { + public static final ThreadPoolExecutor POOL = + (ThreadPoolExecutor) Executors.newFixedThreadPool(10); + + public static final UUID ZERO_UUID = UUID.fromString("00000000-0000-0000-0000-000000000000"); + + /** + * Wait until the {@link System#currentTimeMillis} / 1000 advances. + * + * This method takes 0-1000ms to run, 500ms on average. + */ + public static void advanceCurrentTimeSeconds() throws InterruptedException { + long currentTimeSeconds = System.currentTimeMillis() / 1000; + do { + Thread.sleep(50); + } while (currentTimeSeconds == System.currentTimeMillis() / 1000); + } + + public static ThreadPoolExecutor getPool() { + return POOL; + } + + public static String tmpDir() { + return tmpDirFile().getAbsolutePath(); + } + + static String getUserValue(String key) { + String value = System.getProperty(key); + if (value == null) { + value = System.getenv(key); + } + return value; + } + + public static File tmpDirFile() { + File tmpDir; + + // Flag value specified in environment? + String tmpDirStr = getUserValue("TEST_TMPDIR"); + + if (tmpDirStr != null && tmpDirStr.length() > 0) { + tmpDir = new File(tmpDirStr); + } else { + // Fallback default $TEMP/$USER/tmp/$TESTNAME + String baseTmpDir = System.getProperty("java.io.tmpdir"); + tmpDir = new File(baseTmpDir).getAbsoluteFile(); + + // .. Add username + String username = System.getProperty("user.name"); + username = username.replace('/', '_'); + username = username.replace('\\', '_'); + username = username.replace('\000', '_'); + tmpDir = new File(tmpDir, username); + tmpDir = new File(tmpDir, "tmp"); + } + + // Ensure tmpDir exists + if (!tmpDir.isDirectory()) { + tmpDir.mkdirs(); + } + return tmpDir; + } + + public static File makeTempDir() throws IOException { + File dir = File.createTempFile(TestUtils.class.getName(), ".temp", tmpDirFile()); + if (!dir.delete()) { + throw new IOException("Cannot remove a temporary file " + dir); + } + if (!dir.mkdir()) { + throw new IOException("Cannot create a temporary directory " + dir); + } + return dir; + } + + public static int getRandomSeed() { + // Default value if not running under framework + int randomSeed = 301; + + // Value specified in environment by framework? + String value = getUserValue("TEST_RANDOM_SEED"); + if ((value != null) && (value.length() > 0)) { + try { + randomSeed = Integer.parseInt(value); + } catch (NumberFormatException e) { + // throw new AssertionError("TEST_RANDOM_SEED must be an integer"); + throw new RuntimeException("TEST_RANDOM_SEED must be an integer"); + } + } + + return randomSeed; + } + + public static byte[] serializeObject(Object obj) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (ObjectOutputStream objectStream = new ObjectOutputStream(outputStream)) { + objectStream.writeObject(obj); + } + return outputStream.toByteArray(); + } + + public static Object deserializeObject(byte[] buf) throws IOException, ClassNotFoundException { + try (ObjectInputStream inStream = new ObjectInputStream(new ByteArrayInputStream(buf))) { + return inStream.readObject(); + } + } + + /** + * Timeouts for asserting that an arbitrary event occurs eventually. + * + * <p>In general, it's not appropriate to use a small constant timeout for an arbitrary + * computation since there is no guarantee that a snippet of code will execute within a given + * amount of time - you are at the mercy of the jvm, your machine, and your OS. In theory we + * could try to take all of these factors into account but instead we took the simpler and + * obviously correct approach of not having timeouts. + * + * <p>If a test that uses these timeout values is failing due to a "timeout" at the + * 'blaze test' level, it could be because of a legitimate deadlock that would have been caught + * if the timeout values below were small. So you can rule out such a deadlock by changing these + * values to small numbers (also note that the --test_timeout blaze flag may be useful). + */ + public static final long WAIT_TIMEOUT_MILLISECONDS = Long.MAX_VALUE; + public static final long WAIT_TIMEOUT_SECONDS = WAIT_TIMEOUT_MILLISECONDS / 1000; +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/UnknownRuleConfiguredTarget.java b/src/test/java/com/google/devtools/build/lib/testutil/UnknownRuleConfiguredTarget.java new file mode 100644 index 0000000000..d3e5457a67 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/UnknownRuleConfiguredTarget.java @@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.FailAction; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * A null implementation of ConfiguredTarget for rules we don't know how to build. + */ +public class UnknownRuleConfiguredTarget implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext context) { + // TODO(bazel-team): (2009) why isn't this an error? It would stop the build more promptly... + context.ruleWarning("cannot build " + context.getRule().getRuleClass() + " rules"); + + ImmutableList<Artifact> outputArtifacts = context.getOutputArtifacts(); + NestedSet<Artifact> filesToBuild; + if (outputArtifacts.isEmpty()) { + // Gotta build *something*... + filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, + context.createOutputArtifact()); + } else { + filesToBuild = NestedSetBuilder.wrap(Order.STABLE_ORDER, outputArtifacts); + } + + Rule rule = context.getRule(); + context.registerAction(new FailAction(context.getActionOwner(), + filesToBuild, "cannot build " + rule.getRuleClass() + " rules such as " + rule.getLabel())); + return new RuleConfiguredTargetBuilder(context) + .setFilesToBuild(filesToBuild) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)) + .build(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutiltests/JunitTestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/testutiltests/JunitTestUtilsTest.java new file mode 100644 index 0000000000..4bfe0faf37 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutiltests/JunitTestUtilsTest.java @@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutiltests; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.testutil.JunitTestUtils.assertContainsSublist; +import static com.google.devtools.build.lib.testutil.JunitTestUtils.assertDoesNotContainSublist; +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.List; + +/** + * Tests {@link com.google.devtools.build.lib.testutil.JunitTestUtils}. + */ +@RunWith(JUnit4.class) +public class JunitTestUtilsTest { + + @Test + public void testAssertContainsSublistSuccess() { + List<String> actual = Arrays.asList("a", "b", "c"); + + // All single-string combinations. + assertContainsSublist(actual, "a"); + assertContainsSublist(actual, "b"); + assertContainsSublist(actual, "c"); + + // All two-string combinations. + assertContainsSublist(actual, "a", "b"); + assertContainsSublist(actual, "b", "c"); + + // The whole list. + assertContainsSublist(actual, "a", "b", "c"); + } + + @Test + public void testAssertContainsSublistFailure() { + List<String> actual = Arrays.asList("a", "b", "c"); + + try { + assertContainsSublist(actual, "d"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e.getMessage()).startsWith("Did not find [d] as a sublist of [a, b, c]"); + } + + try { + assertContainsSublist(actual, "a", "c"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e.getMessage()).startsWith("Did not find [a, c] as a sublist of [a, b, c]"); + } + + try { + assertContainsSublist(actual, "b", "c", "d"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e.getMessage()).startsWith("Did not find [b, c, d] as a sublist of [a, b, c]"); + } + } + + @Test + public void testAssertDoesNotContainSublistSuccess() { + List<String> actual = Arrays.asList("a", "b", "c"); + assertDoesNotContainSublist(actual, "d"); + assertDoesNotContainSublist(actual, "a", "c"); + assertDoesNotContainSublist(actual, "b", "c", "d"); + } + + @Test + public void testAssertDoesNotContainSublistFailure() { + List<String> actual = Arrays.asList("a", "b", "c"); + + // All single-string combinations. + try { + assertDoesNotContainSublist(actual, "a"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e).hasMessage("Found [a] as a sublist of [a, b, c]"); + } + try { + assertDoesNotContainSublist(actual, "b"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e).hasMessage("Found [b] as a sublist of [a, b, c]"); + } + try { + assertDoesNotContainSublist(actual, "c"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e).hasMessage("Found [c] as a sublist of [a, b, c]"); + } + + // All two-string combinations. + try { + assertDoesNotContainSublist(actual, "a", "b"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e).hasMessage("Found [a, b] as a sublist of [a, b, c]"); + } + try { + assertDoesNotContainSublist(actual, "b", "c"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e).hasMessage("Found [b, c] as a sublist of [a, b, c]"); + } + + // The whole list. + try { + assertDoesNotContainSublist(actual, "a", "b", "c"); + fail("no exception thrown"); + } catch (AssertionError e) { + assertThat(e).hasMessage("Found [a, b, c] as a sublist of [a, b, c]"); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/testutiltests/TestSizeAnnotationTest.java b/src/test/java/com/google/devtools/build/lib/testutiltests/TestSizeAnnotationTest.java new file mode 100644 index 0000000000..5380cba7bd --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutiltests/TestSizeAnnotationTest.java @@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.testutiltests; + +import static com.google.devtools.build.lib.testutil.Suite.getSize; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.devtools.build.lib.testutil.Suite; +import com.google.devtools.build.lib.testutil.TestSpec; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests {@link com.google.devtools.build.lib.testutil.Suite#getSize(Class)}. + */ +@RunWith(JUnit4.class) +public class TestSizeAnnotationTest { + + private static class HasNoTestSpecAnnotation { + + } + + @TestSpec(flaky = true) + private static class FlakyTestSpecAnnotation { + + } + + @TestSpec(suite = "foo") + private static class HasNoSizeAnnotationElement { + + } + + @TestSpec(size = Suite.SMALL_TESTS) + private static class IsAnnotatedWithSmallSize { + + } + + @TestSpec(size = Suite.MEDIUM_TESTS) + private static class IsAnnotatedWithMediumSize { + + } + + @TestSpec(size = Suite.LARGE_TESTS) + private static class IsAnnotatedWithLargeSize { + + } + + private static class SuperclassHasAnnotationButNoSizeElement + extends HasNoSizeAnnotationElement { + + } + + @TestSpec(size = Suite.LARGE_TESTS) + private static class HasSizeElementAndSuperclassHasAnnotationButNoSizeElement + extends HasNoSizeAnnotationElement { + + } + + private static class SuperclassHasAnnotationWithSizeElement + extends IsAnnotatedWithSmallSize { + + } + + @TestSpec(size = Suite.LARGE_TESTS) + private static class HasSizeElementAndSuperclassHasAnnotationWithSizeElement + extends IsAnnotatedWithSmallSize { + + } + + @Test + public void testHasNoTestSpecAnnotationIsSmall() { + assertEquals(Suite.SMALL_TESTS, getSize(HasNoTestSpecAnnotation.class)); + } + + @Test + public void testHasNoSizeAnnotationElementIsSmall() { + assertEquals(Suite.SMALL_TESTS, getSize(HasNoSizeAnnotationElement.class)); + } + + @Test + public void testIsAnnotatedWithSmallSizeIsSmall() { + assertEquals(Suite.SMALL_TESTS, getSize(IsAnnotatedWithSmallSize.class)); + } + + @Test + public void testIsAnnotatedWithMediumSizeIsMedium() { + assertEquals(Suite.MEDIUM_TESTS, getSize(IsAnnotatedWithMediumSize.class)); + } + + @Test + public void testIsAnnotatedWithLargeSizeIsLarge() { + assertEquals(Suite.LARGE_TESTS, getSize(IsAnnotatedWithLargeSize.class)); + } + + @Test + public void testSuperclassHasAnnotationButNoSizeElement() { + assertEquals(Suite.SMALL_TESTS, getSize(SuperclassHasAnnotationButNoSizeElement.class)); + } + + @Test + public void testHasSizeElementAndSuperclassHasAnnotationButNoSizeElement() { + assertEquals(Suite.LARGE_TESTS, + getSize(HasSizeElementAndSuperclassHasAnnotationButNoSizeElement.class)); + } + + @Test + public void testSuperclassHasAnnotationWithSizeElement() { + assertEquals(Suite.SMALL_TESTS, getSize(SuperclassHasAnnotationWithSizeElement.class)); + } + + @Test + public void testHasSizeElementAndSuperclassHasAnnotationWithSizeElement() { + assertEquals(Suite.LARGE_TESTS, + getSize(HasSizeElementAndSuperclassHasAnnotationWithSizeElement.class)); + } + + @Test + public void testIsNotFlaky() { + assertFalse(Suite.isFlaky(HasNoTestSpecAnnotation.class)); + } + + @Test + public void testIsFlaky() { + assertTrue(Suite.isFlaky(FlakyTestSpecAnnotation.class)); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/unix/FilesystemUtilsTest.java b/src/test/java/com/google/devtools/build/lib/unix/FilesystemUtilsTest.java new file mode 100644 index 0000000000..48a67c12df --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/unix/FilesystemUtilsTest.java @@ -0,0 +1,76 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.unix; + +import static org.junit.Assert.assertEquals; + +import com.google.common.hash.HashCode; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.UnixFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.util.HashMap; + +/** + * This class tests the FilesystemUtils class. + */ +@RunWith(JUnit4.class) +public class FilesystemUtilsTest { + private FileSystem testFS; + private Path workingDir; + private Path testFile; + + @Before + public void setUp() throws Exception { + testFS = new UnixFileSystem(); + workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath()); + testFile = workingDir.getRelative("test"); + FileSystemUtils.createEmptyFile(testFile); + } + + /** + * This test validates that the md5sum() method returns hashes that match the official test + * vectors specified in RFC 1321, The MD5 Message-Digest Algorithm. + * + * @throws Exception + */ + @Test + public void testValidateMd5Sum() throws Exception { + HashMap<String, String> testVectors = new HashMap<String, String>(); + testVectors.put("", "d41d8cd98f00b204e9800998ecf8427e"); + testVectors.put("a", "0cc175b9c0f1b6a831c399e269772661"); + testVectors.put("abc", "900150983cd24fb0d6963f7d28e17f72"); + testVectors.put("message digest", "f96b697d7cb7938d525a2f31aaf161d0"); + testVectors.put("abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b"); + testVectors.put("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f"); + testVectors.put( + "12345678901234567890123456789012345678901234567890123456789012345678901234567890", + "57edf4a22be3c955ac49da2e2107b67a"); + + for (String testInput : testVectors.keySet()) { + FileSystemUtils.writeContentAsLatin1(testFile, testInput); + HashCode result = FilesystemUtils.md5sum(testFile.getPathString()); + assertEquals(result.toString(), testVectors.get(testInput)); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStreamTest.java new file mode 100644 index 0000000000..41428cbd55 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/AnsiStrippingOutputStreamTest.java @@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +/** + * Tests for {@link AnsiStrippingOutputStream}. + */ +@RunWith(JUnit4.class) +public class AnsiStrippingOutputStreamTest { + ByteArrayOutputStream output; + PrintStream input; + + private static final String ESCAPE = "\u001b["; + + @Before + public void setUp() throws Exception { + output = new ByteArrayOutputStream(); + OutputStream inputStream = new AnsiStrippingOutputStream(output); + input = new PrintStream(inputStream); + } + + private String getOutput(String... fragments) throws Exception { + for (String fragment: fragments) { + input.print(fragment); + } + + return new String(output.toByteArray(), "ISO8859-1"); + } + + @Test + public void doesNotFailHorribly() throws Exception { + assertEquals("Love", getOutput("Love")); + } + + @Test + public void canStripAnsiCode() throws Exception { + assertEquals("Love", getOutput(ESCAPE + "32mLove" + ESCAPE + "m")); + } + + @Test + public void recognizesAnsiCodeWhenBrokenUp() throws Exception { + assertEquals("Love", getOutput("\u001b", "[", "mLove")); + } + + @Test + public void handlesOnlyEscCorrectly() throws Exception { + assertEquals("\u001bLove", getOutput("\u001bLove")); + } + + @Test + public void handlesEscInPlaceOfControlCharCorrectly() throws Exception { + assertEquals(ESCAPE + "31;42Love", + getOutput(ESCAPE + "31;42" + ESCAPE + "1mLove")); + } + + @Test + public void handlesTwoEscapeSequencesCorrectly() throws Exception { + assertEquals("Love", + getOutput(ESCAPE + "32m" + ESCAPE + "1m" + "Love")); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/CommandBuilderTest.java b/src/test/java/com/google/devtools/build/lib/util/CommandBuilderTest.java new file mode 100644 index 0000000000..566da258a3 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/CommandBuilderTest.java @@ -0,0 +1,104 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; + +/** + * Tests for the {@link CommandBuilder} class. + */ +@RunWith(JUnit4.class) +public class CommandBuilderTest { + + private CommandBuilder linuxBuilder() { + return new CommandBuilder(OS.LINUX).useTempDir(); + } + + private CommandBuilder winBuilder() { + return new CommandBuilder(OS.WINDOWS).useTempDir(); + } + + private void assertArgv(CommandBuilder builder, String... expected) { + assertThat(Arrays.asList(builder.build().getCommandLineElements())).containsExactlyElementsIn( + Arrays.asList(expected)).inOrder(); + } + + private void assertWinCmdArgv(CommandBuilder builder, String expected) { + assertArgv(builder, "CMD.EXE", "/S", "/E:ON", "/V:ON", "/D", "/C", "\"" + expected + "\""); + } + + private void assertFailure(CommandBuilder builder, String expected) { + try { + builder.build(); + fail("Expected exception"); + } catch (Exception e) { + assertEquals(expected, e.getMessage()); + } + } + + @Test + public void linuxBuilderTest() { + assertArgv(linuxBuilder().addArg("abc"), "abc"); + assertArgv(linuxBuilder().addArg("abc def"), "abc def"); + assertArgv(linuxBuilder().addArgs("abc", "def"), "abc", "def"); + assertArgv(linuxBuilder().addArgs(ImmutableList.of("abc", "def")), "abc", "def"); + assertArgv(linuxBuilder().addArg("abc").useShell(true), "/bin/sh", "-c", "abc"); + assertArgv(linuxBuilder().addArg("abc def").useShell(true), "/bin/sh", "-c", "abc def"); + assertArgv(linuxBuilder().addArgs("abc", "def").useShell(true), "/bin/sh", "-c", "abc def"); + assertArgv(linuxBuilder().addArgs("/bin/sh", "-c", "abc").useShell(true), + "/bin/sh", "-c", "abc"); + assertArgv(linuxBuilder().addArgs("/bin/sh", "-c"), "/bin/sh", "-c"); + assertArgv(linuxBuilder().addArgs("/bin/bash", "-c"), "/bin/bash", "-c"); + assertArgv(linuxBuilder().addArgs("/bin/sh", "-c").useShell(true), "/bin/sh", "-c"); + assertArgv(linuxBuilder().addArgs("/bin/bash", "-c").useShell(true), "/bin/bash", "-c"); + } + + @Test + public void windowsBuilderTest() { + assertArgv(winBuilder().addArg("abc.exe"), "abc.exe"); + assertArgv(winBuilder().addArg("abc.exe -o"), "abc.exe -o"); + assertArgv(winBuilder().addArg("ABC.EXE"), "ABC.EXE"); + assertWinCmdArgv(winBuilder().addArg("abc def.exe"), "abc def.exe"); + assertArgv(winBuilder().addArgs("abc.exe", "def"), "abc.exe", "def"); + assertArgv(winBuilder().addArgs(ImmutableList.of("abc.exe", "def")), "abc.exe", "def"); + assertWinCmdArgv(winBuilder().addArgs("abc.exe", "def").useShell(true), "abc.exe def"); + assertWinCmdArgv(winBuilder().addArg("abc"), "abc"); + assertWinCmdArgv(winBuilder().addArgs("abc", "def"), "abc def"); + assertWinCmdArgv(winBuilder().addArgs("/bin/sh", "-c", "abc", "def"), "abc def"); + assertWinCmdArgv(winBuilder().addArgs("/bin/sh", "-c"), ""); + assertWinCmdArgv(winBuilder().addArgs("/bin/bash", "-c"), ""); + assertWinCmdArgv(winBuilder().addArgs("/bin/sh", "-c").useShell(true), ""); + assertWinCmdArgv(winBuilder().addArgs("/bin/bash", "-c").useShell(true), ""); + } + + @Test + public void failureScenarios() { + assertFailure(linuxBuilder(), "At least one argument is expected"); + assertFailure(new CommandBuilder(OS.UNKNOWN).useTempDir().addArg("a"), + "Unidentified operating system"); + assertFailure(new CommandBuilder(OS.LINUX).addArg("a"), + "Working directory must be set"); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/CommandFailureUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/CommandFailureUtilsTest.java new file mode 100644 index 0000000000..f0c2c4a7ec --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/CommandFailureUtilsTest.java @@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +@RunWith(JUnit4.class) +public class CommandFailureUtilsTest { + + @Test + public void describeCommandError() throws Exception { + String[] args = new String[40]; + args[0] = "some_command"; + for (int i = 1; i < args.length; i++) { + args[i] = "arg" + i; + } + args[7] = "with spaces"; // Test embedded spaces in argument. + args[9] = "*"; // Test shell meta characters. + Map<String, String> env = new HashMap<>(); + env.put("PATH", "/usr/bin:/bin:/sbin"); + env.put("FOO", "foo"); + String cwd = "/my/working/directory"; + String message = CommandFailureUtils.describeCommandError(false, Arrays.asList(args), env, cwd); + String verboseMessage = CommandFailureUtils.describeCommandError(true, Arrays.asList(args), env, + cwd); + assertEquals( + "error executing command some_command arg1 " + + "arg2 arg3 arg4 arg5 arg6 'with spaces' arg8 '*' arg10 " + + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 " + + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 " + + "arg27 arg28 arg29 arg30 arg31 " + + "... (remaining 8 argument(s) skipped)", + message); + assertEquals( + "error executing command \n" + + " (cd /my/working/directory && \\\n" + + " exec env - \\\n" + + " FOO=foo \\\n" + + " PATH=/usr/bin:/bin:/sbin \\\n" + + " some_command arg1 arg2 arg3 arg4 arg5 arg6 'with spaces' arg8 '*' arg10 " + + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 " + + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 " + + "arg27 arg28 arg29 arg30 arg31 arg32 arg33 arg34 " + + "arg35 arg36 arg37 arg38 arg39)", + verboseMessage); + } + + @Test + public void describeCommandFailure() throws Exception { + String[] args = new String[3]; + args[0] = "/bin/sh"; + args[1] = "-c"; + args[2] = "echo Some errors 1>&2; echo Some output; exit 42"; + Map<String, String> env = new HashMap<>(); + env.put("FOO", "foo"); + env.put("PATH", "/usr/bin:/bin:/sbin"); + String cwd = null; + String message = CommandFailureUtils.describeCommandFailure(false, Arrays.asList(args), + env, cwd); + String verboseMessage = CommandFailureUtils.describeCommandFailure(true, Arrays.asList(args), + env, cwd); + assertEquals( + "sh failed: error executing command " + + "/bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42'", + message); + assertEquals( + "sh failed: error executing command \n" + + " (exec env - \\\n" + + " FOO=foo \\\n" + + " PATH=/usr/bin:/bin:/sbin \\\n" + + " /bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42')", + verboseMessage); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/CommandUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/CommandUtilsTest.java new file mode 100644 index 0000000000..c7639a2be4 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/CommandUtilsTest.java @@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.util.Map; + +@RunWith(JUnit4.class) +public class CommandUtilsTest { + + @Test + public void longCommand() throws Exception { + String[] args = new String[40]; + args[0] = "this_command_will_not_be_found"; + for (int i = 1; i < args.length; i++) { + args[i] = "arg" + i; + } + Map<String, String> env = Maps.newTreeMap(); + env.put("PATH", "/usr/bin:/bin:/sbin"); + env.put("FOO", "foo"); + File directory = new File("/tmp"); + try { + new Command(args, env, directory).execute(); + fail(); + } catch (CommandException exception) { + String message = CommandUtils.describeCommandError(false, exception.getCommand()); + String verboseMessage = CommandUtils.describeCommandError(true, exception.getCommand()); + assertEquals( + "error executing command this_command_will_not_be_found arg1 " + + "arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 " + + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 " + + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 " + + "arg27 arg28 arg29 arg30 " + + "... (remaining 9 argument(s) skipped)", + message); + assertEquals( + "error executing command \n" + + " (cd /tmp && \\\n" + + " exec env - \\\n" + + " FOO=foo \\\n" + + " PATH=/usr/bin:/bin:/sbin \\\n" + + " this_command_will_not_be_found arg1 " + + "arg2 arg3 arg4 arg5 arg6 arg7 arg8 arg9 arg10 " + + "arg11 arg12 arg13 arg14 arg15 arg16 arg17 arg18 " + + "arg19 arg20 arg21 arg22 arg23 arg24 arg25 arg26 " + + "arg27 arg28 arg29 arg30 arg31 arg32 arg33 arg34 " + + "arg35 arg36 arg37 arg38 arg39)", + verboseMessage); + } + } + + @Test + public void failingCommand() throws Exception { + String[] args = new String[3]; + args[0] = "/bin/sh"; + args[1] = "-c"; + args[2] = "echo Some errors 1>&2; echo Some output; exit 42"; + Map<String, String> env = Maps.newTreeMap(); + env.put("FOO", "foo"); + env.put("PATH", "/usr/bin:/bin:/sbin"); + try { + new Command(args, env, null).execute(); + fail(); + } catch (CommandException exception) { + String message = CommandUtils.describeCommandFailure(false, exception); + String verboseMessage = CommandUtils.describeCommandFailure(true, exception); + assertEquals( + "sh failed: error executing command " + + "/bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42': " + + "Process exited with status 42\n" + + "Some output\n" + + "Some errors\n", + message); + assertEquals( + "sh failed: error executing command \n" + + " (exec env - \\\n" + + " FOO=foo \\\n" + + " PATH=/usr/bin:/bin:/sbin \\\n" + + " /bin/sh -c 'echo Some errors 1>&2; echo Some output; exit 42'): " + + "Process exited with status 42\n" + + "Some output\n" + + "Some errors\n", + verboseMessage); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/DependencySetTest.java b/src/test/java/com/google/devtools/build/lib/util/DependencySetTest.java new file mode 100644 index 0000000000..40edf36b03 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/DependencySetTest.java @@ -0,0 +1,230 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Collection; + +@RunWith(JUnit4.class) +public class DependencySetTest { + + private FsApparatus scratch = FsApparatus.newInMemory(); + + private DependencySet newDependencySet() { + return new DependencySet(scratch.fs().getRootDirectory()); + } + + @Test + public void dotDParser_simple() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + String filename = "hello.o"; + Path dotd = scratch.file("/tmp/foo.d", + filename + ": \\", + " " + file1 + " \\", + " " + file2 + " "); + DependencySet depset = newDependencySet().read(dotd); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2), + depset.getDependencies()); + assertEquals(depset.getOutputFileName(), filename); + } + + @Test + public void dotDParser_simple_crlf() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + String filename = "hello.o"; + Path dotd = scratch.file("/tmp/foo.d", + filename + ": \\\r", + " " + file1 + " \\\r", + " " + file2 + " "); + DependencySet depset = newDependencySet().read(dotd); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2), + depset.getDependencies()); + assertEquals(depset.getOutputFileName(), filename); + } + + @Test + public void dotDParser_simple_cr() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + String filename = "hello.o"; + Path dotd = scratch.file("/tmp/foo.d", + filename + ": \\\r" + + " " + file1 + " \\\r" + + " " + file2 + " "); + DependencySet depset = newDependencySet().read(dotd); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2), + depset.getDependencies()); + assertEquals(depset.getOutputFileName(), filename); + } + + @Test + public void dotDParser_leading_crlf() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + String filename = "hello.o"; + Path dotd = scratch.file("/tmp/foo.d", + "\r\n" + filename + ": \\\r\n" + + " " + file1 + " \\\r\n" + + " " + file2 + " "); + DependencySet depset = newDependencySet().read(dotd); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2), + depset.getDependencies()); + assertEquals(depset.getOutputFileName(), filename); + } + + @Test + public void dotDParser_oddFormatting() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h"); + PathFragment file4 = new PathFragment("/usr/local/blah/blah/genhello/onemore.h"); + String filename = "hello.o"; + Path dotd = scratch.file("/tmp/foo.d", + filename + ": " + file1 + " \\", + " " + file2 + "\\", + " " + file3 + " " + file4); + DependencySet depset = newDependencySet().read(dotd); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2, file3, file4), + depset.getDependencies()); + assertEquals(depset.getOutputFileName(), filename); + } + + @Test + public void dotDParser_relativeFilenames() throws Exception { + PathFragment file1 = new PathFragment("hello.cc"); + PathFragment file2 = new PathFragment("hello.h"); + String filename = "hello.o"; + Path dotd = scratch.file("/tmp/foo.d", + filename + ": \\", + " " + file1 + " \\", + " " + file2 + " "); + DependencySet depset = newDependencySet().read(dotd); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2), + depset.getDependencies()); + assertEquals(depset.getOutputFileName(), filename); + } + + @Test + public void dotDParser_emptyFile() throws Exception { + Path dotd = scratch.file("/tmp/empty.d"); + DependencySet depset = newDependencySet().read(dotd); + Collection<PathFragment> headers = depset.getDependencies(); + if (!headers.isEmpty()) { + fail("Not empty: " + headers.size() + " " + headers); + } + assertEquals(depset.getOutputFileName(), null); + } + + @Test + public void dotDParser_multipleTargets() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + Path dotd = scratch.file("/tmp/foo.d", + "hello.o: \\", + " " + file1, + "hello2.o: \\", + " " + file2); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2), + newDependencySet().read(dotd).getDependencies()); + } + + /* + * Regression test: if gcc fails to execute remotely, and we retry locally, then the behavior + * of gcc's DEPENDENCIES_OUTPUT option is to append, not overwrite, the .d file. As a result, + * during retry, a second stanza is written to the file. + * + * We handle this by merging all of the stanzas. + */ + @Test + public void dotDParser_duplicateStanza() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h"); + Path dotd = scratch.file("/tmp/foo.d", + "hello.o: \\", + " " + file1 + " \\", + " " + file2 + " ", + "hello.o: \\", + " " + file1 + " \\", + " " + file3 + " "); + MoreAsserts.assertSameContents(Sets.newHashSet(file1, file2, file3), + newDependencySet().read(dotd).getDependencies()); + } + + @Test + public void writeSet() throws Exception { + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h"); + String filename = "/usr/local/blah/blah/genhello/hello.o"; + + DependencySet depSet1 = newDependencySet(); + depSet1.addDependency(file1); + depSet1.addDependency(file2); + depSet1.addDependency(file3); + depSet1.setOutputFileName(filename); + + Path outfile = scratch.path(filename); + Path dotd = scratch.path("/usr/local/blah/blah/genhello/hello.d"); + FileSystemUtils.createDirectoryAndParents(dotd.getParentDirectory()); + depSet1.write(outfile, ".d"); + + String dotdContents = new String(FileSystemUtils.readContentAsLatin1(dotd)); + String expected = + "usr/local/blah/blah/genhello/hello.o: \\\n" + + " /usr/local/blah/blah/genhello/hello.cc \\\n" + + " /usr/local/blah/blah/genhello/hello.h \\\n" + + " /usr/local/blah/blah/genhello/other.h\n"; + assertEquals(expected, dotdContents); + assertEquals(filename, depSet1.getOutputFileName()); + } + + @Test + public void writeReadSet() throws Exception { + String filename = "/usr/local/blah/blah/genhello/hello.d"; + PathFragment file1 = new PathFragment("/usr/local/blah/blah/genhello/hello.cc"); + PathFragment file2 = new PathFragment("/usr/local/blah/blah/genhello/hello.h"); + PathFragment file3 = new PathFragment("/usr/local/blah/blah/genhello/other.h"); + DependencySet depSet1 = newDependencySet(); + depSet1.addDependency(file1); + depSet1.addDependency(file2); + depSet1.addDependency(file3); + depSet1.setOutputFileName(filename); + + Path dotd = scratch.path(filename); + FileSystemUtils.createDirectoryAndParents(dotd.getParentDirectory()); + depSet1.write(dotd, ".d"); + + DependencySet depSet2 = newDependencySet().read(dotd); + assertEquals(depSet1, depSet2); + // due to how pic.d files are written, absolute paths are changed into relatives + assertEquals(depSet1.getOutputFileName(), "/" + depSet2.getOutputFileName()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/DependencySetWindowsTest.java b/src/test/java/com/google/devtools/build/lib/util/DependencySetWindowsTest.java new file mode 100644 index 0000000000..cdda6d182b --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/DependencySetWindowsTest.java @@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Set; + +@RunWith(JUnit4.class) +public class DependencySetWindowsTest { + + private FsApparatus scratch = FsApparatus.newInMemory(); + + private DependencySet newDependencySet() { + return new DependencySet(scratch.fs().getRootDirectory()); + } + + @Test + public void dotDParser_windowsPaths() throws Exception { + Path dotd = scratch.file("/tmp/foo.d", + "bazel-out/hello-lib/cpp/hello-lib.o: \\", + " cpp/hello-lib.cc cpp/hello-lib.h c:\\mingw\\include\\stdio.h \\", + " c:\\mingw\\include\\_mingw.h \\", + " c:\\mingw\\lib\\gcc\\mingw32\\4.8.1\\include\\stdarg.h"); + + Set<PathFragment> expected = Sets.newHashSet( + new PathFragment("cpp/hello-lib.cc"), + new PathFragment("cpp/hello-lib.h"), + new PathFragment("C:/mingw/include/stdio.h"), + new PathFragment("C:/mingw/include/_mingw.h"), + new PathFragment("C:/mingw/lib/gcc/mingw32/4.8.1/include/stdarg.h")); + + MoreAsserts.assertSameContents(expected, + newDependencySet().read(dotd).getDependencies()); + } + + @Test + public void dotDParser_windowsPathsWithSpaces() throws Exception { + Path dotd = scratch.file("/tmp/foo.d", + "bazel-out/hello-lib/cpp/hello-lib.o: \\", + "C:\\Program\\ Files\\ (x86)\\LLVM\\stddef.h"); + MoreAsserts.assertSameContents( + Sets.newHashSet(new PathFragment("C:/Program Files (x86)/LLVM/stddef.h")), + newDependencySet().read(dotd).getDependencies()); + } + + @Test + public void dotDParser_mixedWindowsPaths() throws Exception { + // This is (slightly simplified) actual output from clang. Yes, clang will happily mix + // forward slashes and backslashes in a single path, not to mention using backslashes as + // separators next to backslashes as escape characters. + Path dotd = scratch.file("/tmp/foo.d", + "bazel-out/hello-lib/cpp/hello-lib.o: \\", + "cpp/hello-lib.cc cpp/hello-lib.h /mingw/include\\stdio.h \\", + "/mingw/include\\_mingw.h \\", + "C:\\Program\\ Files\\ (x86)\\LLVM\\bin\\..\\lib\\clang\\3.5.0\\include\\stddef.h \\", + "C:\\Program\\ Files\\ (x86)\\LLVM\\bin\\..\\lib\\clang\\3.5.0\\include\\stdarg.h"); + + Set<PathFragment> expected = Sets.newHashSet( + new PathFragment("cpp/hello-lib.cc"), + new PathFragment("cpp/hello-lib.h"), + new PathFragment("/mingw/include/stdio.h"), + new PathFragment("/mingw/include/_mingw.h"), + new PathFragment("C:/Program Files (x86)/LLVM/lib/clang/3.5.0/include/stddef.h"), + new PathFragment("C:/Program Files (x86)/LLVM/lib/clang/3.5.0/include/stdarg.h")); + + MoreAsserts.assertSameContents(expected, + newDependencySet().read(dotd).getDependencies()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/FileTypeTest.java b/src/test/java/com/google/devtools/build/lib/util/FileTypeTest.java new file mode 100644 index 0000000000..301875ca66 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/FileTypeTest.java @@ -0,0 +1,244 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.util.FileType.HasFilename; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; + +/** + * Test for {@link FileType} and {@link FileTypeSet}. + */ +@RunWith(JUnit4.class) +public class FileTypeTest { + private static final FileType CFG = FileType.of(".cfg"); + private static final FileType HTML = FileType.of(".html"); + private static final FileType TEXT = FileType.of(".txt"); + private static final FileType CPP_SOURCE = FileType.of(".cc", ".cpp", ".cxx", ".C"); + private static final FileType JAVA_SOURCE = FileType.of(".java"); + private static final FileType PYTHON_SOURCE = FileType.of(".py"); + + private static final class HasFilenameImpl implements HasFilename { + private final String path; + + private HasFilenameImpl(String path) { + this.path = path; + } + + @Override + public String getFilename() { + return path; + } + + @Override + public String toString() { + return path; + } + } + + @Test + public void simpleDotMatch() { + assertTrue(TEXT.matches("readme.txt")); + } + + @Test + public void doubleDotMatches() { + assertTrue(TEXT.matches("read.me.txt")); + } + + @Test + public void noExtensionMatches() { + assertTrue(FileType.NO_EXTENSION.matches("hello")); + assertTrue(FileType.NO_EXTENSION.matches("/path/to/hello")); + } + + @Test + public void picksLastExtension() { + assertTrue(TEXT.matches("server.cfg.txt")); + } + + @Test + public void onlyExtensionStillMatches() { + assertTrue(TEXT.matches(".txt")); + } + + @Test + public void handlesPathObjects() { + Path readme = new InMemoryFileSystem().getPath("/readme.txt"); + assertTrue(TEXT.matches(readme)); + } + + @Test + public void handlesPathFragmentObjects() { + PathFragment readme = new PathFragment("some/where/readme.txt"); + assertTrue(TEXT.matches(readme)); + } + + @Test + public void fileTypeSetContains() { + FileTypeSet allowedTypes = FileTypeSet.of(TEXT, HTML); + + assertTrue(allowedTypes.matches("readme.txt")); + assertTrue(!allowedTypes.matches("style.css")); + } + + private List<HasFilename> getArtifacts() { + return Lists.<HasFilename>newArrayList( + new HasFilenameImpl("Foo.java"), + new HasFilenameImpl("bar.cc"), + new HasFilenameImpl("baz.py")); + } + + private String filterAll(FileType... fileTypes) { + return Joiner.on(" ").join(FileType.filter(getArtifacts(), fileTypes)); + } + + @Test + public void justJava() { + assertEquals("Foo.java", filterAll(JAVA_SOURCE)); + } + + @Test + public void javaAndCpp() { + assertEquals("Foo.java bar.cc", filterAll(JAVA_SOURCE, CPP_SOURCE)); + } + + @Test + public void allThree() { + assertEquals("Foo.java bar.cc baz.py", filterAll(JAVA_SOURCE, CPP_SOURCE, PYTHON_SOURCE)); + } + + private HasFilename filename(final String name) { + return new HasFilename() { + @Override + public String getFilename() { + return name; + } + }; + } + + @Test + public void checkingSingleWithTypePredicate() throws Exception { + FileType.HasFilename item = filename("config.txt"); + + assertTrue(FileType.contains(item, TEXT)); + assertFalse(FileType.contains(item, CFG)); + } + + @Test + public void checkingListWithTypePredicate() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("README.txt")); + + assertTrue(FileType.contains(unfiltered, TEXT)); + assertFalse(FileType.contains(unfiltered, CFG)); + } + + @Test + public void filteringWithTypePredicate() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("README.txt"), + filename("archive.zip")); + + assertThat(FileType.filter(unfiltered, TEXT)).containsExactly(unfiltered.get(0), + unfiltered.get(2)).inOrder(); + } + + @Test + public void filteringWithMatcherPredicate() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("README.txt"), + filename("archive.zip")); + + Predicate<String> textFileTypeMatcher = new Predicate<String>() { + @Override + public boolean apply(String input) { + return TEXT.matches(input); + } + }; + + assertThat(FileType.filter(unfiltered, textFileTypeMatcher)).containsExactly(unfiltered.get(0), + unfiltered.get(2)).inOrder(); + } + + @Test + public void filteringWithAlwaysFalse() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("binary"), + filename("archive.zip")); + + assertThat(FileType.filter(unfiltered, FileTypeSet.NO_FILE)).isEmpty(); + } + + @Test + public void filteringWithAlwaysTrue() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("binary"), + filename("archive.zip")); + + assertThat(FileType.filter(unfiltered, FileTypeSet.ANY_FILE)).containsExactly(unfiltered.get(0), + unfiltered.get(1), unfiltered.get(2), unfiltered.get(3)).inOrder(); + } + + @Test + public void exclusionWithTypePredicate() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("README.txt"), + filename("server.cfg")); + + assertThat(FileType.except(unfiltered, TEXT)).containsExactly(unfiltered.get(1), + unfiltered.get(3)).inOrder(); + } + + @Test + public void listFiltering() throws Exception { + ImmutableList<FileType.HasFilename> unfiltered = ImmutableList.of( + filename("config.txt"), + filename("index.html"), + filename("README.txt"), + filename("server.cfg")); + FileTypeSet filter = FileTypeSet.of(HTML, CFG); + + assertThat(FileType.filterList(unfiltered, filter)).containsExactly(unfiltered.get(1), + unfiltered.get(3)).inOrder(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/FingerprintTest.java b/src/test/java/com/google/devtools/build/lib/util/FingerprintTest.java new file mode 100644 index 0000000000..5158019a38 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/FingerprintTest.java @@ -0,0 +1,137 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Tests for Fingerprint. + */ +@RunWith(JUnit4.class) +public class FingerprintTest { + + private static void assertFingerprintsDiffer(List<String> list1, List<String>list2) { + Fingerprint f1 = new Fingerprint(); + Fingerprint f1Latin1 = new Fingerprint(); + for (String s : list1) { + f1.addString(s); + f1Latin1.addStringLatin1(s); + } + Fingerprint f2 = new Fingerprint(); + Fingerprint f2Latin1 = new Fingerprint(); + for (String s : list2) { + f2.addString(s); + f2Latin1.addStringLatin1(s); + } + assertThat(f1.hexDigestAndReset()).isNotEqualTo(f2.hexDigestAndReset()); + assertThat(f1Latin1.hexDigestAndReset()).isNotEqualTo(f2Latin1.hexDigestAndReset()); + } + + // You can validate the md5 of the simple string against + // echo -n 'Hello World!'| md5sum + @Test + public void bytesFingerprint() { + assertThat("ed076287532e86365e841e92bfc50d8c").isEqualTo( + new Fingerprint().addBytes("Hello World!".getBytes(UTF_8)).hexDigestAndReset()); + assertThat("ed076287532e86365e841e92bfc50d8c").isEqualTo(Fingerprint.md5Digest("Hello World!")); + } + + @Test + public void otherStringFingerprint() { + assertFingerprintsDiffer(ImmutableList.of("Hello World!"), + ImmutableList.of("Goodbye World.")); + } + + @Test + public void multipleUpdatesDiffer() throws Exception { + assertFingerprintsDiffer(ImmutableList.of("Hello ", "World!"), + ImmutableList.of("Hello World!")); + } + + @Test + public void multipleUpdatesShiftedDiffer() throws Exception { + assertFingerprintsDiffer(ImmutableList.of("Hello ", "World!"), + ImmutableList.of("Hello", " World!")); + } + + @Test + public void listFingerprintNotSameAsIndividualElements() throws Exception { + Fingerprint f1 = new Fingerprint(); + f1.addString("Hello "); + f1.addString("World!"); + Fingerprint f2 = new Fingerprint(); + f2.addStrings(ImmutableList.of("Hello ", "World!")); + assertThat(f1.hexDigestAndReset()).isNotEqualTo(f2.hexDigestAndReset()); + } + + @Test + public void mapFingerprintNotSameAsIndividualElements() throws Exception { + Fingerprint f1 = new Fingerprint(); + Map<String, String> map = new HashMap<>(); + map.put("Hello ", "World!"); + f1.addStringMap(map); + Fingerprint f2 = new Fingerprint(); + f2.addStrings(ImmutableList.of("Hello ", "World!")); + assertThat(f1.hexDigestAndReset()).isNotEqualTo(f2.hexDigestAndReset()); + } + + @Test + public void toStringTest() throws Exception { + Fingerprint f1 = new Fingerprint(); + f1.addString("Hello "); + f1.addString("World!"); + String fp = f1.hexDigestAndReset(); + Fingerprint f2 = new Fingerprint(); + f2.addString("Hello "); + // make sure that you can call toString on the intermediate result + // and continue with the operation. + assertThat(fp).isNotEqualTo(f2.toString()); + f2.addString("World!"); + assertThat(fp).isEqualTo(f2.hexDigestAndReset()); + } + + @Test + public void addBoolean() throws Exception { + String f1 = new Fingerprint().addBoolean(true).hexDigestAndReset(); + String f2 = new Fingerprint().addBoolean(false).hexDigestAndReset(); + String f3 = new Fingerprint().addBoolean(true).hexDigestAndReset(); + + assertThat(f1).isEqualTo(f3); + assertThat(f1).isNotEqualTo(f2); + } + + @Test + public void addPath() throws Exception { + PathFragment pf = new PathFragment("/etc/pwd"); + assertThat("01cc3eeea3a2f58e447e824f9f62d3d1").isEqualTo( + new Fingerprint().addPath(pf).hexDigestAndReset()); + Path p = new InMemoryFileSystem(BlazeClock.instance()).getPath(pf); + assertThat("01cc3eeea3a2f58e447e824f9f62d3d1").isEqualTo( + new Fingerprint().addPath(p).hexDigestAndReset()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/GroupedListTest.java b/src/test/java/com/google/devtools/build/lib/util/GroupedListTest.java new file mode 100644 index 0000000000..87cd8c9aba --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/GroupedListTest.java @@ -0,0 +1,247 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assert_; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +@RunWith(JUnit4.class) +public class GroupedListTest { + @Test + public void empty() { + createSizeN(0); + } + + @Test + public void sizeOne() { + createSizeN(1); + } + + @Test + public void sizeTwo() { + createSizeN(2); + } + + @Test + public void sizeN() { + createSizeN(10); + } + + private void createSizeN(int size) { + List<String> list = new ArrayList<>(); + for (int i = 0; i < size; i++) { + list.add("test" + i); + } + Object compressedList = createAndCompress(list); + assertTrue(Iterables.elementsEqual(iterable(compressedList), list)); + assertElementsEqual(compressedList, list); + } + + @Test + public void elementsNotEqualDifferentOrder() { + List<String> list = Lists.newArrayList("a", "b", "c"); + Object compressedList = createAndCompress(list); + ArrayList<String> reversed = new ArrayList<>(list); + Collections.reverse(reversed); + assertFalse(elementsEqual(compressedList, reversed)); + } + + @Test + public void elementsNotEqualDifferentSizes() { + for (int size1 = 0; size1 < 10; size1++) { + List<String> firstList = new ArrayList<>(); + for (int i = 0; i < size1; i++) { + firstList.add("test" + i); + } + Object array = createAndCompress(firstList); + for (int size2 = 0; size2 < 10; size2++) { + List<String> secondList = new ArrayList<>(); + for (int i = 0; i < size2; i++) { + secondList.add("test" + i); + } + assertEquals(GroupedList.create(array) + ", " + secondList + ", " + size1 + ", " + size2, + size1 == size2, elementsEqual(array, secondList)); + } + } + } + + @Test + public void group() { + GroupedList<String> groupedList = new GroupedList<>(); + assertTrue(groupedList.isEmpty()); + GroupedListHelper<String> helper = new GroupedListHelper<>(); + List<ImmutableList<String>> elements = ImmutableList.of( + ImmutableList.of("1"), + ImmutableList.of("2a", "2b"), + ImmutableList.of("3"), + ImmutableList.of("4"), + ImmutableList.of("5a", "5b", "5c"), + ImmutableList.of("6a", "6b", "6c") + ); + List<String> allElts = new ArrayList<>(); + for (List<String> group : elements) { + if (group.size() > 1) { + helper.startGroup(); + } + for (String elt : group) { + helper.add(elt); + } + if (group.size() > 1) { + helper.endGroup(); + } + allElts.addAll(group); + } + groupedList.append(helper); + assertEquals(allElts.size(), groupedList.size()); + assertFalse(groupedList.isEmpty()); + Object compressed = groupedList.compress(); + assertElementsEqual(compressed, allElts); + assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder(); + assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder(); + } + + @Test + public void singletonAndEmptyGroups() { + GroupedList<String> groupedList = new GroupedList<>(); + assertTrue(groupedList.isEmpty()); + GroupedListHelper<String> helper = new GroupedListHelper<>(); + @SuppressWarnings("unchecked") // varargs + List<ImmutableList<String>> elements = Lists.newArrayList( + ImmutableList.of("1"), + ImmutableList.<String>of(), + ImmutableList.of("2a", "2b"), + ImmutableList.of("3") + ); + List<String> allElts = new ArrayList<>(); + for (List<String> group : elements) { + helper.startGroup(); // Start a group even if the group has only one element or is empty. + for (String elt : group) { + helper.add(elt); + } + helper.endGroup(); + allElts.addAll(group); + } + groupedList.append(helper); + assertEquals(allElts.size(), groupedList.size()); + assertFalse(groupedList.isEmpty()); + Object compressed = groupedList.compress(); + assertElementsEqual(compressed, allElts); + // Get rid of empty list -- it was not stored in groupedList. + elements.remove(1); + assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder(); + assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder(); + } + + @Test + public void removeMakesEmpty() { + GroupedList<String> groupedList = new GroupedList<>(); + assertTrue(groupedList.isEmpty()); + GroupedListHelper<String> helper = new GroupedListHelper<>(); + @SuppressWarnings("unchecked") // varargs + List<List<String>> elements = Lists.newArrayList( + (List<String>) ImmutableList.of("1"), + ImmutableList.<String>of(), + Lists.newArrayList("2a", "2b"), + ImmutableList.of("3"), + ImmutableList.of("removedGroup1", "removedGroup2"), + ImmutableList.of("4") + ); + List<String> allElts = new ArrayList<>(); + for (List<String> group : elements) { + helper.startGroup(); // Start a group even if the group has only one element or is empty. + for (String elt : group) { + helper.add(elt); + } + helper.endGroup(); + allElts.addAll(group); + } + groupedList.append(helper); + Set<String> removed = ImmutableSet.of("2a", "3", "removedGroup1", "removedGroup2"); + groupedList.remove(removed); + Object compressed = groupedList.compress(); + allElts.removeAll(removed); + assertElementsEqual(compressed, allElts); + elements.get(2).remove("2a"); + elements.remove(ImmutableList.of("3")); + elements.remove(ImmutableList.of()); + elements.remove(ImmutableList.of("removedGroup1", "removedGroup2")); + assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder(); + assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder(); + } + + @Test + public void removeGroupFromSmallList() { + GroupedList<String> groupedList = new GroupedList<>(); + assertTrue(groupedList.isEmpty()); + GroupedListHelper<String> helper = new GroupedListHelper<>(); + List<List<String>> elements = new ArrayList<>(); + List<String> group = Lists.newArrayList("1a", "1b", "1c", "1d"); + elements.add(group); + List<String> allElts = new ArrayList<>(); + helper.startGroup(); + for (String item : elements.get(0)) { + helper.add(item); + } + allElts.addAll(group); + helper.endGroup(); + groupedList.append(helper); + Set<String> removed = ImmutableSet.of("1b", "1c"); + groupedList.remove(removed); + Object compressed = groupedList.compress(); + allElts.removeAll(removed); + assertElementsEqual(compressed, allElts); + elements.get(0).removeAll(removed); + assert_().that(GroupedList.create(compressed)).containsExactlyElementsIn(elements).inOrder(); + assert_().that(groupedList).containsExactlyElementsIn(elements).inOrder(); + } + + private static Object createAndCompress(Collection<String> list) { + GroupedList<String> result = new GroupedList<>(); + result.append(GroupedListHelper.create(list)); + return result.compress(); + } + + private static Iterable<String> iterable(Object compressed) { + return GroupedList.<String>create(compressed).toSet(); + } + + private static boolean elementsEqual(Object compressed, Iterable<String> expected) { + return Iterables.elementsEqual(GroupedList.<String>create(compressed).toSet(), expected); + } + + private static void assertElementsEqual(Object compressed, Iterable<String> expected) { + assert_() + .that(GroupedList.<String>create(compressed).toSet()) + .containsExactlyElementsIn(expected) + .inOrder(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/JavaClockTest.java b/src/test/java/com/google/devtools/build/lib/util/JavaClockTest.java new file mode 100644 index 0000000000..d5d39c0fc2 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/JavaClockTest.java @@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for the Clock instance based on the Java System class. + */ +@RunWith(JUnit4.class) +public class JavaClockTest { + + @Test + public void javaClockIsAdvancing() throws Exception { + Clock clock = new JavaClock(); + long millis = clock.currentTimeMillis(); + long nanos = clock.nanoTime(); + + Thread.sleep(10); + + assertThat(clock.currentTimeMillis()).isNotEqualTo(millis); + assertThat(clock.nanoTime()).isNotEqualTo(nanos); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/OptionsUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/OptionsUtilsTest.java new file mode 100644 index 0000000000..9888061c4a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/OptionsUtilsTest.java @@ -0,0 +1,157 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.util.OptionsUtils.PathFragmentListConverter; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.List; + +/** + * Test for {@link OptionsUtils}. + */ +@RunWith(JUnit4.class) +public class OptionsUtilsTest { + + public static class IntrospectionExample extends OptionsBase { + @Option(name = "alpha", + category = "one", + defaultValue = "alpha") + public String alpha; + + @Option(name = "beta", + category = "one", + defaultValue = "beta") + public String beta; + + @Option(name = "gamma", + category = "undocumented", + defaultValue = "gamma") + public String gamma; + + @Option(name = "delta", + category = "undocumented", + defaultValue = "delta") + public String delta; + + @Option(name = "echo", + category = "hidden", + defaultValue = "echo") + public String echo; + } + + @Test + public void asStringOfExplicitOptions() throws Exception { + OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class); + parser.parse("--alpha=no", "--gamma=no", "--echo=no"); + assertEquals("--alpha=no --gamma=no", OptionsUtils.asShellEscapedString(parser)); + } + + @Test + public void asStringOfExplicitOptionsCorrectSortingByPriority() throws Exception { + OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class); + parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--alpha=no")); + parser.parse(OptionPriority.COMPUTED_DEFAULT, null, Arrays.asList("--beta=no")); + assertEquals("--beta=no --alpha=no", OptionsUtils.asShellEscapedString(parser)); + } + + public static class BooleanOpts extends OptionsBase { + @Option(name = "b_one", + category = "xyz", + defaultValue = "true") + public boolean bOne; + + @Option(name = "b_two", + category = "123", // Not printed in usage messages! + defaultValue = "false") + public boolean bTwo; + } + + @Test + public void asStringOfExplicitOptionsWithBooleans() throws Exception { + OptionsParser parser = OptionsParser.newOptionsParser(BooleanOpts.class); + parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--b_one", "--nob_two")); + assertEquals("--b_one --nob_two", OptionsUtils.asShellEscapedString(parser)); + + parser = OptionsParser.newOptionsParser(BooleanOpts.class); + parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--b_one=true", "--b_two=0")); + assertTrue(parser.getOptions(BooleanOpts.class).bOne); + assertFalse(parser.getOptions(BooleanOpts.class).bTwo); + assertEquals("--b_one --nob_two", OptionsUtils.asShellEscapedString(parser)); + } + + @Test + public void asStringOfExplicitOptionsMultipleOptionsAreMultipleTimes() throws Exception { + OptionsParser parser = OptionsParser.newOptionsParser(IntrospectionExample.class); + parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--alpha=one")); + parser.parse(OptionPriority.COMMAND_LINE, null, Arrays.asList("--alpha=two")); + assertEquals("--alpha=one --alpha=two", OptionsUtils.asShellEscapedString(parser)); + } + + private static List<PathFragment> list(PathFragment... fragments) { + return Lists.newArrayList(fragments); + } + + private PathFragment fragment(String string) { + return new PathFragment(string); + } + + private List<PathFragment> convert(String input) throws Exception { + return new PathFragmentListConverter().convert(input); + } + + @Test + public void emptyStringYieldsEmptyList() throws Exception { + assertEquals(list(), convert("")); + } + + @Test + public void lonelyDotYieldsLonelyDot() throws Exception { + assertEquals(list(fragment(".")), convert(".")); + } + + @Test + public void converterSkipsEmptyStrings() throws Exception { + assertEquals(list(fragment("foo"), fragment("bar")), convert("foo::bar:")); + } + + @Test + public void multiplePaths() throws Exception { + assertEquals(list(fragment("foo"), fragment("/bar/baz"), fragment("."), + fragment("/tmp/bang")), convert("foo:/bar/baz:.:/tmp/bang")); + } + + @Test + public void valueisUnmodifiable() throws Exception { + try { + new PathFragmentListConverter().convert("value").add(new PathFragment("other")); + fail("could modify value"); + } catch (UnsupportedOperationException expected) {} + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/PairTest.java b/src/test/java/com/google/devtools/build/lib/util/PairTest.java new file mode 100644 index 0000000000..f82e12f293 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/PairTest.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Objects; + +/** + * Tests for {@link Pair}. + */ +@RunWith(JUnit4.class) +public class PairTest { + + @Test + public void constructor() { + Object a = new Object(); + Object b = new Object(); + Pair<Object, Object> p = Pair.of(a, b); + assertSame(a, p.first); + assertSame(b, p.second); + assertEquals(Pair.of(a, b), p); + assertEquals(Objects.hash(a, b), p.hashCode()); + } + + @Test + public void nullable() { + Pair<Object, Object> p = Pair.of(null, null); + assertNull(p.first); + assertNull(p.second); + p.hashCode(); // Should not throw. + assertTrue(p.equals(p)); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/PathFragmentFilterTest.java b/src/test/java/com/google/devtools/build/lib/util/PathFragmentFilterTest.java new file mode 100644 index 0000000000..e911324966 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/PathFragmentFilterTest.java @@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.devtools.build.lib.vfs.PathFragment; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A test for {@link PathFragmentFilter}. + */ +@RunWith(JUnit4.class) +public class PathFragmentFilterTest { + protected PathFragmentFilter filter = null; + + protected void createFilter(String filterString) { + filter = new PathFragmentFilter.PathFragmentFilterConverter().convert(filterString); + } + + protected void assertIncluded(String path) { + assertTrue(filter.isIncluded(new PathFragment(path))); + } + + protected void assertExcluded(String path) { + assertFalse(filter.isIncluded(new PathFragment(path))); + } + + @Test + public void emptyFilter() { + createFilter(""); + assertIncluded("a/b/c"); + assertIncluded("d"); + } + + @Test + public void inclusions() { + createFilter("a/b,c"); + assertIncluded("a/b"); + assertIncluded("a/b/c"); + assertIncluded("c"); + assertIncluded("c/d"); + assertExcluded("a"); + assertExcluded("a/c"); + assertExcluded("d"); + assertExcluded("e/f/g"); + } + + @Test + public void exclusions() { + createFilter("-a/b,-c"); + assertExcluded("a/b"); + assertExcluded("a/b/c"); + assertExcluded("c"); + assertExcluded("c/d"); + assertIncluded("a"); + assertIncluded("a/c"); + assertIncluded("d"); + assertIncluded("e/f/g"); + } + + @Test + public void inclusionsAndExclusions() { + createFilter("a,-c,,d,a/b/c,-a/b,a/b/d"); + assertIncluded("a"); + assertIncluded("a/c"); + assertExcluded("a/b"); + assertExcluded("a/b/c"); // Exclusions take precedence over inclusions. Order is not important. + assertExcluded("a/b/d"); // Exclusions take precedence over inclusions. Order is not important. + assertExcluded("c"); + assertExcluded("c/d"); + assertIncluded("d/e"); + assertExcluded("e"); + // When converted back to string, inclusion entries will be put first, followed by exclusion + // entries. + assertEquals("a,d,a/b/c,a/b/d,-c,-a/b", filter.toString()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/PersistentMapTest.java b/src/test/java/com/google/devtools/build/lib/util/PersistentMapTest.java new file mode 100644 index 0000000000..44aa538265 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/PersistentMapTest.java @@ -0,0 +1,225 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for the {@link PersistentMap}. + */ +@RunWith(JUnit4.class) +public class PersistentMapTest { + public static class PersistentStringMap extends PersistentMap<String, String> { + boolean updateJournal = true; + boolean keepJournal = false; + + public PersistentStringMap(Map<String, String> map, Path mapFile, + Path journalFile) throws IOException { + super(0x0, map, mapFile, journalFile); + load(); + } + + @Override + protected String readKey(DataInputStream in) throws IOException { + return in.readUTF(); + } + @Override + protected String readValue(DataInputStream in) throws IOException { + return in.readUTF(); + } + @Override + protected void writeKey(String key, DataOutputStream out) + throws IOException { + out.writeUTF(key); + } + @Override + protected void writeValue(String value, DataOutputStream out) + throws IOException { + out.writeUTF(value); + } + @Override + protected boolean updateJournal() { + return updateJournal; + } + @Override + protected boolean keepJournal() { + return keepJournal; + } + } + + private FsApparatus scratch = FsApparatus.newInMemory(); + + private PersistentStringMap map; + private Path mapFile; + private Path journalFile; + + @Before + public void setUp() throws Exception { + mapFile = scratch.fs().getPath("/tmp/map.txt"); + journalFile = scratch.fs().getPath("/tmp/journal.txt"); + createMap(); + } + + private void createMap() throws Exception { + Map<String, String> map = new HashMap<>(); + this.map = new PersistentStringMap(map, mapFile, journalFile); + } + + @Test + public void map() throws Exception { + createMap(); + map.put("foo", "bar"); + map.put("baz", "bang"); + assertEquals("bar", map.get("foo")); + assertEquals("bang", map.get("baz")); + assertEquals(2, map.size()); + long size = map.save(); + assertEquals(mapFile.getFileSize(), size); + assertEquals("bar", map.get("foo")); + assertEquals("bang", map.get("baz")); + assertEquals(2, map.size()); + + createMap(); // create a new map + assertEquals("bar", map.get("foo")); + assertEquals("bang", map.get("baz")); + assertEquals(2, map.size()); + } + + @Test + public void remove() throws Exception { + createMap(); + map.put("foo", "bar"); + map.put("baz", "bang"); + long size = map.save(); + assertEquals(mapFile.getFileSize(), size); + assertFalse(journalFile.exists()); + map.remove("foo"); + assertEquals(1, map.size()); + assertTrue(journalFile.exists()); + createMap(); // create a new map + assertEquals(1, map.size()); + } + + @Test + public void clear() throws Exception { + createMap(); + map.put("foo", "bar"); + map.put("baz", "bang"); + map.save(); + assertTrue(mapFile.exists()); + assertFalse(journalFile.exists()); + map.clear(); + assertEquals(0, map.size()); + assertTrue(mapFile.exists()); + assertFalse(journalFile.exists()); + createMap(); // create a new map + assertEquals(0, map.size()); + } + + @Test + public void noUpdateJournal() throws Exception { + createMap(); + map.put("foo", "bar"); + map.put("baz", "bang"); + map.save(); + assertFalse(journalFile.exists()); + // prevent updating the journal + map.updateJournal = false; + // remove an entry + map.remove("foo"); + assertEquals(1, map.size()); + // no journal file written + assertFalse(journalFile.exists()); + createMap(); // create a new map + // both entries are still in the map on disk + assertEquals(2, map.size()); + } + + @Test + public void keepJournal() throws Exception { + createMap(); + map.put("foo", "bar"); + map.put("baz", "bang"); + map.save(); + assertFalse(journalFile.exists()); + + // Keep the journal through the save. + map.updateJournal = false; + map.keepJournal = true; + + // remove an entry + map.remove("foo"); + assertEquals(1, map.size()); + // no journal file written + assertFalse(journalFile.exists()); + + long size = map.save(); + assertEquals(1, map.size()); + // The journal must be serialzed on save(), even if !updateJournal. + assertTrue(journalFile.exists()); + assertEquals(journalFile.getFileSize() + mapFile.getFileSize(), size); + + map.load(); + assertEquals(1, map.size()); + assertTrue(journalFile.exists()); + + createMap(); // create a new map + assertEquals(1, map.size()); + + map.keepJournal = false; + map.save(); + assertEquals(1, map.size()); + assertFalse(journalFile.exists()); + } + + @Test + public void multipleJournalUpdates() throws Exception { + createMap(); + map.put("foo", "bar"); + map.save(); + assertFalse(journalFile.exists()); + // add an entry + map.put("baz", "bang"); + assertEquals(2, map.size()); + // journal file written + assertTrue(journalFile.exists()); + createMap(); // create a new map + // both entries are still in the map on disk + assertEquals(2, map.size()); + // add another entry + map.put("baz2", "bang2"); + assertEquals(3, map.size()); + // journal file written + assertTrue(journalFile.exists()); + createMap(); // create a new map + // all three entries are still in the map on disk + assertEquals(3, map.size()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/ProcMeminfoParserTest.java b/src/test/java/com/google/devtools/build/lib/util/ProcMeminfoParserTest.java new file mode 100644 index 0000000000..9d04f287f8 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/ProcMeminfoParserTest.java @@ -0,0 +1,91 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.devtools.build.lib.vfs.util.FsApparatus; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; + +/** + * Tests for ProcMeminfoParser. + */ +@RunWith(JUnit4.class) +public class ProcMeminfoParserTest { + + private FsApparatus scratch = FsApparatus.newNative(); + + @Test + public void memInfo() throws IOException { + String meminfoContent = StringUtilities.joinLines( + "MemTotal: 3091732 kB", + "MemFree: 2167344 kB", + "Buffers: 60644 kB", + "Cached: 509940 kB", + "SwapCached: 0 kB", + "Active: 636892 kB", + "Inactive: 212760 kB", + "HighTotal: 0 kB", + "HighFree: 0 kB", + "LowTotal: 3091732 kB", + "LowFree: 2167344 kB", + "SwapTotal: 9124880 kB", + "SwapFree: 9124880 kB", + "Dirty: 0 kB", + "Writeback: 0 kB", + "AnonPages: 279028 kB", + "Mapped: 54404 kB", + "Slab: 42820 kB", + "PageTables: 5184 kB", + "NFS_Unstable: 0 kB", + "Bounce: 0 kB", + "CommitLimit: 10670744 kB", + "Committed_AS: 665840 kB", + "VmallocTotal: 34359738367 kB", + "VmallocUsed: 300484 kB", + "VmallocChunk: 34359437307 kB", + "HugePages_Total: 0", + "HugePages_Free: 0", + "HugePages_Rsvd: 0", + "Hugepagesize: 2048 kB", + "Bogus: not_a_number", + "Bogus2: 1000000000000000000000000000000000000000000000000 kB" + ); + + String meminfoFile = scratch.file("test_meminfo", meminfoContent).getPathString(); + ProcMeminfoParser memInfo = new ProcMeminfoParser(meminfoFile); + + assertEquals(2356756, memInfo.getFreeRamKb()); + assertEquals(509940, memInfo.getRamKb("Cached")); + assertEquals(3091732, memInfo.getTotalKb()); + assertNotAvailable("Bogus", memInfo); + assertNotAvailable("Bogus2", memInfo); + } + + private static void assertNotAvailable(String field, ProcMeminfoParser memInfo) { + try { + memInfo.getRamKb(field); + fail(); + } catch (IllegalArgumentException e) { + // Expected. + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/RegexFilterTest.java b/src/test/java/com/google/devtools/build/lib/util/RegexFilterTest.java new file mode 100644 index 0000000000..bb502c71f0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/RegexFilterTest.java @@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.testing.EqualsTester; +import com.google.devtools.common.options.OptionsParsingException; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A test for {@link RegexFilter}. + */ +@RunWith(JUnit4.class) +public class RegexFilterTest { + protected RegexFilter filter = null; + + protected RegexFilter createFilter(String filterString) throws OptionsParsingException { + filter = new RegexFilter.RegexFilterConverter().convert(filterString); + return filter; + } + + protected void assertIncluded(String value) { + assertTrue(filter.isIncluded(value)); + } + + protected void assertExcluded(String value) { + assertFalse(filter.isIncluded(value)); + } + + @Test + public void emptyFilter() throws Exception { + createFilter(""); + assertIncluded("a/b/c"); + assertIncluded("d"); + } + + @Test + public void inclusions() throws Exception { + createFilter("a/b,+^c,_test$"); + assertEquals("(?:(?>a/b)|(?>^c)|(?>_test$))", filter.toString()); + assertIncluded("a/b"); + assertIncluded("a/b/c"); + assertIncluded("c"); + assertIncluded("c/d"); + assertIncluded("e/a/b"); + assertIncluded("f/1/2/3/_test"); + assertExcluded("a"); + assertExcluded("a/c"); + assertExcluded("d"); + assertExcluded("e/f/g"); + assertExcluded("f/_test2"); + } + + @Test + public void exclusions() throws Exception { + createFilter("-a/b,-^c,-_test$"); + assertEquals("-(?:(?>a/b)|(?>^c)|(?>_test$))", filter.toString()); + assertExcluded("a/b"); + assertExcluded("a/b/c"); + assertExcluded("c"); + assertExcluded("c/d"); + assertExcluded("f/a/b/d"); + assertExcluded("f/a_test"); + assertIncluded("a"); + assertIncluded("a/c"); + assertIncluded("d"); + assertIncluded("e/f/g"); + assertIncluded("f/a_test_case"); + } + + @Test + public void inclusionsAndExclusions() throws Exception { + createFilter("a,-^c,,-,+,d,+a/b/c,-a/b,a/b/d"); + assertEquals("(?:(?>a)|(?>d)|(?>a/b/c)|(?>a/b/d)),-(?:(?>^c)|(?>a/b))", filter.toString()); + assertIncluded("a"); + assertIncluded("a/c"); + assertExcluded("a/b"); + assertExcluded("a/b/c"); // Exclusions take precedence over inclusions. Order is not important. + assertExcluded("a/b/d"); // Exclusions take precedence over inclusions. Order is not important. + assertExcluded("a/c/a/b/d"); + assertExcluded("c"); + assertExcluded("c/d"); + assertIncluded("d/e"); + assertExcluded("e"); + } + + @Test + public void commas() throws Exception { + createFilter("a\\,b,c\\,d"); + assertEquals("(?:(?>a\\,b)|(?>c\\,d))", filter.toString()); + assertIncluded("a,b"); + assertIncluded("c,d"); + assertExcluded("a"); + assertExcluded("b,c"); + assertExcluded("d"); + } + + @Test + public void invalidExpression() throws Exception { + try { + createFilter("*a"); + fail(); // OptionsParsingException should be thrown. + } catch (OptionsParsingException e) { + assertThat(e.getMessage()) + .contains("Failed to build valid regular expression: Dangling meta character '*' " + + "near index"); + } + } + + @Test + public void equals() throws Exception { + new EqualsTester() + .addEqualityGroup(createFilter("a,b,c"), createFilter("a,b,c")) + .addEqualityGroup(createFilter("a,b,c,d")) + .addEqualityGroup(createFilter("a,b,-c"), createFilter("a,b,-c")) + .addEqualityGroup(createFilter("a,b,-c,-d")) + .addEqualityGroup(createFilter("-a,-b,-c"), createFilter("-a,-b,-c")) + .addEqualityGroup(createFilter("-a,-b,-c,-d")) + .addEqualityGroup(createFilter(""), createFilter("")) + .testEquals(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.java b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.java new file mode 100644 index 0000000000..e821ca1266 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; + +/** + * A test for {@link ResourceFileLoader}. + */ +@RunWith(JUnit4.class) +public class ResourceFileLoaderTest { + + @Test + public void loader() throws IOException { + String message = ResourceFileLoader.loadResource( + ResourceFileLoaderTest.class, "ResourceFileLoaderTest.message"); + assertEquals("Hello, world.", message); + } + + @Test + public void resourceNotFound() { + try { + ResourceFileLoader.loadResource(ResourceFileLoaderTest.class, + "does_not_exist.txt"); + fail(); + } catch (IOException e) { + assertEquals("does_not_exist.txt not found.", e.getMessage()); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.message b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.message new file mode 100644 index 0000000000..c872090cf1 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/ResourceFileLoaderTest.message @@ -0,0 +1 @@ +Hello, world.
\ No newline at end of file diff --git a/src/test/java/com/google/devtools/build/lib/util/ShellEscaperTest.java b/src/test/java/com/google/devtools/build/lib/util/ShellEscaperTest.java new file mode 100644 index 0000000000..ac5d4130e5 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/ShellEscaperTest.java @@ -0,0 +1,83 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.devtools.build.lib.util.ShellEscaper.escapeString; +import static org.junit.Assert.assertEquals; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableSet; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.Set; + +/** + * Tests for {@link ShellEscaper}. + * + * <p>Based on {@code com.google.io.base.shell.ShellUtilsTest}. + */ +@RunWith(JUnit4.class) +public class ShellEscaperTest { + + @Test + public void shellEscape() throws Exception { + assertEquals("''", escapeString("")); + assertEquals("foo", escapeString("foo")); + assertEquals("'foo bar'", escapeString("foo bar")); + assertEquals("''\\''foo'\\'''", escapeString("'foo'")); + assertEquals("'\\'\\''foo\\'\\'''", escapeString("\\'foo\\'")); + assertEquals("'${filename%.c}.o'", escapeString("${filename%.c}.o")); + assertEquals("'<html!>'", escapeString("<html!>")); + } + + @Test + public void escapeAll() throws Exception { + Set<String> escaped = ImmutableSet.copyOf( + ShellEscaper.escapeAll(Arrays.asList("foo", "@bar", "baz'qux"))); + assertEquals(ImmutableSet.of("foo", "@bar", "'baz'\\''qux'"), escaped); + } + + @Test + public void escapeJoinAllIntoAppendable() throws Exception { + Appendable appendable = ShellEscaper.escapeJoinAll( + new StringBuilder("initial"), Arrays.asList("foo", "$BAR")); + assertEquals("initialfoo '$BAR'", appendable.toString()); + } + + @Test + public void escapeJoinAllIntoAppendableWithCustomJoiner() throws Exception { + Appendable appendable = ShellEscaper.escapeJoinAll( + new StringBuilder("initial"), Arrays.asList("foo", "$BAR"), Joiner.on('|')); + assertEquals("initialfoo|'$BAR'", appendable.toString()); + } + + @Test + public void escapeJoinAll() throws Exception { + String actual = ShellEscaper.escapeJoinAll( + Arrays.asList("foo", "@echo:-", "100", "$US", "a b", "\"qu'ot'es\"", "\"quot\"", "\\")); + assertEquals("foo @echo:- 100 '$US' 'a b' '\"qu'\\''ot'\\''es\"' '\"quot\"' '\\'", actual); + } + + @Test + public void escapeJoinAllWithCustomJoiner() throws Exception { + String actual = ShellEscaper.escapeJoinAll( + Arrays.asList("foo", "@echo:-", "100", "$US", "a b", "\"qu'ot'es\"", "\"quot\"", "\\"), + Joiner.on('|')); + assertEquals("foo|@echo:-|100|'$US'|'a b'|'\"qu'\\''ot'\\''es\"'|'\"quot\"'|'\\'", actual); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/StringCanonicalizerTest.java b/src/test/java/com/google/devtools/build/lib/util/StringCanonicalizerTest.java new file mode 100644 index 0000000000..64acddc708 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/StringCanonicalizerTest.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertSame; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for String canonicalizer. + */ +@RunWith(JUnit4.class) +public class StringCanonicalizerTest { + + @Test + public void twoDifferentStringsAreDifferent() { + String stringA = StringCanonicalizer.intern("A"); + String stringB = StringCanonicalizer.intern("B"); + assertThat(stringA).isNotEqualTo(stringB); + } + + @Test + public void twoSameStringsAreCanonicalized() { + String stringA1 = StringCanonicalizer.intern(new String("A")); + String stringA2 = StringCanonicalizer.intern(new String("A")); + assertSame(stringA1, stringA2); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/StringIndexerTest.java b/src/test/java/com/google/devtools/build/lib/util/StringIndexerTest.java new file mode 100644 index 0000000000..693e11a736 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/StringIndexerTest.java @@ -0,0 +1,314 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.common.base.Function; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.testutil.TestUtils; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.List; +import java.util.SortedMap; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Test for the StringIndexer classes. + */ +public abstract class StringIndexerTest { + + private static final int ATTEMPTS = 1000; + private SortedMap<Integer, String> mappings; + + protected StringIndexer indexer; + + @Before + public void setUp() throws Exception { + indexer = newIndexer(); + mappings = Maps.newTreeMap(); + } + + protected abstract StringIndexer newIndexer(); + + protected void assertSize(int expected) { + assertEquals(expected, indexer.size()); + } + + protected void assertNoIndex(String s) { + int size = indexer.size(); + assertEquals(-1, indexer.getIndex(s)); + assertEquals(size, indexer.size()); + } + + protected void assertIndex(int expected, String s) { + // System.out.println("Adding " + s + ", expecting " + expected); + int index = indexer.getOrCreateIndex(s); + // System.out.println(csi); + assertEquals(expected, index); + mappings.put(expected, s); + } + + protected void assertContent() { + for (int i = 0; i < indexer.size(); i++) { + assertNotNull(mappings.get(i)); + assertEquals(mappings.get(i), indexer.getStringForIndex(i)); + } + } + + private void assertConcurrentUpdates(Function<Integer, String> keyGenerator) throws Exception { + final AtomicInteger safeIndex = new AtomicInteger(-1); + List<String> keys = Lists.newArrayListWithCapacity(ATTEMPTS); + ThreadPoolExecutor executor = new ThreadPoolExecutor(3, 3, 5, TimeUnit.SECONDS, + new ArrayBlockingQueue<Runnable>(ATTEMPTS)); + synchronized(indexer) { + for (int i = 0; i < ATTEMPTS; i++) { + final String key = keyGenerator.apply(i); + keys.add(key); + executor.execute(new Runnable() { + @Override + public void run() { + int index = indexer.getOrCreateIndex(key); + if (safeIndex.get() < index) { safeIndex.set(index); } + indexer.addString(key); + } + }); + } + } + try { + while(!executor.getQueue().isEmpty()) { + // Validate that we can execute concurrent queries too. + if (safeIndex.get() >= 0) { + int index = safeIndex.get(); + // Retrieve string using random existing index and validate reverse mapping. + String key = indexer.getStringForIndex(index); + assertNotNull(key); + assertEquals(index, indexer.getIndex(key)); + } + } + } finally { + executor.shutdown(); + executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + for (String key : keys) { + // Validate mapping between keys and indices. + assertEquals(key, indexer.getStringForIndex(indexer.getIndex(key))); + } + } + + @Test + public void concurrentAddChildNode() throws Exception { + assertConcurrentUpdates(new Function<Integer, String>() { + @Override + public String apply(Integer from) { return Strings.repeat("a", from + 1); } + }); + } + + @Test + public void concurrentSplitNodeSuffix() throws Exception { + assertConcurrentUpdates(new Function<Integer, String>() { + @Override + public String apply(Integer from) { return Strings.repeat("b", ATTEMPTS - from); } + }); + } + + @Test + public void concurrentAddBranch() throws Exception { + assertConcurrentUpdates(new Function<Integer, String>() { + @Override + public String apply(Integer from) { return String.format("%08o", from); } + }); + } + + @RunWith(JUnit4.class) + public static class CompactStringIndexerTest extends StringIndexerTest { + @Override + protected StringIndexer newIndexer() { + return new CompactStringIndexer(1); + } + + @Test + public void basicOperations() { + assertSize(0); + assertNoIndex("abcdef"); + assertIndex(0, "abcdef"); // root node creation + assertIndex(0, "abcdef"); // root node match + assertSize(1); + assertIndex(2, "abddef"); // node branching, index 1 went to "ab" node. + assertSize(3); + assertIndex(1, "ab"); + assertSize(3); + assertIndex(3, "abcdefghik"); // new leaf creation + assertSize(4); + assertIndex(4, "abcdefgh"); // node split + assertSize(5); + assertNoIndex("a"); + assertNoIndex("abc"); + assertNoIndex("abcdefg"); + assertNoIndex("abcdefghil"); + assertNoIndex("abcdefghikl"); + assertContent(); + indexer.clear(); + assertSize(0); + assertNull(indexer.getStringForIndex(0)); + assertNull(indexer.getStringForIndex(1000)); + } + + @Test + public void parentIndexUpdate() { + assertSize(0); + assertIndex(0, "abcdefghik"); // Create 3 nodes with single common parent "abcdefgh". + assertIndex(2, "abcdefghlm"); // Index 1 went to "abcdefgh". + assertIndex(3, "abcdefghxyz"); + assertSize(4); + assertIndex(5, "abcdpqr"); // Split parent. Index 4 went to "abcd". + assertSize(6); + assertIndex(1, "abcdefgh"); // Check branch node indices. + assertIndex(4, "abcd"); + assertSize(6); + assertContent(); + } + + @Test + public void emptyRootNode() { + assertSize(0); + assertIndex(0, "abc"); + assertNoIndex(""); + assertIndex(2, "def"); // root node key is now empty string and has index 1. + assertSize(3); + assertIndex(1, ""); + assertSize(3); + assertContent(); + } + + protected void setupTestContent() { + assertSize(0); + assertIndex(0, "abcdefghi"); // Create leafs + assertIndex(2, "abcdefjkl"); + assertIndex(3, "abcdefmno"); + assertIndex(4, "abcdefjklpr"); + assertIndex(6, "abcdstr"); + assertIndex(8, "012345"); + assertSize(9); + assertIndex(1, "abcdef"); // Validate inner nodes + assertIndex(5, "abcd"); + assertIndex(7, ""); + assertSize(9); + assertContent(); + } + + @Test + public void dumpContent() { + indexer = newIndexer(); + indexer.addString("abc"); + String content = indexer.toString(); + assertThat(content).contains("size = 1"); + assertThat(content).contains("contentSize = 5"); + indexer = newIndexer(); + setupTestContent(); + content = indexer.toString(); + assertThat(content).contains("size = 9"); + assertThat(content).contains("contentSize = 60"); + System.out.println(indexer); + } + + @Test + public void addStringResult() { + assertSize(0); + assertTrue(indexer.addString("abcdef")); + assertTrue(indexer.addString("abcdgh")); + assertFalse(indexer.addString("abcd")); + assertTrue(indexer.addString("ab")); + } + } + + @RunWith(JUnit4.class) + public static class CanonicalStringIndexerTest extends StringIndexerTest{ + @Override + protected StringIndexer newIndexer() { + return new CanonicalStringIndexer(new MapMaker().<String, Integer>makeMap(), + new MapMaker().<Integer, String>makeMap()); + } + + @Test + public void basicOperations() { + assertSize(0); + assertNoIndex("abcdef"); + assertIndex(0, "abcdef"); + assertIndex(0, "abcdef"); + assertSize(1); + assertIndex(1, "abddef"); + assertSize(2); + assertIndex(2, "ab"); + assertSize(3); + assertIndex(3, "abcdefghik"); + assertSize(4); + assertIndex(4, "abcdefgh"); + assertSize(5); + assertNoIndex("a"); + assertNoIndex("abc"); + assertNoIndex("abcdefg"); + assertNoIndex("abcdefghil"); + assertNoIndex("abcdefghikl"); + assertContent(); + indexer.clear(); + assertSize(0); + assertNull(indexer.getStringForIndex(0)); + assertNull(indexer.getStringForIndex(1000)); + } + + @Test + public void addStringResult() { + assertSize(0); + assertTrue(indexer.addString("abcdef")); + assertTrue(indexer.addString("abcdgh")); + assertTrue(indexer.addString("abcd")); + assertTrue(indexer.addString("ab")); + assertFalse(indexer.addString("ab")); + } + + protected void setupTestContent() { + assertSize(0); + assertIndex(0, "abcdefghi"); + assertIndex(1, "abcdefjkl"); + assertIndex(2, "abcdefmno"); + assertIndex(3, "abcdefjklpr"); + assertIndex(4, "abcdstr"); + assertIndex(5, "012345"); + assertSize(6); + assertIndex(6, "abcdef"); + assertIndex(7, "abcd"); + assertIndex(8, ""); + assertIndex(2, "abcdefmno"); + assertSize(9); + assertContent(); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/StringTrieTest.java b/src/test/java/com/google/devtools/build/lib/util/StringTrieTest.java new file mode 100644 index 0000000000..dcb9205c54 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/StringTrieTest.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link StringTrie}. + */ +@RunWith(JUnit4.class) +public class StringTrieTest { + @Test + public void empty() { + StringTrie<Integer> cut = new StringTrie<>(); + assertNull(cut.get("")); + assertNull(cut.get("a")); + assertNull(cut.get("ab")); + } + + @Test + public void simple() { + StringTrie<Integer> cut = new StringTrie<>(); + cut.put("a", 1); + cut.put("b", 2); + + assertNull(cut.get("")); + assertEquals(1, cut.get("a").intValue()); + assertEquals(1, cut.get("ab").intValue()); + assertEquals(1, cut.get("abc").intValue()); + + assertEquals(2, cut.get("b").intValue()); + } + + @Test + public void ancestors() { + StringTrie<Integer> cut = new StringTrie<>(); + cut.put("abc", 3); + assertNull(cut.get("")); + assertNull(cut.get("a")); + assertNull(cut.get("ab")); + assertEquals(3, cut.get("abc").intValue()); + assertEquals(3, cut.get("abcd").intValue()); + + cut.put("a", 1); + assertEquals(1, cut.get("a").intValue()); + assertEquals(1, cut.get("ab").intValue()); + assertEquals(3, cut.get("abc").intValue()); + + cut.put("", 0); + assertEquals(0, cut.get("").intValue()); + assertEquals(0, cut.get("b").intValue()); + assertEquals(1, cut.get("a").intValue()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/StringUtilTest.java b/src/test/java/com/google/devtools/build/lib/util/StringUtilTest.java new file mode 100644 index 0000000000..96d9a53b3a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/StringUtilTest.java @@ -0,0 +1,110 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.devtools.build.lib.util.StringUtil.capitalize; +import static com.google.devtools.build.lib.util.StringUtil.indent; +import static com.google.devtools.build.lib.util.StringUtil.joinEnglishList; +import static com.google.devtools.build.lib.util.StringUtil.splitAndInternString; +import static com.google.devtools.build.lib.util.StringUtil.stripSuffix; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * A test for {@link StringUtil}. + */ +@RunWith(JUnit4.class) +public class StringUtilTest { + + @Test + public void testJoinEnglishList() throws Exception { + assertEquals("nothing", + joinEnglishList(Collections.emptyList())); + assertEquals("one", + joinEnglishList(Arrays.asList("one"))); + assertEquals("one or two", + joinEnglishList(Arrays.asList("one", "two"))); + assertEquals("one and two", + joinEnglishList(Arrays.asList("one", "two"), "and")); + assertEquals("one, two or three", + joinEnglishList(Arrays.asList("one", "two", "three"))); + assertEquals("one, two and three", + joinEnglishList(Arrays.asList("one", "two", "three"), "and")); + assertEquals("'one', 'two' and 'three'", + joinEnglishList(Arrays.asList("one", "two", "three"), "and", "'")); + } + + @Test + public void splitAndIntern() throws Exception { + assertEquals(ImmutableList.of(), splitAndInternString(" ")); + assertEquals(ImmutableList.of(), splitAndInternString(null)); + List<String> list1 = splitAndInternString(" x y z z"); + List<String> list2 = splitAndInternString("a z c z"); + + assertEquals(ImmutableList.of("x", "y", "z", "z"), list1); + assertEquals(ImmutableList.of("a", "z", "c", "z"), list2); + assertSame(list1.get(2), list1.get(3)); + assertSame(list1.get(2), list2.get(1)); + assertSame(list2.get(1), list2.get(3)); + } + + @Test + public void listItemsWithLimit() throws Exception { + assertEquals("begin/a, b, c/end", StringUtil.listItemsWithLimit( + new StringBuilder("begin/"), 3, ImmutableList.of("a", "b", "c")).append("/end").toString()); + + assertEquals("begin/a, b, c ...(omitting 2 more item(s))/end", StringUtil.listItemsWithLimit( + new StringBuilder("begin/"), 3, ImmutableList.of("a", "b", "c", "d", "e")) + .append("/end").toString()); + } + + @Test + public void testIndent() throws Exception { + assertEquals("", indent("", 0)); + assertEquals("", indent("", 1)); + assertEquals("a", indent("a", 1)); + assertEquals("\n a", indent("\na", 2)); + assertEquals("a\n b", indent("a\nb", 2)); + assertEquals("a\n b\n c\n d", indent("a\nb\nc\nd", 1)); + assertEquals("\n ", indent("\n", 1)); + } + + @Test + public void testStripSuffix() throws Exception { + assertEquals("", stripSuffix("", "")); + assertEquals(null, stripSuffix("", "a")); + assertEquals("a", stripSuffix("a", "")); + assertEquals("a", stripSuffix("aa", "a")); + assertEquals(null, stripSuffix("ab", "c")); + } + + @Test + public void testCapitalize() throws Exception { + assertEquals("", capitalize("")); + assertEquals("Joe", capitalize("joe")); + assertEquals("Joe", capitalize("Joe")); + assertEquals("O", capitalize("o")); + assertEquals("O", capitalize("O")); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/StringUtilitiesTest.java b/src/test/java/com/google/devtools/build/lib/util/StringUtilitiesTest.java new file mode 100644 index 0000000000..c8ed641da0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/StringUtilitiesTest.java @@ -0,0 +1,195 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static com.google.devtools.build.lib.util.StringUtilities.combineKeys; +import static com.google.devtools.build.lib.util.StringUtilities.joinLines; +import static com.google.devtools.build.lib.util.StringUtilities.layoutTable; +import static com.google.devtools.build.lib.util.StringUtilities.prettyPrintBytes; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Maps; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * A test for {@link StringUtilities}. + */ +@RunWith(JUnit4.class) +public class StringUtilitiesTest { + + // Tests of StringUtilities.joinLines() + + @Test + public void emptyLinesYieldsEmptyString() { + assertEquals("", joinLines()); + } + + @Test + public void twoLinesGetjoinedNicely() { + assertEquals("line 1\nline 2", joinLines("line 1", "line 2")); + } + + @Test + public void aTrailingNewlineIsAvailableWhenYouNeedIt() { + assertEquals("two lines\nwith trailing newline\n", + joinLines("two lines", "with trailing newline", "")); + } + + // Tests of StringUtilities.combineKeys() + + /** Simple sanity test of format */ + @Test + public void combineKeysFormat() { + assertEquals("<a><b!!c><!<d!>>", combineKeys("a", "b!c", "<d>")); + } + + /** + * Test that combining different keys gives different results, + * i.e. that there are no collisions. + * We test all combinations of up to 3 keys from the test_keys + * array (defined below). + */ + @Test + public void testCombineKeys() { + // This map is really just used as a set, but + // if the test fails, the values in the map may be + // useful for debugging. + Map<String,String[]> map = new HashMap<>(); + for (int numKeys = 0; numKeys <= 3; numKeys++) { + testCombineKeys(map, numKeys, new String[numKeys]); + } + } + + private void testCombineKeys(Map<String,String[]> map, + int n, String[] keys) { + if (n == 0) { + String[] keys_copy = keys.clone(); + String combined_key = combineKeys(keys_copy); + String[] prev_keys = map.put(combined_key, keys_copy); + if (prev_keys != null) { + fail("combineKeys collision:\n" + + "key sequence 1: " + Arrays.deepToString(prev_keys) + "\n" + + "key sequence 2: " + Arrays.deepToString(keys_copy) + "\n" + + "combined key sequence 1: " + combineKeys(prev_keys) + "\n" + + "combined key sequence 2: " + combineKeys(keys_copy) + "\n"); + } + } else { + for (String key : test_keys) { + keys[n - 1] = key; + testCombineKeys(map, n - 1, keys); + } + } + } + + private static final String[] test_keys = { + // ordinary strings + "", "a", "word", "//depot/foo/bar", + // likely delimiter characters + " ", ",", "\\", "\"", "\'", "\0", "\u00ff", + // strings starting in special delimiter + " foo", ",foo", "\\foo", "\"foo", "\'foo", "\0foo", "\u00fffoo", + // strings ending in special delimiter + "bar ", "bar,", "bar\\", "bar\"", "bar\'", "bar\0", "bar\u00ff", + // white-box testing of the delimiters that combineKeys() uses + "<", ">", "!", "!<", "!>", "!!", "<!", ">!" + }; + + @Test + public void replaceAllLiteral() throws Exception { + assertEquals("ababab", + StringUtilities.replaceAllLiteral("bababa", "ba", "ab")); + assertEquals("", + StringUtilities.replaceAllLiteral("bababa", "ba", "")); + assertEquals("bababa", + StringUtilities.replaceAllLiteral("bababa", "", "ab")); + } + + @Test + public void testLayoutTable() throws Exception { + Map<String, String> data = Maps.newTreeMap(); + data.put("foo", "bar"); + data.put("bang", "baz"); + data.put("lengthy key", "lengthy value"); + + assertEquals(joinLines("bang: baz", + "foo: bar", + "lengthy key: lengthy value"), layoutTable(data)); + } + + @Test + public void testPrettyPrintBytes() { + String[] expected = { + "2B", + "23B", + "234B", + "2345B", + "23KB", + "234KB", + "2345KB", + "23MB", + "234MB", + "2345MB", + "23456MB", + "234GB", + "2345GB", + "23456GB", + }; + double x = 2.3456; + for (int ii = 0; ii < expected.length; ++ii) { + assertEquals(expected[ii], prettyPrintBytes((long) x)); + x = x * 10.0; + } + } + + @Test + public void sanitizeControlChars() { + assertEquals("<?>", StringUtilities.sanitizeControlChars("\000")); + assertEquals("<?>", StringUtilities.sanitizeControlChars("\001")); + assertEquals("\\r", StringUtilities.sanitizeControlChars("\r")); + assertEquals(" abc123", StringUtilities.sanitizeControlChars(" abc123")); + } + + @Test + public void containsSubarray() { + assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "ab".toCharArray())); + assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "de".toCharArray())); + assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "bc".toCharArray())); + assertTrue(StringUtilities.containsSubarray("abcde".toCharArray(), "".toCharArray())); + } + + @Test + public void notContainsSubarray() { + assertFalse(StringUtilities.containsSubarray("abc".toCharArray(), "abcd".toCharArray())); + assertFalse(StringUtilities.containsSubarray("abc".toCharArray(), "def".toCharArray())); + assertFalse(StringUtilities.containsSubarray("abcde".toCharArray(), "bd".toCharArray())); + } + + @Test + public void toPythonStyleFunctionName() { + assertEquals("a", StringUtilities.toPythonStyleFunctionName("a")); + assertEquals("a_b", StringUtilities.toPythonStyleFunctionName("aB")); + assertEquals("a_b_c", StringUtilities.toPythonStyleFunctionName("aBC")); + assertEquals("a_bc_d", StringUtilities.toPythonStyleFunctionName("aBcD")); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/TargetUtilsTest.java b/src/test/java/com/google/devtools/build/lib/util/TargetUtilsTest.java new file mode 100644 index 0000000000..33b0fe36d0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/TargetUtilsTest.java @@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util; + +import static org.junit.Assert.assertEquals; + +import com.google.devtools.build.lib.packages.TargetUtils; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link TargetUtils} + */ +@RunWith(JUnit4.class) +public class TargetUtilsTest { + + @Test + public void getRuleLanguage() { + assertEquals("java", TargetUtils.getRuleLanguage("java_binary")); + assertEquals("foobar", TargetUtils.getRuleLanguage("foobar")); + assertEquals("", TargetUtils.getRuleLanguage("")); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinterTest.java b/src/test/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinterTest.java new file mode 100644 index 0000000000..d75208f40e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/AnsiTerminalPrinterTest.java @@ -0,0 +1,94 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.devtools.build.lib.testutil.MoreAsserts; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; + +/** + * A test for {@link AnsiTerminalPrinter}. + */ +@RunWith(JUnit4.class) +public class AnsiTerminalPrinterTest { + private ByteArrayOutputStream stream; + private AnsiTerminalPrinter printer; + + @Before + public void setUp() throws Exception { + stream = new ByteArrayOutputStream(1000); + printer = new AnsiTerminalPrinter(stream, true); + } + + private void setPlainPrinter() { + printer = new AnsiTerminalPrinter(stream, false); + } + + private void assertString(String string) { + assertEquals(string, stream.toString()); + } + + private void assertRegex(String regex) { + MoreAsserts.assertStdoutContainsRegex(regex, stream.toString(), ""); + } + + @Test + public void testPlainPrinter() throws Exception { + setPlainPrinter(); + printer.print("1" + Mode.INFO + "2" + Mode.ERROR + "3" + Mode.WARNING + "4" + + Mode.DEFAULT + "5"); + assertString("12345"); + } + + @Test + public void testDefaultModeIsDefault() throws Exception { + printer.print("1" + Mode.DEFAULT + "2"); + assertString("12"); + } + + @Test + public void testDuplicateMode() throws Exception { + printer.print("_A_" + Mode.INFO); + printer.print("_B_" + Mode.INFO + "_C_"); + assertRegex("^_A_.+_B__C_$"); + } + + @Test + public void testModeCodes() throws Exception { + printer.print(Mode.INFO + "XXX" + Mode.ERROR + "XXX" + Mode.WARNING +"XXX" + Mode.DEFAULT + + "XXX" + Mode.INFO + "XXX" + Mode.ERROR + "XXX" + Mode.WARNING +"XXX" + Mode.DEFAULT); + String[] codes = stream.toString().split("XXX"); + assertEquals(8, codes.length); + for (int i = 0; i < 4; i++) { + assertTrue(codes[i].length() > 0); + assertEquals(codes[i], codes[i+4]); + } + assertFalse(codes[0].equals(codes[1])); + assertFalse(codes[0].equals(codes[2])); + assertFalse(codes[0].equals(codes[3])); + assertFalse(codes[1].equals(codes[2])); + assertFalse(codes[1].equals(codes[3])); + assertFalse(codes[2].equals(codes[3])); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/DelegatingOutErrTest.java b/src/test/java/com/google/devtools/build/lib/util/io/DelegatingOutErrTest.java new file mode 100644 index 0000000000..ed644d150a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/DelegatingOutErrTest.java @@ -0,0 +1,72 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static com.google.devtools.build.lib.util.StringUtilities.joinLines; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A test for {@link DelegatingOutErr}. + */ +@RunWith(JUnit4.class) +public class DelegatingOutErrTest { + + @Test + public void testNewDelegateIsLikeDevNull() { + DelegatingOutErr delegate = new DelegatingOutErr(); + delegate.printOut("Hello, world.\n"); + delegate.printErr("Feel free to ignore me.\n"); + } + + @Test + public void testSubscribeAndUnsubscribeSink() { + DelegatingOutErr delegate = new DelegatingOutErr(); + delegate.printOut("Nobody will listen to this.\n"); + RecordingOutErr sink = new RecordingOutErr(); + delegate.addSink(sink); + delegate.printOutLn("Hello, sink."); + delegate.removeSink(sink); + delegate.printOutLn("... and alone again ..."); + delegate.addSink(sink); + delegate.printOutLn("How are things?"); + assertEquals("Hello, sink.\nHow are things?\n", sink.outAsLatin1()); + } + + @Test + public void testSubscribeMultipleSinks() { + DelegatingOutErr delegate = new DelegatingOutErr(); + RecordingOutErr left = new RecordingOutErr(); + RecordingOutErr right = new RecordingOutErr(); + delegate.addSink(left); + delegate.printOutLn("left only"); + delegate.addSink(right); + delegate.printOutLn("both"); + delegate.removeSink(left); + delegate.printOutLn("right only"); + delegate.removeSink(right); + delegate.printOutLn("silence"); + delegate.addSink(left); + delegate.addSink(right); + delegate.printOutLn("left and right"); + assertEquals(joinLines("left only", "both", "left and right", ""), + left.outAsLatin1()); + assertEquals(joinLines("both", "right only", "left and right", ""), + right.outAsLatin1()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStreamTest.java b/src/test/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStreamTest.java new file mode 100644 index 0000000000..b060beb50a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/LinePrefixingOutputStreamTest.java @@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * Tests {@link LinePrefixingOutputStream}. + */ +@RunWith(JUnit4.class) +public class LinePrefixingOutputStreamTest { + + private byte[] bytes(String string) { + return string.getBytes(UTF_8); + } + + private String string(byte[] bytes) { + return new String(bytes, UTF_8); + } + + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + private LinePrefixingOutputStream prefixOut = + new LinePrefixingOutputStream("Prefix: ", out); + + @Test + public void testNoOutputUntilNewline() throws IOException { + prefixOut.write(bytes("We won't be seeing any output.")); + assertEquals("", string(out.toByteArray())); + } + + @Test + public void testOutputIfFlushed() throws IOException { + prefixOut.write(bytes("We'll flush after this line.")); + prefixOut.flush(); + assertEquals("Prefix: We'll flush after this line.\n", + string(out.toByteArray())); + } + + @Test + public void testAutoflushUponNewline() throws IOException { + prefixOut.write(bytes("Hello, newline.\n")); + assertEquals("Prefix: Hello, newline.\n", string(out.toByteArray())); + } + + @Test + public void testAutoflushUponEmbeddedNewLine() throws IOException { + prefixOut.write(bytes("Hello line1.\nHello line2.\nHello line3.\n")); + assertEquals( + "Prefix: Hello line1.\nPrefix: Hello line2.\nPrefix: Hello line3.\n", + string(out.toByteArray())); + } + + @Test + public void testBufferMaxLengthFlush() throws IOException { + String junk = "lots of characters of non-newline junk. "; + while (junk.length() < LineFlushingOutputStream.BUFFER_LENGTH) { + junk = junk + junk; + } + junk = junk.substring(0, LineFlushingOutputStream.BUFFER_LENGTH); + + // Also test bug where write on a full buffer blows up + prefixOut.write(bytes(junk + junk)); + prefixOut.write(bytes(junk + junk)); + prefixOut.write(bytes("x")); + assertEquals("Prefix: " + junk + "\n" + "Prefix: " + junk + "\n" + + "Prefix: " + junk + "\n" + "Prefix: " + junk + "\n", + string(out.toByteArray())); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/OutErrTest.java b/src/test/java/com/google/devtools/build/lib/util/io/OutErrTest.java new file mode 100644 index 0000000000..1419f42c47 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/OutErrTest.java @@ -0,0 +1,79 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; + +/** + * Tests {@link OutErr}. + */ +@RunWith(JUnit4.class) +public class OutErrTest { + + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + private ByteArrayOutputStream err = new ByteArrayOutputStream(); + private OutErr outErr = OutErr.create(out, err); + + @Test + public void testRetainsOutErr() { + assertSame(out, outErr.getOutputStream()); + assertSame(err, outErr.getErrorStream()); + } + + @Test + public void testPrintsToOut() { + outErr.printOut("Hello, world."); + assertEquals("Hello, world.", new String(out.toByteArray())); + } + + @Test + public void testPrintsToErr() { + outErr.printErr("Hello, moon."); + assertEquals("Hello, moon.", new String(err.toByteArray())); + } + + @Test + public void testPrintsToOutWithANewline() { + outErr.printOutLn("With a newline."); + assertEquals("With a newline.\n", new String(out.toByteArray())); + } + + @Test + public void testPrintsToErrWithANewline(){ + outErr.printErrLn("With a newline."); + assertEquals("With a newline.\n", new String(err.toByteArray())); + } + + @Test + public void testPrintsTwoLinesToOut() { + outErr.printOutLn("line 1"); + outErr.printOutLn("line 2"); + assertEquals("line 1\nline 2\n", new String(out.toByteArray())); + } + + @Test + public void testPrintsTwoLinesToErr() { + outErr.printErrLn("line 1"); + outErr.printErrLn("line 2"); + assertEquals("line 1\nline 2\n", new String(err.toByteArray())); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/RecordingOutErrTest.java b/src/test/java/com/google/devtools/build/lib/util/io/RecordingOutErrTest.java new file mode 100644 index 0000000000..b55eb4c1b0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/RecordingOutErrTest.java @@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.PrintWriter; + +/** + * A test for {@link RecordingOutErr}. + */ +@RunWith(JUnit4.class) +public class RecordingOutErrTest { + + protected RecordingOutErr getRecordingOutErr() { + return new RecordingOutErr(); + } + + @Test + public void testRecordingOutErrRecords() { + RecordingOutErr outErr = getRecordingOutErr(); + + outErr.printOut("Test"); + outErr.printOutLn("out1"); + PrintWriter writer = new PrintWriter(outErr.getOutputStream()); + writer.println("Testout2"); + writer.flush(); + + outErr.printErr("Test"); + outErr.printErrLn("err1"); + writer = new PrintWriter(outErr.getErrorStream()); + writer.println("Testerr2"); + writer.flush(); + + assertEquals(outErr.outAsLatin1(), "Testout1\nTestout2\n"); + assertEquals(outErr.errAsLatin1(), "Testerr1\nTesterr2\n"); + + assertTrue(outErr.hasRecordedOutput()); + + outErr.reset(); + + assertEquals(outErr.outAsLatin1(), ""); + assertEquals(outErr.errAsLatin1(), ""); + assertFalse(outErr.hasRecordedOutput()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/StreamDemultiplexerTest.java b/src/test/java/com/google/devtools/build/lib/util/io/StreamDemultiplexerTest.java new file mode 100644 index 0000000000..59277df538 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/StreamDemultiplexerTest.java @@ -0,0 +1,150 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import com.google.devtools.build.lib.util.StringUtilities; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.util.Random; + +/** + * Tests {@link StreamDemultiplexer}. + */ +@RunWith(JUnit4.class) +public class StreamDemultiplexerTest { + + private ByteArrayOutputStream out = new ByteArrayOutputStream(); + private ByteArrayOutputStream err = new ByteArrayOutputStream(); + private ByteArrayOutputStream ctl = new ByteArrayOutputStream(); + + private byte[] lines(String... lines) { + try { + return StringUtilities.joinLines(lines).getBytes("ISO-8859-1"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } + + private String toAnsi(ByteArrayOutputStream stream) { + try { + return new String(stream.toByteArray(), "ISO-8859-1"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } + + private byte[] inAnsi(String string) { + try { + return string.getBytes("ISO-8859-1"); + } catch (UnsupportedEncodingException e) { + throw new AssertionError(e); + } + } + + @Test + public void testHelloWorldOnStandardOut() throws Exception { + byte[] multiplexed = lines("@1@", "Hello, world."); + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out); + try { + demux.write(multiplexed); + demux.flush(); + } finally { + demux.close(); + } + assertEquals("Hello, world.", out.toString("ISO-8859-1")); + } + + @Test + public void testOutErrCtl() throws Exception { + byte[] multiplexed = lines("@1@", "out", "@2@", "err", "@3@", "ctl", ""); + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out, err, ctl); + try { + demux.write(multiplexed); + demux.flush(); + } finally { + demux.close(); + } + assertEquals("out", toAnsi(out)); + assertEquals("err", toAnsi(err)); + assertEquals("ctl", toAnsi(ctl)); + } + + @Test + public void testWithoutLineBreaks() throws Exception { + byte[] multiplexed = lines("@1@", "just ", "@1@", "one ", "@1@", "line", ""); + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out); + try { + demux.write(multiplexed); + demux.flush(); + } finally { + demux.close(); + } + assertEquals("just one line", out.toString("ISO-8859-1")); + } + + @Test + public void testLineBreaks() throws Exception { + byte[] multiplexed = lines("@1", "two", "@1", "lines", ""); + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out); + try { + demux.write(multiplexed); + demux.flush(); + assertEquals("two\nlines\n", out.toString("ISO-8859-1")); + } finally { + demux.close(); + } + } + + @Test + public void testMultiplexAndBackWithHelloWorld() throws Exception { + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out); + StreamMultiplexer mux = new StreamMultiplexer(demux); + OutputStream out = mux.createStdout(); + out.write(inAnsi("Hello, world.")); + out.flush(); + assertEquals("Hello, world.", toAnsi(this.out)); + } + + @Test + public void testMultiplexDemultiplexBinaryStress() throws Exception { + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', out, err, ctl); + StreamMultiplexer mux = new StreamMultiplexer(demux); + OutputStream[] muxOuts = {mux.createStdout(), mux.createStderr(), mux.createControl()}; + ByteArrayOutputStream[] expectedOuts = + {new ByteArrayOutputStream(), new ByteArrayOutputStream(), new ByteArrayOutputStream()}; + + Random random = new Random(0xdeadbeef); + for (int round = 0; round < 100; round++) { + byte[] buffer = new byte[random.nextInt(100)]; + random.nextBytes(buffer); + int streamId = random.nextInt(3); + expectedOuts[streamId].write(buffer); + expectedOuts[streamId].flush(); + muxOuts[streamId].write(buffer); + muxOuts[streamId].flush(); + } + assertArrayEquals(expectedOuts[0].toByteArray(), out.toByteArray()); + assertArrayEquals(expectedOuts[1].toByteArray(), err.toByteArray()); + assertArrayEquals(expectedOuts[2].toByteArray(), ctl.toByteArray()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerParallelStressTest.java b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerParallelStressTest.java new file mode 100644 index 0000000000..db833a51e6 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerParallelStressTest.java @@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import com.google.common.io.ByteStreams; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * Exercise {@link StreamMultiplexer} in a parallel setting and ensure there's + * no corruption. + */ +@RunWith(JUnit4.class) +public class StreamMultiplexerParallelStressTest { + + /** + * Characters that could likely cause corruption (they're used as control + * characters). + */ + char[] toughCharsToTry = {'\n', '@', '1', '2', '\0', '0'}; + + /** + * We use a demultiplexer as a simple sanity checker only - that is, we don't + * care what the demultiplexer writes, but we are taking advantage of its + * built in error checking. + */ + OutputStream devNull = ByteStreams.nullOutputStream(); + + StreamDemultiplexer demux = new StreamDemultiplexer((byte)'1', + devNull, devNull, devNull); + + /** + * The multiplexer under test. + */ + StreamMultiplexer mux = new StreamMultiplexer(demux); + + /** + * Streams is the out / err / control output streams of the multiplexer which + * we will write to in parallel. + */ + OutputStream[] streams = { + mux.createStdout(), mux.createStderr(), mux.createControl()}; + + /** + * We will create a bunch of threads that write random data to the streams of + * the mux. + */ + class RandomDataPump implements Callable<Object> { + + private Random random; + + public RandomDataPump(int threadId) { + random = new Random(threadId * 0xdeadbeefL); + } + + @Override + public Object call() throws Exception { + Thread.yield(); + OutputStream out = streams[random.nextInt(2)]; + for (int i = 0; i < 10000; i++) { + switch (random.nextInt(5)) { + case 0: + out.write(random.nextInt()); + break; + case 1: + int index = random.nextInt(toughCharsToTry.length); + out.write(toughCharsToTry[index]); + break; + case 2: + byte[] buffer = new byte[random.nextInt(312)]; + random.nextBytes(buffer); + out.write(buffer); + break; + case 3: + out.flush(); + break; + case 4: + out = streams[random.nextInt(3)]; + break; + } + } + return null; + } + } + + @Test + public void testSingleThreadedStress() throws Exception { + new RandomDataPump(1).call(); + } + + @Test + public void testMultiThreadedStress() + throws InterruptedException, ExecutionException { + ExecutorService service = Executors.newFixedThreadPool(50); + + List<Future<?>> futures = new ArrayList<>(); + for (int threadId = 0; threadId < 50; threadId++) { + futures.add(service.submit(new RandomDataPump(threadId))); + } + for (Future<?> future : futures) { + future.get(); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerTest.java b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerTest.java new file mode 100644 index 0000000000..aef6da376e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/util/io/StreamMultiplexerTest.java @@ -0,0 +1,149 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.util.io; + +import static com.google.devtools.build.lib.util.StringUtilities.joinLines; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import com.google.common.io.ByteStreams; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; + +/** + * Test for {@link StreamMultiplexer}. + */ +@RunWith(JUnit4.class) +public class StreamMultiplexerTest { + + private ByteArrayOutputStream multiplexed; + private OutputStream out; + private OutputStream err; + private OutputStream ctl; + + @Before + public void setUp() throws Exception { + multiplexed = new ByteArrayOutputStream(); + StreamMultiplexer multiplexer = new StreamMultiplexer(multiplexed); + out = multiplexer.createStdout(); + err = multiplexer.createStderr(); + ctl = multiplexer.createControl(); + } + + @Test + public void testEmptyWire() throws IOException { + out.flush(); + err.flush(); + ctl.flush(); + assertEquals(0, multiplexed.toByteArray().length); + } + + private static byte[] getLatin(String string) + throws UnsupportedEncodingException { + return string.getBytes("ISO-8859-1"); + } + + private static String getLatin(byte[] bytes) + throws UnsupportedEncodingException { + return new String(bytes, "ISO-8859-1"); + } + + @Test + public void testHelloWorldOnStdOut() throws IOException { + out.write(getLatin("Hello, world.")); + out.flush(); + assertEquals(joinLines("@1@", "Hello, world.", ""), + getLatin(multiplexed.toByteArray())); + } + + @Test + public void testInterleavedStdoutStderrControl() throws Exception { + out.write(getLatin("Hello, stdout.")); + out.flush(); + err.write(getLatin("Hello, stderr.")); + err.flush(); + ctl.write(getLatin("Hello, control.")); + ctl.flush(); + out.write(getLatin("... and back!")); + out.flush(); + assertEquals(joinLines("@1@", "Hello, stdout.", + "@2@", "Hello, stderr.", + "@3@", "Hello, control.", + "@1@", "... and back!", + ""), + getLatin(multiplexed.toByteArray())); + } + + @Test + public void testWillNotCommitToUnderlyingStreamUnlessFlushOrNewline() + throws Exception { + out.write(getLatin("There are no newline characters in here, so it won't" + + " get written just yet.")); + assertArrayEquals(multiplexed.toByteArray(), new byte[0]); + } + + @Test + public void testNewlineTriggersFlush() throws Exception { + out.write(getLatin("No newline just yet, so no flushing. ")); + assertArrayEquals(multiplexed.toByteArray(), new byte[0]); + out.write(getLatin("OK, here we go:\nAnd more to come.")); + + String expected = joinLines("@1", + "No newline just yet, so no flushing. OK, here we go:", ""); + + assertEquals(expected, getLatin(multiplexed.toByteArray())); + + out.write((byte) '\n'); + expected += joinLines("@1", "And more to come.", ""); + + assertEquals(expected, getLatin(multiplexed.toByteArray())); + } + + @Test + public void testFlush() throws Exception { + out.write(getLatin("Don't forget to flush!")); + assertArrayEquals(new byte[0], multiplexed.toByteArray()); + out.flush(); // now the output will appear in multiplexed. + assertEquals(joinLines("@1@", "Don't forget to flush!", ""), + getLatin(multiplexed.toByteArray())); + } + + @Test + public void testByteEncoding() throws IOException { + OutputStream devNull = ByteStreams.nullOutputStream(); + StreamDemultiplexer demux = new StreamDemultiplexer((byte) '1', devNull); + StreamMultiplexer mux = new StreamMultiplexer(demux); + OutputStream out = mux.createStdout(); + + // When we cast 266 to a byte, we get 10. So basically, we ended up + // comparing 266 with 10 as an integer (because out.write takes an int), + // and then later cast it to 10. This way we'd end up with a control + // character \n in the middle of the payload which would then screw things + // up when the real control character arrived. The fixed version of the + // StreamMultiplexer avoids this problem by always casting to a byte before + // carrying out any comparisons. + + out.write(266); + out.write(10); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemConcurrencyTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemConcurrencyTest.java new file mode 100644 index 0000000000..c90de18082 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemConcurrencyTest.java @@ -0,0 +1,97 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.vfs.util.FileSystems; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.io.IOException; + +/** + * (Slow) tests of FileSystem under concurrency. + * + * These tests are nondeterministic but provide good coverage nonetheless. + */ +@RunWith(JUnit4.class) +public class FileSystemConcurrencyTest { + + Path workingDir; + + @Before + public void setUp() throws Exception { + FileSystem testFS = FileSystems.initDefaultAsNative(); + + // Resolve symbolic links in the temp dir: + workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath()); + } + + @Test + public void testConcurrentSymlinkModifications() throws Exception { + final Path xFile = workingDir.getRelative("file"); + FileSystemUtils.createEmptyFile(xFile); + + final Path xLinkToFile = workingDir.getRelative("link"); + + // "Boxed" for pass-by-reference. + final boolean[] run = { true }; + final IOException[] exception = { null }; + Thread createThread = new Thread() { + @Override + public void run() { + while (run[0]) { + if (!xLinkToFile.exists()) { + try { + xLinkToFile.createSymbolicLink(xFile); + } catch (IOException e) { + exception[0] = e; + return; + } + } + } + } + }; + Thread deleteThread = new Thread() { + @Override + public void run() { + while (run[0]) { + if (xLinkToFile.exists(Symlinks.NOFOLLOW)) { + try { + xLinkToFile.delete(); + } catch (IOException e) { + exception[0] = e; + return; + } + } + } + } + }; + createThread.start(); + deleteThread.start(); + Thread.sleep(1000); + run[0] = false; + createThread.join(0); + deleteThread.join(0); + + if (exception[0] != null) { + throw exception[0]; + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemTest.java new file mode 100644 index 0000000000..b6e88d8976 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemTest.java @@ -0,0 +1,1356 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Preconditions; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.util.Fingerprint; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class handles the generic tests that any filesystem must pass. + * + * <p>Each filesystem-test should inherit from this class, thereby obtaining + * all the tests. + */ +public abstract class FileSystemTest { + + private long savedTime; + protected FileSystem testFS; + protected boolean supportsSymlinks; + protected Path workingDir; + + // Some useful examples of various kinds of files (mnemonic: "x" = "eXample") + protected Path xNothing; + protected Path xFile; + protected Path xNonEmptyDirectory; + protected Path xNonEmptyDirectoryFoo; + protected Path xEmptyDirectory; + + @Before + public void setUp() throws Exception { + testFS = getFreshFileSystem(); + workingDir = testFS.getPath(getTestTmpDir()); + cleanUpWorkingDirectory(workingDir); + supportsSymlinks = testFS.supportsSymbolicLinks(); + + // % ls -lR + // -rw-rw-r-- xFile + // drwxrwxr-x xNonEmptyDirectory + // -rw-rw-r-- xNonEmptyDirectory/foo + // drwxrwxr-x xEmptyDirectory + + xNothing = absolutize("xNothing"); + xFile = absolutize("xFile"); + xNonEmptyDirectory = absolutize("xNonEmptyDirectory"); + xNonEmptyDirectoryFoo = xNonEmptyDirectory.getChild("foo"); + xEmptyDirectory = absolutize("xEmptyDirectory"); + + FileSystemUtils.createEmptyFile(xFile); + xNonEmptyDirectory.createDirectory(); + FileSystemUtils.createEmptyFile(xNonEmptyDirectoryFoo); + xEmptyDirectory.createDirectory(); + } + + @After + public void tearDown() throws Exception { + destroyFileSystem(testFS); + } + + /** + * Returns an instance of the file system to test. + */ + protected abstract FileSystem getFreshFileSystem() throws IOException; + + protected boolean isSymbolicLink(File file) { + return com.google.devtools.build.lib.unix.FilesystemUtils.isSymbolicLink(file); + } + + protected void setWritable(File file) throws IOException { + com.google.devtools.build.lib.unix.FilesystemUtils.setWritable(file); + } + + protected void setExecutable(File file) throws IOException { + com.google.devtools.build.lib.unix.FilesystemUtils.setExecutable(file); + } + + private static final Pattern STAT_SUBDIR_ERROR = Pattern.compile("(.*) \\(Not a directory\\)"); + + // Test that file is not present, using statIfFound. Base implementation throws an exception, but + // subclasses may override statIfFound to return null, in which case their tests should override + // this method. + @SuppressWarnings("unused") // Subclasses may throw. + protected void expectNotFound(Path path) throws IOException { + try { + assertNull(path.statIfFound()); + } catch (IOException e) { + // May be because of a non-directory path component. Parse exception to check this. + Matcher matcher = STAT_SUBDIR_ERROR.matcher(e.getMessage()); + if (!matcher.matches() || !path.getPathString().startsWith(matcher.group(1))) { + // Throw if this doesn't match what an ENOTDIR error looks like. + throw e; + } + } + } + + /** + * Removes all stuff from the test filesystem. + */ + protected void destroyFileSystem(FileSystem fileSystem) throws IOException { + Preconditions.checkArgument(fileSystem.equals(workingDir.getFileSystem())); + cleanUpWorkingDirectory(workingDir); + } + + /** + * Cleans up the working directory by removing everything. + */ + protected void cleanUpWorkingDirectory(Path workingPath) + throws IOException { + if (workingPath.exists()) { + removeEntireDirectory(workingPath.getPathFile()); // uses java.io.File! + } + FileSystemUtils.createDirectoryAndParents(workingPath); + } + + /** + * This function removes an entire directory and all of its contents. + * Much like rm -rf directoryToRemove + */ + protected void removeEntireDirectory(File directoryToRemove) + throws IOException { + // make sure that we do not remove anything outside the test directory + Path testDirPath = testFS.getPath(getTestTmpDir()); + if (!testFS.getPath(directoryToRemove.getAbsolutePath()).startsWith(testDirPath)) { + throw new IOException("trying to remove files outside of the testdata directory"); + } + // Some tests set the directories read-only and/or non-executable, so + // override that: + setWritable(directoryToRemove); + setExecutable(directoryToRemove); + + File[] files = directoryToRemove.listFiles(); + if (files != null) { + for (File currentFile : files) { + boolean isSymbolicLink = isSymbolicLink(currentFile); + if (!isSymbolicLink && currentFile.isDirectory()) { + removeEntireDirectory(currentFile); + } else { + if (!isSymbolicLink) { + setWritable(currentFile); + } + if (!currentFile.delete()) { + throw new IOException("Failed to delete '" + currentFile + "'"); + } + } + } + } + if (!directoryToRemove.delete()) { + throw new IOException("Failed to delete '" + directoryToRemove + "'"); + } + } + + /** + * Returns the directory to use as the FileSystem's working directory. + * Canonicalized to make tests hermetic against symbolic links in TEST_TMPDIR. + */ + protected final String getTestTmpDir() throws IOException { + return new File(TestUtils.tmpDir()).getCanonicalPath() + "/testdir"; + } + + /** + * Indirection to create links so we can test FileSystems that do not support + * link creation. For example, JavaFileSystemTest overrides this method + * and creates the link with an alternate FileSystem. + */ + protected void createSymbolicLink(Path link, Path target) throws IOException { + createSymbolicLink(link, target.asFragment()); + } + + /** + * Indirection to create links so we can test FileSystems that do not support + * link creation. For example, JavaFileSystemTest overrides this method + * and creates the link with an alternate FileSystem. + */ + protected void createSymbolicLink(Path link, PathFragment target) throws IOException { + link.createSymbolicLink(target); + } + + /** + * Indirection to setReadOnly(false) on FileSystems that do not + * support setReadOnly(false). For example, JavaFileSystemTest overrides this + * method and makes the Path writable with an alternate FileSystem. + */ + protected void makeWritable(Path target) throws IOException { + target.setWritable(true); + } + + /** + * Indirection to {@link Path#setExecutable(boolean)} on FileSystems that do + * not support setExecutable. For example, JavaFileSystemTest overrides this + * method and makes the Path executable with an alternate FileSystem. + */ + protected void setExecutable(Path target, boolean mode) throws IOException { + target.setExecutable(mode); + } + + // TODO(bazel-team): (2011) Put in a setLastModifiedTime into the various objects + // and clobber the current time of the object we're currently handling. + // Otherwise testing the thing might get a little hard, depending on the clock. + void storeReferenceTime(long timeToMark) { + savedTime = timeToMark; + } + + boolean isLaterThanreferenceTime(long testTime) { + return (savedTime <= testTime); + } + + Path getTestFile() throws IOException { + Path tempPath = absolutize("test-file"); + FileSystemUtils.createEmptyFile(tempPath); + return tempPath; + } + + protected Path absolutize(String relativePathName) { + return workingDir.getRelative(relativePathName); + } + + // Here the tests begin. + + @Test + public void testIsFileForNonexistingPath() { + Path nonExistingPath = testFS.getPath("/something/strange"); + assertFalse(nonExistingPath.isFile()); + } + + @Test + public void testIsDirectoryForNonexistingPath() { + Path nonExistingPath = testFS.getPath("/something/strange"); + assertFalse(nonExistingPath.isDirectory()); + } + + @Test + public void testIsLinkForNonexistingPath() { + Path nonExistingPath = testFS.getPath("/something/strange"); + assertFalse(nonExistingPath.isSymbolicLink()); + } + + @Test + public void testExistsForNonexistingPath() throws Exception { + Path nonExistingPath = testFS.getPath("/something/strange"); + assertFalse(nonExistingPath.exists()); + expectNotFound(nonExistingPath); + } + + @Test + public void testBadPermissionsThrowsExceptionOnStatIfFound() throws Exception { + Path inaccessible = absolutize("inaccessible"); + inaccessible.createDirectory(); + Path child = inaccessible.getChild("child"); + FileSystemUtils.createEmptyFile(child); + inaccessible.setExecutable(false); + assertFalse(child.exists()); + try { + child.statIfFound(); + fail(); + } catch (IOException expected) { + // Expected. + } + } + + @Test + public void testStatIfFoundReturnsNullForChildOfNonDir() throws Exception { + Path foo = absolutize("foo"); + foo.createDirectory(); + Path nonDir = foo.getRelative("bar"); + FileSystemUtils.createEmptyFile(nonDir); + assertNull(nonDir.getRelative("file").statIfFound()); + } + + // The following tests check the handling of the current working directory. + @Test + public void testCreatePathRelativeToWorkingDirectory() { + Path relativeCreatedPath = absolutize("some-file"); + Path expectedResult = workingDir.getRelative(new PathFragment("some-file")); + + assertEquals(expectedResult, relativeCreatedPath); + } + + // The following tests check the handling of the root directory + @Test + public void testRootIsDirectory() { + Path rootPath = testFS.getPath("/"); + assertTrue(rootPath.isDirectory()); + } + + @Test + public void testRootHasNoParent() { + Path rootPath = testFS.getPath("/"); + assertNull(rootPath.getParentDirectory()); + } + + // The following functions test the creation of files/links/directories. + @Test + public void testFileExists() throws Exception { + Path someFile = absolutize("some-file"); + FileSystemUtils.createEmptyFile(someFile); + assertTrue(someFile.exists()); + assertNotNull(someFile.statIfFound()); + } + + @Test + public void testFileIsFile() throws Exception { + Path someFile = absolutize("some-file"); + FileSystemUtils.createEmptyFile(someFile); + assertTrue(someFile.isFile()); + } + + @Test + public void testFileIsNotDirectory() throws Exception { + Path someFile = absolutize("some-file"); + FileSystemUtils.createEmptyFile(someFile); + assertFalse(someFile.isDirectory()); + } + + @Test + public void testFileIsNotSymbolicLink() throws Exception { + Path someFile = absolutize("some-file"); + FileSystemUtils.createEmptyFile(someFile); + assertFalse(someFile.isSymbolicLink()); + } + + @Test + public void testDirectoryExists() throws Exception { + Path someDirectory = absolutize("some-dir"); + someDirectory.createDirectory(); + assertTrue(someDirectory.exists()); + assertNotNull(someDirectory.statIfFound()); + } + + @Test + public void testDirectoryIsDirectory() throws Exception { + Path someDirectory = absolutize("some-dir"); + someDirectory.createDirectory(); + assertTrue(someDirectory.isDirectory()); + } + + @Test + public void testDirectoryIsNotFile() throws Exception { + Path someDirectory = absolutize("some-dir"); + someDirectory.createDirectory(); + assertFalse(someDirectory.isFile()); + } + + @Test + public void testDirectoryIsNotSymbolicLink() throws Exception { + Path someDirectory = absolutize("some-dir"); + someDirectory.createDirectory(); + assertFalse(someDirectory.isSymbolicLink()); + } + + @Test + public void testSymbolicFileLinkExists() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xFile); + assertTrue(someLink.exists()); + assertNotNull(someLink.statIfFound()); + } + } + + @Test + public void testSymbolicFileLinkIsSymbolicLink() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xFile); + assertTrue(someLink.isSymbolicLink()); + } + } + + @Test + public void testSymbolicFileLinkIsFile() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xFile); + assertTrue(someLink.isFile()); + } + } + + @Test + public void testSymbolicFileLinkIsNotDirectory() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xFile); + assertFalse(someLink.isDirectory()); + } + } + + @Test + public void testSymbolicDirLinkExists() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xEmptyDirectory); + assertTrue(someLink.exists()); + assertNotNull(someLink.statIfFound()); + } + } + + @Test + public void testSymbolicDirLinkIsSymbolicLink() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xEmptyDirectory); + assertTrue(someLink.isSymbolicLink()); + } + } + + @Test + public void testSymbolicDirLinkIsDirectory() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xEmptyDirectory); + assertTrue(someLink.isDirectory()); + } + } + + @Test + public void testSymbolicDirLinkIsNotFile() throws Exception { + if (supportsSymlinks) { + Path someLink = absolutize("some-link"); + someLink.createSymbolicLink(xEmptyDirectory); + assertFalse(someLink.isFile()); + } + } + + @Test + public void testChildOfNonDirectory() throws Exception { + Path somePath = absolutize("file-name"); + FileSystemUtils.createEmptyFile(somePath); + Path childOfNonDir = somePath.getChild("child"); + assertFalse(childOfNonDir.exists()); + expectNotFound(childOfNonDir); + } + + @Test + public void testCreateDirectoryIsEmpty() throws Exception { + Path newPath = xEmptyDirectory.getChild("new-dir"); + newPath.createDirectory(); + assertEquals(newPath.getDirectoryEntries().size(), 0); + } + + @Test + public void testCreateDirectoryIsOnlyChildInParent() throws Exception { + Path newPath = xEmptyDirectory.getChild("new-dir"); + newPath.createDirectory(); + assertEquals(1, newPath.getParentDirectory().getDirectoryEntries().size()); + assertThat(newPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath); + } + + @Test + public void testCreateDirectories() throws Exception { + Path newPath = absolutize("new-dir/sub/directory"); + assertTrue(FileSystemUtils.createDirectoryAndParents(newPath)); + } + + @Test + public void testCreateDirectoriesIsDirectory() throws Exception { + Path newPath = absolutize("new-dir/sub/directory"); + FileSystemUtils.createDirectoryAndParents(newPath); + assertTrue(newPath.isDirectory()); + } + + @Test + public void testCreateDirectoriesIsNotFile() throws Exception { + Path newPath = absolutize("new-dir/sub/directory"); + FileSystemUtils.createDirectoryAndParents(newPath); + assertFalse(newPath.isFile()); + } + + @Test + public void testCreateDirectoriesIsNotSymbolicLink() throws Exception { + Path newPath = absolutize("new-dir/sub/directory"); + FileSystemUtils.createDirectoryAndParents(newPath); + assertFalse(newPath.isSymbolicLink()); + } + + @Test + public void testCreateDirectoriesIsEmpty() throws Exception { + Path newPath = absolutize("new-dir/sub/directory"); + FileSystemUtils.createDirectoryAndParents(newPath); + assertEquals(newPath.getDirectoryEntries().size(), 0); + } + + @Test + public void testCreateDirectoriesIsOnlyChildInParent() throws Exception { + Path newPath = absolutize("new-dir/sub/directory"); + FileSystemUtils.createDirectoryAndParents(newPath); + assertEquals(1, newPath.getParentDirectory().getDirectoryEntries().size()); + assertThat(newPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath); + } + + @Test + public void testCreateEmptyFileIsEmpty() throws Exception { + Path newPath = xEmptyDirectory.getChild("new-file"); + FileSystemUtils.createEmptyFile(newPath); + + assertEquals(newPath.getFileSize(), 0); + } + + @Test + public void testCreateFileIsOnlyChildInParent() throws Exception { + Path newPath = xEmptyDirectory.getChild("new-file"); + FileSystemUtils.createEmptyFile(newPath); + assertEquals(1, newPath.getParentDirectory().getDirectoryEntries().size()); + assertThat(newPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath); + } + + // The following functions test the behavior if errors occur during the + // creation of files/links/directories. + @Test + public void testCreateDirectoryWhereDirectoryAlreadyExists() throws Exception { + assertFalse(xEmptyDirectory.createDirectory()); + } + + @Test + public void testCreateDirectoryWhereFileAlreadyExists() { + try { + xFile.createDirectory(); + fail(); + } catch (IOException e) { + assertEquals(xFile + " (File exists)", e.getMessage()); + } + } + + @Test + public void testCannotCreateDirectoryWithoutExistingParent() throws Exception { + Path newPath = testFS.getPath("/deep/new-dir"); + try { + newPath.createDirectory(); + fail(); + } catch (FileNotFoundException e) { + MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage()); + } + } + + @Test + public void testCannotCreateDirectoryWithReadOnlyParent() throws Exception { + xEmptyDirectory.setWritable(false); + Path xChildOfReadonlyDir = xEmptyDirectory.getChild("x"); + try { + xChildOfReadonlyDir.createDirectory(); + fail(); + } catch (IOException e) { + assertEquals(xChildOfReadonlyDir + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotCreateFileWithoutExistingParent() throws Exception { + Path newPath = testFS.getPath("/non-existing-dir/new-file"); + try { + FileSystemUtils.createEmptyFile(newPath); + fail(); + } catch (FileNotFoundException e) { + MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage()); + } + } + + @Test + public void testCannotCreateFileWithReadOnlyParent() throws Exception { + xEmptyDirectory.setWritable(false); + Path xChildOfReadonlyDir = xEmptyDirectory.getChild("x"); + try { + FileSystemUtils.createEmptyFile(xChildOfReadonlyDir); + fail(); + } catch (IOException e) { + assertEquals(xChildOfReadonlyDir + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotCreateFileWithinFile() throws Exception { + Path newFilePath = absolutize("some-file"); + FileSystemUtils.createEmptyFile(newFilePath); + Path wrongPath = absolutize("some-file/new-file"); + try { + FileSystemUtils.createEmptyFile(wrongPath); + fail(); + } catch (IOException e) { + MoreAsserts.assertEndsWith(" (Not a directory)", e.getMessage()); + } + } + + @Test + public void testCannotCreateDirectoryWithinFile() throws Exception { + Path newFilePath = absolutize("some-file"); + FileSystemUtils.createEmptyFile(newFilePath); + Path wrongPath = absolutize("some-file/new-file"); + try { + wrongPath.createDirectory(); + fail(); + } catch (IOException e) { + MoreAsserts.assertEndsWith(" (Not a directory)", e.getMessage()); + } + } + + // Test directory contents + @Test + public void testCreateMultipleChildren() throws Exception { + Path theDirectory = absolutize("foo/"); + theDirectory.createDirectory(); + Path newPath1 = absolutize("foo/new-file-1"); + Path newPath2 = absolutize("foo/new-file-2"); + Path newPath3 = absolutize("foo/new-file-3"); + + FileSystemUtils.createEmptyFile(newPath1); + FileSystemUtils.createEmptyFile(newPath2); + FileSystemUtils.createEmptyFile(newPath3); + + assertThat(theDirectory.getDirectoryEntries()).containsExactly(newPath1, newPath2, newPath3); + } + + @Test + public void testGetDirectoryEntriesThrowsExceptionWhenRunOnFile() throws Exception { + try { + xFile.getDirectoryEntries(); + fail("No Exception thrown."); + } catch (IOException ex) { + if (ex instanceof FileNotFoundException) { + fail("The method should throw an object of class IOException."); + } + assertEquals(xFile + " (Not a directory)", ex.getMessage()); + } + } + + @Test + public void testGetDirectoryEntriesThrowsExceptionForNonexistingPath() { + Path somePath = testFS.getPath("/non-existing-path"); + try { + somePath.getDirectoryEntries(); + fail("FileNotFoundException not thrown."); + } catch (Exception x) { + assertEquals(somePath + " (No such file or directory)", x.getMessage()); + } + } + + // Test the removal of items + @Test + public void testDeleteDirectory() throws Exception { + assertTrue(xEmptyDirectory.delete()); + } + + @Test + public void testDeleteDirectoryIsNotDirectory() throws Exception { + xEmptyDirectory.delete(); + assertFalse(xEmptyDirectory.isDirectory()); + } + + @Test + public void testDeleteDirectoryParentSize() throws Exception { + int parentSize = workingDir.getDirectoryEntries().size(); + xEmptyDirectory.delete(); + assertEquals(workingDir.getDirectoryEntries().size(), parentSize - 1); + } + + @Test + public void testDeleteFile() throws Exception { + assertTrue(xFile.delete()); + } + + @Test + public void testDeleteFileIsNotFile() throws Exception { + xFile.delete(); + assertFalse(xEmptyDirectory.isFile()); + } + + @Test + public void testDeleteFileParentSize() throws Exception { + int parentSize = workingDir.getDirectoryEntries().size(); + xFile.delete(); + assertEquals(workingDir.getDirectoryEntries().size(), parentSize - 1); + } + + @Test + public void testDeleteRemovesCorrectFile() throws Exception { + Path newPath1 = xEmptyDirectory.getChild("new-file-1"); + Path newPath2 = xEmptyDirectory.getChild("new-file-2"); + Path newPath3 = xEmptyDirectory.getChild("new-file-3"); + + FileSystemUtils.createEmptyFile(newPath1); + FileSystemUtils.createEmptyFile(newPath2); + FileSystemUtils.createEmptyFile(newPath3); + + assertTrue(newPath2.delete()); + assertThat(xEmptyDirectory.getDirectoryEntries()).containsExactly(newPath1, newPath3); + } + + @Test + public void testDeleteNonExistingDir() throws Exception { + Path path = xEmptyDirectory.getRelative("non-existing-dir"); + assertFalse(path.delete()); + } + + @Test + public void testDeleteNotADirectoryPath() throws Exception { + Path path = xFile.getChild("new-file"); + assertFalse(path.delete()); + } + + // Here we test the situations where delete should throw exceptions. + @Test + public void testDeleteNonEmptyDirectoryThrowsException() throws Exception { + try { + xNonEmptyDirectory.delete(); + fail(); + } catch (IOException e) { + assertEquals(xNonEmptyDirectory + " (Directory not empty)", e.getMessage()); + } + } + + @Test + public void testDeleteNonEmptyDirectoryNotDeletedDirectory() throws Exception { + try { + xNonEmptyDirectory.delete(); + fail(); + } catch (IOException e) { + // Expected + } + + assertTrue(xNonEmptyDirectory.isDirectory()); + } + + @Test + public void testDeleteNonEmptyDirectoryNotDeletedFile() throws Exception { + try { + xNonEmptyDirectory.delete(); + fail(); + } catch (IOException e) { + // Expected + } + + assertTrue(xNonEmptyDirectoryFoo.isFile()); + } + + @Test + public void testCannotRemoveRoot() { + Path rootDirectory = testFS.getRootDirectory(); + try { + rootDirectory.delete(); + fail(); + } catch (IOException e) { + String msg = e.getMessage(); + assertTrue(String.format("got %s want EBUSY or ENOTEMPTY", msg), + msg.endsWith(" (Directory not empty)") + || msg.endsWith(" (Device or resource busy)") + || msg.endsWith(" (Is a directory)")); // Happens on OS X. + } + } + + // Test the date functions + @Test + public void testCreateFileChangesTimeOfDirectory() throws Exception { + storeReferenceTime(workingDir.getLastModifiedTime()); + Path newPath = absolutize("new-file"); + FileSystemUtils.createEmptyFile(newPath); + assertTrue(isLaterThanreferenceTime(workingDir.getLastModifiedTime())); + } + + @Test + public void testRemoveFileChangesTimeOfDirectory() throws Exception { + Path newPath = absolutize("new-file"); + FileSystemUtils.createEmptyFile(newPath); + storeReferenceTime(workingDir.getLastModifiedTime()); + newPath.delete(); + assertTrue(isLaterThanreferenceTime(workingDir.getLastModifiedTime())); + } + + // This test is a little bit strange, as we cannot test the progression + // of the time directly. As the Java time and the OS time are slightly different. + // Therefore, we first create an unrelated file to get a notion + // of the current OS time and use that as a baseline. + @Test + public void testCreateFileTimestamp() throws Exception { + Path syncFile = absolutize("sync-file"); + FileSystemUtils.createEmptyFile(syncFile); + + Path newFile = absolutize("new-file"); + storeReferenceTime(syncFile.getLastModifiedTime()); + FileSystemUtils.createEmptyFile(newFile); + assertTrue(isLaterThanreferenceTime(newFile.getLastModifiedTime())); + } + + @Test + public void testCreateDirectoryTimestamp() throws Exception { + Path syncFile = absolutize("sync-file"); + FileSystemUtils.createEmptyFile(syncFile); + + Path newPath = absolutize("new-dir"); + storeReferenceTime(syncFile.getLastModifiedTime()); + assertTrue(newPath.createDirectory()); + assertTrue(isLaterThanreferenceTime(newPath.getLastModifiedTime())); + } + + @Test + public void testWriteChangesModifiedTime() throws Exception { + storeReferenceTime(xFile.getLastModifiedTime()); + FileSystemUtils.writeContentAsLatin1(xFile, "abc19"); + assertTrue(isLaterThanreferenceTime(xFile.getLastModifiedTime())); + } + + @Test + public void testGetLastModifiedTimeThrowsExceptionForNonexistingPath() throws Exception { + Path newPath = testFS.getPath("/non-existing-dir"); + try { + newPath.getLastModifiedTime(); + fail("FileNotFoundException not thrown!"); + } catch (FileNotFoundException x) { + assertEquals(newPath + " (No such file or directory)", x.getMessage()); + } + } + + // Test file size + @Test + public void testFileSizeThrowsExceptionForNonexistingPath() throws Exception { + Path newPath = testFS.getPath("/non-existing-file"); + try { + newPath.getFileSize(); + fail("FileNotFoundException not thrown."); + } catch (FileNotFoundException e) { + assertEquals(newPath + " (No such file or directory)", e.getMessage()); + } + } + + @Test + public void testFileSizeAfterWrite() throws Exception { + String testData = "abc19"; + + FileSystemUtils.writeContentAsLatin1(xFile, testData); + assertEquals(testData.length(), xFile.getFileSize()); + } + + // Testing the input/output routines + @Test + public void testFileWriteAndReadAsLatin1() throws Exception { + String testData = "abc19"; + + FileSystemUtils.writeContentAsLatin1(xFile, testData); + String resultData = new String(FileSystemUtils.readContentAsLatin1(xFile)); + + assertEquals(testData,resultData); + } + + @Test + public void testInputAndOutputStreamEOF() throws Exception { + OutputStream outStream = xFile.getOutputStream(); + outStream.write(1); + outStream.close(); + + InputStream inStream = xFile.getInputStream(); + inStream.read(); + assertEquals(-1, inStream.read()); + inStream.close(); + } + + @Test + public void testInputAndOutputStream() throws Exception { + OutputStream outStream = xFile.getOutputStream(); + for (int i = 33; i < 126; i++) { + outStream.write(i); + } + outStream.close(); + + InputStream inStream = xFile.getInputStream(); + for (int i = 33; i < 126; i++) { + int readValue = inStream.read(); + assertEquals(i,readValue); + } + inStream.close(); + } + + @Test + public void testInputAndOutputStreamAppend() throws Exception { + OutputStream outStream = xFile.getOutputStream(); + for (int i = 33; i < 126; i++) { + outStream.write(i); + } + outStream.close(); + + OutputStream appendOut = xFile.getOutputStream(true); + for (int i = 126; i < 155; i++) { + appendOut.write(i); + } + appendOut.close(); + + InputStream inStream = xFile.getInputStream(); + for (int i = 33; i < 155; i++) { + int readValue = inStream.read(); + assertEquals(i,readValue); + } + inStream.close(); + } + + @Test + public void testInputAndOutputStreamNoAppend() throws Exception { + OutputStream outStream = xFile.getOutputStream(); + outStream.write(1); + outStream.close(); + + OutputStream noAppendOut = xFile.getOutputStream(false); + noAppendOut.close(); + + InputStream inStream = xFile.getInputStream(); + assertEquals(-1, inStream.read()); + inStream.close(); + } + + @Test + public void testGetOutputStreamCreatesFile() throws Exception { + Path newFile = absolutize("does_not_exist_yet.txt"); + + OutputStream out = newFile.getOutputStream(); + out.write(42); + out.close(); + + assertTrue(newFile.isFile()); + } + + @Test + public void testInpuStreamThrowExceptionOnDirectory() throws Exception { + try { + xEmptyDirectory.getOutputStream(); + fail("The Exception was not thrown!"); + } catch (IOException ex) { + assertEquals(xEmptyDirectory + " (Is a directory)", ex.getMessage()); + } + } + + @Test + public void testOutputStreamThrowExceptionOnDirectory() throws Exception { + try { + xEmptyDirectory.getInputStream(); + fail("The Exception was not thrown!"); + } catch (IOException ex) { + assertEquals(xEmptyDirectory + " (Is a directory)", ex.getMessage()); + } + } + + // Test renaming + @Test + public void testCanRenameToUnusedName() throws Exception { + xFile.renameTo(xNothing); + assertFalse(xFile.exists()); + assertTrue(xNothing.isFile()); + } + + @Test + public void testCanRenameFileToExistingFile() throws Exception { + Path otherFile = absolutize("otherFile"); + FileSystemUtils.createEmptyFile(otherFile); + xFile.renameTo(otherFile); // succeeds + assertFalse(xFile.exists()); + assertTrue(otherFile.isFile()); + } + + @Test + public void testCanRenameDirToExistingEmptyDir() throws Exception { + xNonEmptyDirectory.renameTo(xEmptyDirectory); // succeeds + assertFalse(xNonEmptyDirectory.exists()); + assertTrue(xEmptyDirectory.isDirectory()); + assertFalse(xEmptyDirectory.getDirectoryEntries().isEmpty()); + } + + @Test + public void testCantRenameDirToExistingNonEmptyDir() throws Exception { + try { + xEmptyDirectory.renameTo(xNonEmptyDirectory); + fail(); + } catch (IOException e) { + MoreAsserts.assertEndsWith(" (Directory not empty)", e.getMessage()); + } + } + + @Test + public void testCantRenameDirToExistingNonEmptyDirNothingChanged() throws Exception { + try { + xEmptyDirectory.renameTo(xNonEmptyDirectory); + fail(); + } catch (IOException e) { + // Expected + } + + assertTrue(xNonEmptyDirectory.isDirectory()); + assertTrue(xEmptyDirectory.isDirectory()); + assertTrue(xEmptyDirectory.getDirectoryEntries().isEmpty()); + assertFalse(xNonEmptyDirectory.getDirectoryEntries().isEmpty()); + } + + @Test + public void testCantRenameDirToExistingFile() { + try { + xEmptyDirectory.renameTo(xFile); + fail(); + } catch (IOException e) { + assertEquals(xEmptyDirectory + " -> " + xFile + " (Not a directory)", e.getMessage()); + } + } + + @Test + public void testCantRenameDirToExistingFileNothingChanged() { + try { + xEmptyDirectory.renameTo(xFile); + fail(); + } catch (IOException e) { + // Expected + } + + assertTrue(xEmptyDirectory.isDirectory()); + assertTrue(xFile.isFile()); + } + + @Test + public void testCantRenameFileToExistingDir() { + try { + xFile.renameTo(xEmptyDirectory); + fail(); + } catch (IOException e) { + assertEquals(xFile + " -> " + xEmptyDirectory + " (Is a directory)", + e.getMessage()); + } + } + + @Test + public void testCantRenameFileToExistingDirNothingChanged() { + try { + xFile.renameTo(xEmptyDirectory); + fail(); + } catch (IOException e) { + // Expected + } + + assertTrue(xEmptyDirectory.isDirectory()); + assertTrue(xFile.isFile()); + } + + @Test + public void testMoveOnNonExistingFileThrowsException() throws Exception { + Path nonExistingPath = absolutize("non-existing"); + Path targetPath = absolutize("does-not-matter"); + try { + nonExistingPath.renameTo(targetPath); + fail(); + } catch (FileNotFoundException e) { + MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage()); + } + } + + // Test the Paths + @Test + public void testGetPathOnlyAcceptsAbsolutePath() { + try { + testFS.getPath("not-absolute"); + fail("The expected Exception was not thrown."); + } catch (IllegalArgumentException ex) { + assertEquals("not-absolute (not an absolute path)", ex.getMessage()); + } + } + + @Test + public void testGetPathOnlyAcceptsAbsolutePathFragment() { + try { + testFS.getPath(new PathFragment("not-absolute")); + fail("The expected Exception was not thrown."); + } catch (IllegalArgumentException ex) { + assertEquals("not-absolute (not an absolute path)", ex.getMessage()); + } + } + + // Test the access permissions + @Test + public void testNewFilesAreWritable() throws Exception { + assertTrue(xFile.isWritable()); + } + + @Test + public void testNewFilesAreReadable() throws Exception { + assertTrue(xFile.isReadable()); + } + + @Test + public void testNewDirsAreWritable() throws Exception { + assertTrue(xEmptyDirectory.isWritable()); + } + + @Test + public void testNewDirsAreReadable() throws Exception { + assertTrue(xEmptyDirectory.isReadable()); + } + + @Test + public void testNewDirsAreExecutable() throws Exception { + assertTrue(xEmptyDirectory.isExecutable()); + } + + @Test + public void testCannotGetExecutableOnNonexistingFile() throws Exception { + try { + xNothing.isExecutable(); + fail("No exception thrown."); + } catch (FileNotFoundException ex) { + assertEquals(xNothing + " (No such file or directory)", ex.getMessage()); + } + } + + @Test + public void testCannotSetExecutableOnNonexistingFile() throws Exception { + try { + xNothing.setExecutable(true); + fail("No exception thrown."); + } catch (FileNotFoundException ex) { + assertEquals(xNothing + " (No such file or directory)", ex.getMessage()); + } + } + + @Test + public void testCannotGetWritableOnNonexistingFile() throws Exception { + try { + xNothing.isWritable(); + fail("No exception thrown."); + } catch (FileNotFoundException ex) { + assertEquals(xNothing + " (No such file or directory)", ex.getMessage()); + } + } + + @Test + public void testCannotSetWritableOnNonexistingFile() throws Exception { + try { + xNothing.setWritable(false); + fail("No exception thrown."); + } catch (FileNotFoundException ex) { + assertEquals(xNothing + " (No such file or directory)", ex.getMessage()); + } + } + + @Test + public void testSetReadableOnFile() throws Exception { + xFile.setReadable(false); + assertFalse(xFile.isReadable()); + xFile.setReadable(true); + assertTrue(xFile.isReadable()); + } + + @Test + public void testSetWritableOnFile() throws Exception { + xFile.setWritable(false); + assertFalse(xFile.isWritable()); + xFile.setWritable(true); + assertTrue(xFile.isWritable()); + } + + @Test + public void testSetExecutableOnFile() throws Exception { + xFile.setExecutable(true); + assertTrue(xFile.isExecutable()); + xFile.setExecutable(false); + assertFalse(xFile.isExecutable()); + } + + @Test + public void testSetExecutableOnDirectory() throws Exception { + setExecutable(xNonEmptyDirectory, false); + + try { + // We can't map names->inodes in a non-executable directory: + xNonEmptyDirectoryFoo.isWritable(); // i.e. stat + fail(); + } catch (IOException e) { + MoreAsserts.assertEndsWith(" (Permission denied)", e.getMessage()); + } + } + + @Test + public void testWritingToReadOnlyFileThrowsException() throws Exception { + xFile.setWritable(false); + try { + FileSystemUtils.writeContent(xFile, "hello, world!".getBytes()); + fail("No exception thrown."); + } catch (IOException e) { + assertEquals(xFile + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testReadingFromUnreadableFileThrowsException() throws Exception { + FileSystemUtils.writeContent(xFile, "hello, world!".getBytes()); + xFile.setReadable(false); + try { + FileSystemUtils.readContent(xFile); + fail("No exception thrown."); + } catch (IOException e) { + assertEquals(xFile + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotCreateFileInReadOnlyDirectory() throws Exception { + Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar"); + xNonEmptyDirectory.setWritable(false); + + try { + FileSystemUtils.createEmptyFile(xNonEmptyDirectoryBar); + fail("No exception thrown."); + } catch (IOException e) { + assertEquals(xNonEmptyDirectoryBar + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotCreateDirectoryInReadOnlyDirectory() throws Exception { + Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar"); + xNonEmptyDirectory.setWritable(false); + + try { + xNonEmptyDirectoryBar.createDirectory(); + fail("No exception thrown."); + } catch (IOException e) { + assertEquals(xNonEmptyDirectoryBar + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotMoveIntoReadOnlyDirectory() throws Exception { + Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar"); + xNonEmptyDirectory.setWritable(false); + + try { + xFile.renameTo(xNonEmptyDirectoryBar); + fail("No exception thrown."); + } catch (IOException e) { + MoreAsserts.assertEndsWith(" (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotMoveFromReadOnlyDirectory() throws Exception { + xNonEmptyDirectory.setWritable(false); + + try { + xNonEmptyDirectoryFoo.renameTo(xNothing); + fail("No exception thrown."); + } catch (IOException e) { + MoreAsserts.assertEndsWith(" (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotDeleteInReadOnlyDirectory() throws Exception { + xNonEmptyDirectory.setWritable(false); + + try { + xNonEmptyDirectoryFoo.delete(); + fail("No exception thrown."); + } catch (IOException e) { + assertEquals(xNonEmptyDirectoryFoo + " (Permission denied)", e.getMessage()); + } + } + + @Test + public void testCannotCreatSymbolicLinkInReadOnlyDirectory() throws Exception { + Path xNonEmptyDirectoryBar = xNonEmptyDirectory.getChild("bar"); + xNonEmptyDirectory.setWritable(false); + + if (supportsSymlinks) { + try { + createSymbolicLink(xNonEmptyDirectoryBar, xNonEmptyDirectoryFoo); + fail("No exception thrown."); + } catch (IOException e) { + assertEquals(xNonEmptyDirectoryBar + " (Permission denied)", e.getMessage()); + } + } + } + + @Test + public void testGetMD5DigestForEmptyFile() throws Exception { + Fingerprint fp = new Fingerprint(); + fp.addBytes(new byte[0]); + assertEquals(BaseEncoding.base16().lowerCase().encode(xFile.getMD5Digest()), + fp.hexDigestAndReset()); + } + + @Test + public void testGetMD5Digest() throws Exception { + byte[] buffer = new byte[500000]; + for (int i = 0; i < buffer.length; ++i) { + buffer[i] = 1; + } + FileSystemUtils.writeContent(xFile, buffer); + Fingerprint fp = new Fingerprint(); + fp.addBytes(buffer); + assertEquals(BaseEncoding.base16().lowerCase().encode(xFile.getMD5Digest()), + fp.hexDigestAndReset()); + } + + @Test + public void testStatFailsFastOnNonExistingFiles() throws Exception { + try { + xNothing.stat(); + fail("Expected IOException"); + } catch(IOException e) { + // Do nothing. + } + } + + @Test + public void testStatNullableFailsFastOnNonExistingFiles() throws Exception { + assertNull(xNothing.statNullable()); + } + + @Test + public void testResolveSymlinks() throws Exception { + if (supportsSymlinks) { + createSymbolicLink(xNothing, xFile); + FileSystemUtils.createEmptyFile(xFile); + assertEquals(xFile.asFragment(), testFS.resolveOneLink(xNothing)); + assertEquals(xFile, xNothing.resolveSymbolicLinks()); + } + } + + @Test + public void testResolveNonSymlinks() throws Exception { + if (supportsSymlinks) { + assertEquals(null, testFS.resolveOneLink(xFile)); + assertEquals(xFile, xFile.resolveSymbolicLinks()); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemUtilsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemUtilsTest.java new file mode 100644 index 0000000000..21ca39b8f0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemUtilsTest.java @@ -0,0 +1,878 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.appendWithoutExtension; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.commonAncestor; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.copyFile; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.copyTool; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.createDirectoryAndParents; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.deleteTree; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.deleteTreesBelowNotPrefixed; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.longestPathPrefix; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.plantLinkForest; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.relativePath; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.removeExtension; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.touchFile; +import static com.google.devtools.build.lib.vfs.FileSystemUtils.traverseTree; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.testutil.BlazeTestUtils; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * This class tests the file system utilities. + */ +@RunWith(JUnit4.class) +public class FileSystemUtilsTest { + private ManualClock clock; + private FileSystem fileSystem; + private Path workingDir; + + @Before + public void setUp() throws Exception { + clock = new ManualClock(); + fileSystem = new InMemoryFileSystem(clock); + workingDir = fileSystem.getPath("/workingDir"); + } + + Path topDir; + Path file1; + Path file2; + Path aDir; + Path file3; + Path innerDir; + Path link1; + Path dirLink; + Path file4; + + /* + * Build a directory tree that looks like: + * top-dir/ + * file-1 + * file-2 + * a-dir/ + * file-3 + * inner-dir/ + * link-1 => file-4 + * dir-link => a-dir + * file-4 + */ + private void createTestDirectoryTree() throws IOException { + topDir = fileSystem.getPath("/top-dir"); + file1 = fileSystem.getPath("/top-dir/file-1"); + file2 = fileSystem.getPath("/top-dir/file-2"); + aDir = fileSystem.getPath("/top-dir/a-dir"); + file3 = fileSystem.getPath("/top-dir/a-dir/file-3"); + innerDir = fileSystem.getPath("/top-dir/a-dir/inner-dir"); + link1 = fileSystem.getPath("/top-dir/a-dir/inner-dir/link-1"); + dirLink = fileSystem.getPath("/top-dir/a-dir/inner-dir/dir-link"); + file4 = fileSystem.getPath("/file-4"); + + topDir.createDirectory(); + FileSystemUtils.createEmptyFile(file1); + FileSystemUtils.createEmptyFile(file2); + aDir.createDirectory(); + FileSystemUtils.createEmptyFile(file3); + innerDir.createDirectory(); + link1.createSymbolicLink(file4); // simple symlink + dirLink.createSymbolicLink(aDir); // creates link loop + FileSystemUtils.createEmptyFile(file4); + } + + private void checkTestDirectoryTreesBelow(Path toPath) throws IOException { + Path copiedFile1 = toPath.getChild("file-1"); + assertTrue(copiedFile1.exists()); + assertTrue(copiedFile1.isFile()); + + Path copiedFile2 = toPath.getChild("file-2"); + assertTrue(copiedFile2.exists()); + assertTrue(copiedFile2.isFile()); + + Path copiedADir = toPath.getChild("a-dir"); + assertTrue(copiedADir.exists()); + assertTrue(copiedADir.isDirectory()); + Collection<Path> aDirEntries = copiedADir.getDirectoryEntries(); + assertEquals(2, aDirEntries.size()); + + Path copiedFile3 = copiedADir.getChild("file-3"); + assertTrue(copiedFile3.exists()); + assertTrue(copiedFile3.isFile()); + + Path copiedInnerDir = copiedADir.getChild("inner-dir"); + assertTrue(copiedInnerDir.exists()); + assertTrue(copiedInnerDir.isDirectory()); + + Path copiedLink1 = copiedInnerDir.getChild("link-1"); + assertTrue(copiedLink1.exists()); + assertTrue(copiedLink1.isSymbolicLink()); + assertEquals(copiedLink1.resolveSymbolicLinks(), file4); + + Path copiedDirLink = copiedInnerDir.getChild("dir-link"); + assertTrue(copiedDirLink.exists()); + assertTrue(copiedDirLink.isSymbolicLink()); + assertEquals(copiedDirLink.resolveSymbolicLinks(), aDir); + } + + // tests + + @Test + public void testChangeModtime() throws IOException { + Path file = fileSystem.getPath("/my-file"); + try { + BlazeTestUtils.changeModtime(file); + fail(); + } catch (FileNotFoundException e) { + /* ok */ + } + FileSystemUtils.createEmptyFile(file); + long prevMtime = file.getLastModifiedTime(); + BlazeTestUtils.changeModtime(file); + assertFalse(prevMtime == file.getLastModifiedTime()); + } + + @Test + public void testCommonAncestor() { + assertEquals(topDir, commonAncestor(topDir, topDir)); + assertEquals(topDir, commonAncestor(file1, file3)); + assertEquals(topDir, commonAncestor(file1, dirLink)); + } + + @Test + public void testRelativePath() throws IOException { + createTestDirectoryTree(); + assertEquals("file-1", relativePath(topDir, file1).getPathString()); + assertEquals(".", relativePath(topDir, topDir).getPathString()); + assertEquals("a-dir/inner-dir/dir-link", relativePath(topDir, dirLink).getPathString()); + assertEquals("../file-4", relativePath(topDir, file4).getPathString()); + assertEquals("../../../file-4", relativePath(innerDir, file4).getPathString()); + } + + private String longestPathPrefixStr(String path, String... prefixStrs) { + Set<PathFragment> prefixes = new HashSet<>(); + for (String prefix : prefixStrs) { + prefixes.add(new PathFragment(prefix)); + } + PathFragment longest = longestPathPrefix(new PathFragment(path), prefixes); + return longest != null ? longest.getPathString() : null; + } + + @Test + public void testLongestPathPrefix() { + assertEquals("A", longestPathPrefixStr("A/b", "A", "B")); // simple parent + assertEquals("A", longestPathPrefixStr("A", "A", "B")); // self + assertEquals("A/B", longestPathPrefixStr("A/B/c", "A", "A/B")); // want longest + assertNull(longestPathPrefixStr("C/b", "A", "B")); // not found in other parents + assertNull(longestPathPrefixStr("A", "A/B", "B")); // not found in child + assertEquals("A/B/C", longestPathPrefixStr("A/B/C/d/e/f.h", "A/B/C", "B/C/d")); + } + + @Test + public void testRemoveExtension_Strings() throws Exception { + assertEquals("foo", removeExtension("foo.c")); + assertEquals("a/foo", removeExtension("a/foo.c")); + assertEquals("a.b/foo", removeExtension("a.b/foo")); + assertEquals("foo", removeExtension("foo")); + assertEquals("foo", removeExtension("foo.")); + } + + @Test + public void testRemoveExtension_Paths() throws Exception { + assertPath("/foo", removeExtension(fileSystem.getPath("/foo.c"))); + assertPath("/a/foo", removeExtension(fileSystem.getPath("/a/foo.c"))); + assertPath("/a.b/foo", removeExtension(fileSystem.getPath("/a.b/foo"))); + assertPath("/foo", removeExtension(fileSystem.getPath("/foo"))); + assertPath("/foo", removeExtension(fileSystem.getPath("/foo."))); + } + + private static void assertPath(String expected, PathFragment actual) { + assertEquals(expected, actual.getPathString()); + } + + private static void assertPath(String expected, Path actual) { + assertEquals(expected, actual.getPathString()); + } + + @Test + public void testReplaceExtension_Path() throws Exception { + assertPath("/foo/bar.baz", + FileSystemUtils.replaceExtension(fileSystem.getPath("/foo/bar"), ".baz")); + assertPath("/foo/bar.baz", + FileSystemUtils.replaceExtension(fileSystem.getPath("/foo/bar.cc"), ".baz")); + assertPath("/foo.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/foo/"), ".baz")); + assertPath("/foo.baz", + FileSystemUtils.replaceExtension(fileSystem.getPath("/foo.cc/"), ".baz")); + assertPath("/foo.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/foo"), ".baz")); + assertPath("/foo.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/foo.cc"), ".baz")); + assertPath("/.baz", FileSystemUtils.replaceExtension(fileSystem.getPath("/.cc"), ".baz")); + assertEquals(null, FileSystemUtils.replaceExtension(fileSystem.getPath("/"), ".baz")); + } + + @Test + public void testReplaceExtension_PathFragment() throws Exception { + assertPath("foo/bar.baz", + FileSystemUtils.replaceExtension(new PathFragment("foo/bar"), ".baz")); + assertPath("foo/bar.baz", + FileSystemUtils.replaceExtension(new PathFragment("foo/bar.cc"), ".baz")); + assertPath("/foo/bar.baz", + FileSystemUtils.replaceExtension(new PathFragment("/foo/bar"), ".baz")); + assertPath("/foo/bar.baz", + FileSystemUtils.replaceExtension(new PathFragment("/foo/bar.cc"), ".baz")); + assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo/"), ".baz")); + assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo.cc/"), ".baz")); + assertPath("/foo.baz", FileSystemUtils.replaceExtension(new PathFragment("/foo/"), ".baz")); + assertPath("/foo.baz", + FileSystemUtils.replaceExtension(new PathFragment("/foo.cc/"), ".baz")); + assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo"), ".baz")); + assertPath("foo.baz", FileSystemUtils.replaceExtension(new PathFragment("foo.cc"), ".baz")); + assertPath("/foo.baz", FileSystemUtils.replaceExtension(new PathFragment("/foo"), ".baz")); + assertPath("/foo.baz", + FileSystemUtils.replaceExtension(new PathFragment("/foo.cc"), ".baz")); + assertPath(".baz", FileSystemUtils.replaceExtension(new PathFragment(".cc"), ".baz")); + assertEquals(null, FileSystemUtils.replaceExtension(new PathFragment("/"), ".baz")); + assertEquals(null, FileSystemUtils.replaceExtension(new PathFragment(""), ".baz")); + assertPath("foo/bar.baz", + FileSystemUtils.replaceExtension(new PathFragment("foo/bar.pony"), ".baz", ".pony")); + assertPath("foo/bar.baz", + FileSystemUtils.replaceExtension(new PathFragment("foo/bar"), ".baz", "")); + assertEquals(null, FileSystemUtils.replaceExtension(new PathFragment(""), ".baz", ".pony")); + assertEquals(null, + FileSystemUtils.replaceExtension(new PathFragment("foo/bar.pony"), ".baz", ".unicorn")); + } + + @Test + public void testAppendWithoutExtension() throws Exception { + assertPath("libfoo-src.jar", + appendWithoutExtension(new PathFragment("libfoo.jar"), "-src")); + assertPath("foo/libfoo-src.jar", + appendWithoutExtension(new PathFragment("foo/libfoo.jar"), "-src")); + assertPath("java/com/google/foo/libfoo-src.jar", + appendWithoutExtension(new PathFragment("java/com/google/foo/libfoo.jar"), "-src")); + assertPath("libfoo.bar-src.jar", + appendWithoutExtension(new PathFragment("libfoo.bar.jar"), "-src")); + assertPath("libfoo-src", + appendWithoutExtension(new PathFragment("libfoo"), "-src")); + assertPath("libfoo-src.jar", + appendWithoutExtension(new PathFragment("libfoo.jar/"), "-src")); + assertPath("libfoo.src.jar", + appendWithoutExtension(new PathFragment("libfoo.jar"), ".src")); + assertEquals(null, appendWithoutExtension(new PathFragment("/"), "-src")); + assertEquals(null, appendWithoutExtension(new PathFragment(""), "-src")); + } + + @Test + public void testReplaceSegments() { + assertPath( + "poo/bar/baz.cc", + FileSystemUtils.replaceSegments(new PathFragment("foo/bar/baz.cc"), "foo", "poo", true)); + assertPath( + "poo/poo/baz.cc", + FileSystemUtils.replaceSegments(new PathFragment("foo/foo/baz.cc"), "foo", "poo", true)); + assertPath( + "poo/foo/baz.cc", + FileSystemUtils.replaceSegments(new PathFragment("foo/foo/baz.cc"), "foo", "poo", false)); + assertPath( + "foo/bar/baz.cc", + FileSystemUtils.replaceSegments(new PathFragment("foo/bar/baz.cc"), "boo", "poo", true)); + } + + @Test + public void testGetWorkingDirectory() { + String userDir = System.getProperty("user.dir"); + + assertEquals(FileSystemUtils.getWorkingDirectory(fileSystem), + fileSystem.getPath(System.getProperty("user.dir", "/"))); + + System.setProperty("user.dir", "/blah/blah/blah"); + assertEquals(FileSystemUtils.getWorkingDirectory(fileSystem), + fileSystem.getPath("/blah/blah/blah")); + + System.setProperty("user.dir", userDir); + } + + @Test + public void testResolveRelativeToFilesystemWorkingDir() { + PathFragment relativePath = new PathFragment("relative/path"); + assertEquals(workingDir.getRelative(relativePath), + workingDir.getRelative(relativePath)); + + PathFragment absolutePath = new PathFragment("/absolute/path"); + assertEquals(fileSystem.getPath(absolutePath), + workingDir.getRelative(absolutePath)); + } + + @Test + public void testTouchFileCreatesFile() throws IOException { + createTestDirectoryTree(); + Path nonExistingFile = fileSystem.getPath("/previously-non-existing"); + assertFalse(nonExistingFile.exists()); + touchFile(nonExistingFile); + + assertTrue(nonExistingFile.exists()); + } + + @Test + public void testTouchFileAdjustsFileTime() throws IOException { + createTestDirectoryTree(); + Path testFile = file4; + long oldTime = testFile.getLastModifiedTime(); + testFile.setLastModifiedTime(42); + touchFile(testFile); + + assertTrue(testFile.getLastModifiedTime() >= oldTime); + } + + @Test + public void testCopyFile() throws IOException { + createTestDirectoryTree(); + Path originalFile = file1; + byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 }; + FileSystemUtils.writeContent(originalFile, content); + + Path copyTarget = file2; + + copyFile(originalFile, copyTarget); + + assertTrue(Arrays.equals(content, FileSystemUtils.readContent(copyTarget))); + } + + @Test + public void testReadContentWithLimit() throws IOException { + createTestDirectoryTree(); + String str = "this is a test of readContentWithLimit method"; + FileSystemUtils.writeContent(file1, StandardCharsets.ISO_8859_1, str); + assertEquals(readStringFromFile(file1, 0), ""); + assertEquals(readStringFromFile(file1, 10), str.substring(0, 10)); + assertEquals(readStringFromFile(file1, 1000000), str); + } + + private String readStringFromFile(Path file, int limit) throws IOException { + byte[] bytes = FileSystemUtils.readContentWithLimit(file, limit); + return new String(bytes, StandardCharsets.ISO_8859_1); + } + + @Test + public void testAppend() throws IOException { + createTestDirectoryTree(); + FileSystemUtils.writeIsoLatin1(file1, "nobody says "); + FileSystemUtils.writeIsoLatin1(file1, "mary had"); + FileSystemUtils.appendIsoLatin1(file1, "a little lamb"); + assertEquals( + "mary had\na little lamb\n", + new String(FileSystemUtils.readContentAsLatin1(file1))); + } + + @Test + public void testCopyFileAttributes() throws IOException { + createTestDirectoryTree(); + Path originalFile = file1; + byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 }; + FileSystemUtils.writeContent(originalFile, content); + file1.setLastModifiedTime(12345L); + file1.setWritable(false); + file1.setExecutable(false); + + Path copyTarget = file2; + copyFile(originalFile, copyTarget); + + assertEquals(12345L, file2.getLastModifiedTime()); + assertFalse(file2.isExecutable()); + assertFalse(file2.isWritable()); + + file1.setWritable(true); + file1.setExecutable(true); + + copyFile(originalFile, copyTarget); + + assertEquals(12345L, file2.getLastModifiedTime()); + assertTrue(file2.isExecutable()); + assertTrue(file2.isWritable()); + + } + + @Test + public void testCopyFileThrowsExceptionIfTargetCantBeDeleted() throws IOException { + createTestDirectoryTree(); + Path originalFile = file1; + byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 }; + FileSystemUtils.writeContent(originalFile, content); + + try { + copyFile(originalFile, aDir); + fail(); + } catch (IOException ex) { + assertEquals("error copying file: couldn't delete destination: " + + aDir + " (Directory not empty)", + ex.getMessage()); + } + } + + @Test + public void testCopyTool() throws IOException { + createTestDirectoryTree(); + Path originalFile = file1; + byte[] content = new byte[] { 'a', 'b', 'c', 23, 42 }; + FileSystemUtils.writeContent(originalFile, content); + + Path copyTarget = copyTool(topDir.getRelative("file-1"), aDir.getRelative("file-1")); + + assertTrue(Arrays.equals(content, FileSystemUtils.readContent(copyTarget))); + assertEquals(file1.isWritable(), copyTarget.isWritable()); + assertEquals(file1.isExecutable(), copyTarget.isExecutable()); + assertEquals(file1.getLastModifiedTime(), copyTarget.getLastModifiedTime()); + } + + @Test + public void testCopyTreesBelow() throws IOException { + createTestDirectoryTree(); + Path toPath = fileSystem.getPath("/copy-here"); + toPath.createDirectory(); + + FileSystemUtils.copyTreesBelow(topDir, toPath); + checkTestDirectoryTreesBelow(toPath); + } + + @Test + public void testCopyTreesBelowWithOverriding() throws IOException { + createTestDirectoryTree(); + Path toPath = fileSystem.getPath("/copy-here"); + toPath.createDirectory(); + toPath.getChild("file-2"); + + FileSystemUtils.copyTreesBelow(topDir, toPath); + checkTestDirectoryTreesBelow(toPath); + } + + @Test + public void testCopyTreesBelowToSubtree() throws IOException { + createTestDirectoryTree(); + try { + FileSystemUtils.copyTreesBelow(topDir, aDir); + fail("Should not be able to copy a directory to a subdir"); + } catch (IllegalArgumentException expected) { + assertEquals("/top-dir/a-dir is a subdirectory of /top-dir", expected.getMessage()); + } + } + + @Test + public void testCopyFileAsDirectoryTree() throws IOException { + createTestDirectoryTree(); + try { + FileSystemUtils.copyTreesBelow(file1, aDir); + fail("Should not be able to copy a file with copyDirectory method"); + } catch (IOException expected) { + assertEquals("/top-dir/file-1 (Not a directory)", expected.getMessage()); + } + } + + @Test + public void testCopyTreesBelowToFile() throws IOException { + createTestDirectoryTree(); + Path copyDir = fileSystem.getPath("/my-dir"); + Path copySubDir = fileSystem.getPath("/my-dir/subdir"); + FileSystemUtils.createDirectoryAndParents(copySubDir); + try { + FileSystemUtils.copyTreesBelow(copyDir, file4); + fail("Should not be able to copy a directory to a file"); + } catch (IOException expected) { + assertEquals("/file-4 (Not a directory)", expected.getMessage()); + } + } + + @Test + public void testCopyTreesBelowFromUnexistingDir() throws IOException { + createTestDirectoryTree(); + + try { + Path unexistingDir = fileSystem.getPath("/unexisting-dir"); + FileSystemUtils.copyTreesBelow(unexistingDir, aDir); + fail("Should not be able to copy from an unexisting path"); + } catch (FileNotFoundException expected) { + assertEquals("/unexisting-dir (No such file or directory)", expected.getMessage()); + } + } + + @Test + public void testTraverseTree() throws IOException { + createTestDirectoryTree(); + + Collection<Path> paths = traverseTree(topDir, new Predicate<Path>() { + @Override + public boolean apply(Path p) { + return !p.getPathString().contains("a-dir"); + } + }); + assertThat(paths).containsExactly(file1, file2); + } + + @Test + public void testTraverseTreeDeep() throws IOException { + createTestDirectoryTree(); + + Collection<Path> paths = traverseTree(topDir, + Predicates.alwaysTrue()); + assertThat(paths).containsExactly(aDir, + file3, + innerDir, + link1, + file1, + file2, + dirLink); + } + + @Test + public void testTraverseTreeLinkDir() throws IOException { + // Use a new little tree for this test: + // top-dir/ + // dir-link2 => linked-dir + // linked-dir/ + // file + topDir = fileSystem.getPath("/top-dir"); + Path dirLink2 = fileSystem.getPath("/top-dir/dir-link2"); + Path linkedDir = fileSystem.getPath("/linked-dir"); + Path linkedDirFile = fileSystem.getPath("/top-dir/dir-link2/file"); + + topDir.createDirectory(); + linkedDir.createDirectory(); + dirLink2.createSymbolicLink(linkedDir); // simple symlink + FileSystemUtils.createEmptyFile(linkedDirFile); // created through the link + + // traverseTree doesn't follow links: + Collection<Path> paths = traverseTree(topDir, Predicates.alwaysTrue()); + assertThat(paths).containsExactly(dirLink2); + + paths = traverseTree(linkedDir, Predicates.alwaysTrue()); + assertThat(paths).containsExactly(fileSystem.getPath("/linked-dir/file")); + } + + @Test + public void testDeleteTreeCommandDeletesTree() throws IOException { + createTestDirectoryTree(); + Path toDelete = topDir; + deleteTree(toDelete); + + assertTrue(file4.exists()); + assertFalse(topDir.exists()); + assertFalse(file1.exists()); + assertFalse(file2.exists()); + assertFalse(aDir.exists()); + assertFalse(file3.exists()); + } + + @Test + public void testDeleteTreeCommandsDeletesUnreadableDirectories() throws IOException { + createTestDirectoryTree(); + Path toDelete = topDir; + + try { + aDir.setReadable(false); + } catch (UnsupportedOperationException e) { + // For file systems that do not support setting readable attribute to + // false, this test is simply skipped. + + return; + } + + deleteTree(toDelete); + assertFalse(topDir.exists()); + assertFalse(aDir.exists()); + + } + + @Test + public void testDeleteTreeCommandDoesNotFollowLinksOut() throws IOException { + createTestDirectoryTree(); + Path toDelete = topDir; + Path outboundLink = fileSystem.getPath("/top-dir/outbound-link"); + outboundLink.createSymbolicLink(file4); + + deleteTree(toDelete); + + assertTrue(file4.exists()); + assertFalse(topDir.exists()); + assertFalse(file1.exists()); + assertFalse(file2.exists()); + assertFalse(aDir.exists()); + assertFalse(file3.exists()); + } + + @Test + public void testDeleteTreesBelowNotPrefixed() throws IOException { + createTestDirectoryTree(); + deleteTreesBelowNotPrefixed(topDir, new String[] { "file-"}); + assertTrue(file1.exists()); + assertTrue(file2.exists()); + assertFalse(aDir.exists()); + } + + @Test + public void testCreateDirectories() throws IOException { + Path mainPath = fileSystem.getPath("/some/where/deep/in/the/hierarchy"); + assertTrue(createDirectoryAndParents(mainPath)); + assertTrue(mainPath.exists()); + assertFalse(createDirectoryAndParents(mainPath)); + } + + @Test + public void testCreateDirectoriesWhenAncestorIsFile() throws IOException { + Path somewhereDeepIn = fileSystem.getPath("/somewhere/deep/in"); + assertTrue(createDirectoryAndParents(somewhereDeepIn.getParentDirectory())); + FileSystemUtils.createEmptyFile(somewhereDeepIn); + Path theHierarchy = somewhereDeepIn.getChild("the-hierarchy"); + try { + createDirectoryAndParents(theHierarchy); + fail(); + } catch (IOException e) { + assertEquals("/somewhere/deep/in (Not a directory)", e.getMessage()); + } + } + + @Test + public void testCreateDirectoriesWhenSymlinkToDir() throws IOException { + Path somewhereDeepIn = fileSystem.getPath("/somewhere/deep/in"); + assertTrue(createDirectoryAndParents(somewhereDeepIn)); + Path realDir = fileSystem.getPath("/real/dir"); + assertTrue(createDirectoryAndParents(realDir)); + + Path theHierarchy = somewhereDeepIn.getChild("the-hierarchy"); + theHierarchy.createSymbolicLink(realDir); + + assertFalse(createDirectoryAndParents(theHierarchy)); + } + + @Test + public void testCreateDirectoriesWhenSymlinkEmbedded() throws IOException { + Path somewhereDeepIn = fileSystem.getPath("/somewhere/deep/in"); + assertTrue(createDirectoryAndParents(somewhereDeepIn)); + Path realDir = fileSystem.getPath("/real/dir"); + assertTrue(createDirectoryAndParents(realDir)); + + Path the = somewhereDeepIn.getChild("the"); + the.createSymbolicLink(realDir); + + Path theHierarchy = somewhereDeepIn.getChild("hierarchy"); + assertTrue(createDirectoryAndParents(theHierarchy)); + } + + PathFragment createPkg(Path rootA, Path rootB, String pkg) throws IOException { + if (rootA != null) { + createDirectoryAndParents(rootA.getRelative(pkg)); + FileSystemUtils.createEmptyFile(rootA.getRelative(pkg).getChild("file")); + } + if (rootB != null) { + createDirectoryAndParents(rootB.getRelative(pkg)); + FileSystemUtils.createEmptyFile(rootB.getRelative(pkg).getChild("file")); + } + return new PathFragment(pkg); + } + + void assertLinksTo(Path fromRoot, Path toRoot, String relpart) throws IOException { + assertTrue(fromRoot.getRelative(relpart).isSymbolicLink()); + assertEquals(toRoot.getRelative(relpart).asFragment(), + fromRoot.getRelative(relpart).readSymbolicLink()); + } + + void assertIsDir(Path root, String relpart) { + assertTrue(root.getRelative(relpart).isDirectory(Symlinks.NOFOLLOW)); + } + + void dumpTree(Path root, PrintStream out) throws IOException { + out.println("\n" + root); + for (Path p : FileSystemUtils.traverseTree(root, Predicates.alwaysTrue())) { + if (p.isDirectory(Symlinks.NOFOLLOW)) { + out.println(" " + p + "/"); + } else if (p.isSymbolicLink()) { + out.println(" " + p + " => " + p.readSymbolicLink()); + } else { + out.println(" " + p + " [" + p.resolveSymbolicLinks() + "]"); + } + } + } + + @Test + public void testPlantLinkForest() throws IOException { + Path rootA = fileSystem.getPath("/A"); + Path rootB = fileSystem.getPath("/B"); + + ImmutableMap<PathFragment, Path> packageRootMap = ImmutableMap.<PathFragment, Path>builder() + .put(createPkg(rootA, rootB, "pkgA"), rootA) + .put(createPkg(rootA, rootB, "dir1/pkgA"), rootA) + .put(createPkg(rootA, rootB, "dir1/pkgB"), rootB) + .put(createPkg(rootA, rootB, "dir2/pkg"), rootA) + .put(createPkg(rootA, rootB, "dir2/pkg/pkg"), rootB) + .put(createPkg(rootA, rootB, "pkgB"), rootB) + .put(createPkg(rootA, rootB, "pkgB/dir/pkg"), rootA) + .put(createPkg(rootA, rootB, "pkgB/pkg"), rootA) + .put(createPkg(rootA, rootB, "pkgB/pkg/pkg"), rootA) + .build(); + createPkg(rootA, rootB, "pkgB/dir"); // create a file in there + + //dumpTree(rootA, System.err); + //dumpTree(rootB, System.err); + + Path linkRoot = fileSystem.getPath("/linkRoot"); + createDirectoryAndParents(linkRoot); + plantLinkForest(packageRootMap, linkRoot); + + //dumpTree(linkRoot, System.err); + + assertLinksTo(linkRoot, rootA, "pkgA"); + assertIsDir(linkRoot, "dir1"); + assertLinksTo(linkRoot, rootA, "dir1/pkgA"); + assertLinksTo(linkRoot, rootB, "dir1/pkgB"); + assertIsDir(linkRoot, "dir2"); + assertIsDir(linkRoot, "dir2/pkg"); + assertLinksTo(linkRoot, rootA, "dir2/pkg/file"); + assertLinksTo(linkRoot, rootB, "dir2/pkg/pkg"); + assertIsDir(linkRoot, "pkgB"); + assertIsDir(linkRoot, "pkgB/dir"); + assertLinksTo(linkRoot, rootB, "pkgB/dir/file"); + assertLinksTo(linkRoot, rootA, "pkgB/dir/pkg"); + assertLinksTo(linkRoot, rootA, "pkgB/pkg"); + } + + @Test + public void testWriteIsoLatin1() throws Exception { + Path file = fileSystem.getPath("/does/not/exist/yet.txt"); + FileSystemUtils.writeIsoLatin1(file, "Line 1", "Line 2", "Line 3"); + String expected = "Line 1\nLine 2\nLine 3\n"; + String actual = new String(FileSystemUtils.readContentAsLatin1(file)); + assertEquals(expected, actual); + } + + @Test + public void testWriteLinesAs() throws Exception { + Path file = fileSystem.getPath("/does/not/exist/yet.txt"); + FileSystemUtils.writeLinesAs(file, UTF_8, "\u00F6"); // an oe umlaut + byte[] expected = new byte[] {(byte) 0xC3, (byte) 0xB6, 0x0A};//"\u00F6\n"; + byte[] actual = FileSystemUtils.readContent(file); + assertArrayEquals(expected, actual); + } + + @Test + public void testGetFileSystem() throws Exception { + Path mountTable = fileSystem.getPath("/proc/mounts"); + FileSystemUtils.writeIsoLatin1(mountTable, + "/dev/sda1 / ext2 blah 0 0", + "/dev/mapper/_dev_sda6 /usr/local/google ext3 blah 0 0", + "devshm /dev/shm tmpfs blah 0 0", + "/dev/fuse /fuse/mnt fuse blah 0 0", + "mtvhome22.nfs:/vol/mtvhome22/johndoe /home/johndoe nfs blah 0 0", + "/dev/foo /foo dummy_foo blah 0 0", + "/dev/foobar /foobar dummy_foobar blah 0 0", + "proc proc proc rw,noexec,nosuid,nodev 0 0"); + Path path = fileSystem.getPath("/usr/local/google/_blaze"); + FileSystemUtils.createDirectoryAndParents(path); + assertEquals("ext3", FileSystemUtils.getFileSystem(path)); + + // Should match the root "/" + path = fileSystem.getPath("/usr/local/tmp"); + FileSystemUtils.createDirectoryAndParents(path); + assertEquals("ext2", FileSystemUtils.getFileSystem(path)); + + // Make sure we don't consider /foobar matches /foo + path = fileSystem.getPath("/foo"); + FileSystemUtils.createDirectoryAndParents(path); + assertEquals("dummy_foo", FileSystemUtils.getFileSystem(path)); + path = fileSystem.getPath("/foobar"); + FileSystemUtils.createDirectoryAndParents(path); + assertEquals("dummy_foobar", FileSystemUtils.getFileSystem(path)); + + path = fileSystem.getPath("/dev/shm/blaze"); + FileSystemUtils.createDirectoryAndParents(path); + assertEquals("tmpfs", FileSystemUtils.getFileSystem(path)); + + Path fusePath = fileSystem.getPath("/fuse/mnt/tmp"); + FileSystemUtils.createDirectoryAndParents(fusePath); + assertEquals("fuse", FileSystemUtils.getFileSystem(fusePath)); + + // Create a symlink and make sure it gives the file system of the symlink target. + path = fileSystem.getPath("/usr/local/google/_blaze/out"); + path.createSymbolicLink(fusePath); + assertEquals("fuse", FileSystemUtils.getFileSystem(path)); + + // Non existent path should return "unknown" + path = fileSystem.getPath("/does/not/exist"); + assertEquals("unknown", FileSystemUtils.getFileSystem(path)); + } + + @Test + public void testStartsWithAnySuccess() throws Exception { + PathFragment a = new PathFragment("a"); + assertTrue(FileSystemUtils.startsWithAny(a, + Arrays.asList(new PathFragment("b"), new PathFragment("a")))); + } + + @Test + public void testStartsWithAnyNotFound() throws Exception { + PathFragment a = new PathFragment("a"); + assertFalse(FileSystemUtils.startsWithAny(a, + Arrays.asList(new PathFragment("b"), new PathFragment("c")))); + } + + @Test + public void testIterateLines() throws Exception { + Path file = fileSystem.getPath("/test.txt"); + FileSystemUtils.writeContent(file, ISO_8859_1, "a\nb"); + assertEquals(Arrays.asList("a", "b"), + Lists.newArrayList(FileSystemUtils.iterateLinesAsLatin1(file))); + + FileSystemUtils.writeContent(file, ISO_8859_1, "a\rb"); + assertEquals(Arrays.asList("a", "b"), + Lists.newArrayList(FileSystemUtils.iterateLinesAsLatin1(file))); + + FileSystemUtils.writeContent(file, ISO_8859_1, "a\r\nb"); + assertEquals(Arrays.asList("a", "b"), + Lists.newArrayList(FileSystemUtils.iterateLinesAsLatin1(file))); + } + + @Test + public void testEnsureSymbolicLinkDoesNotMakeUnnecessaryChanges() throws Exception { + PathFragment target = new PathFragment("/b"); + Path file = fileSystem.getPath("/a"); + file.createSymbolicLink(target); + long prevTimeMillis = clock.currentTimeMillis(); + clock.advanceMillis(1000); + FileSystemUtils.ensureSymbolicLink(file, target); + long timestamp = file.getLastModifiedTime(Symlinks.NOFOLLOW); + assertTrue(timestamp == prevTimeMillis); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/FileSystemsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemsTest.java new file mode 100644 index 0000000000..88a000fea7 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/FileSystemsTest.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; + +import com.google.devtools.build.lib.vfs.util.FileSystems; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * This class handles the tests for the FileSystems class. + */ +@RunWith(JUnit4.class) +public class FileSystemsTest { + + @Test + public void testFileSystemsCreatesOnlyOneDefaultNative() { + assertSame(FileSystems.initDefaultAsNative(), + FileSystems.initDefaultAsNative()); + } + + @Test + public void testFileSystemsCreatesOnlyOneDefaultJavaIo() { + assertSame(FileSystems.initDefaultAsJavaIo(), + FileSystems.initDefaultAsJavaIo()); + } + + @Test + public void testFileSystemsCanSwitchDefaults() { + assertNotSame(FileSystems.initDefaultAsNative(), + FileSystems.initDefaultAsJavaIo()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/GlobTest.java b/src/test/java/com/google/devtools/build/lib/vfs/GlobTest.java new file mode 100644 index 0000000000..37b7dc4957 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/GlobTest.java @@ -0,0 +1,417 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Tests {@link UnixGlob} + */ +@RunWith(JUnit4.class) +public class GlobTest { + + private Path tmpPath; + private FileSystem fs; + @Before + public void setUp() throws Exception { + fs = new InMemoryFileSystem(); + tmpPath = fs.getPath("/globtmp"); + for (String dir : ImmutableList.of("foo/bar/wiz", + "foo/barnacle/wiz", + "food/barnacle/wiz", + "fool/barnacle/wiz")) { + FileSystemUtils.createDirectoryAndParents(tmpPath.getRelative(dir)); + } + FileSystemUtils.createEmptyFile(tmpPath.getRelative("foo/bar/wiz/file")); + } + + @Test + public void testQuestionMarkMatch() throws Exception { + assertGlobMatches("foo?", /* => */"food", "fool"); + } + + @Test + public void testQuestionMarkNoMatch() throws Exception { + assertGlobMatches("food/bar?" /* => nothing */); + } + + @Test + public void testStartsWithStar() throws Exception { + assertGlobMatches("*oo", /* => */"foo"); + } + + @Test + public void testStartsWithStarWithMiddleStar() throws Exception { + assertGlobMatches("*f*o", /* => */"foo"); + } + + @Test + public void testEndsWithStar() throws Exception { + assertGlobMatches("foo*", /* => */"foo", "food", "fool"); + } + + @Test + public void testEndsWithStarWithMiddleStar() throws Exception { + assertGlobMatches("f*oo*", /* => */"foo", "food", "fool"); + } + + @Test + public void testMiddleStar() throws Exception { + assertGlobMatches("f*o", /* => */"foo"); + } + + @Test + public void testTwoMiddleStars() throws Exception { + assertGlobMatches("f*o*o", /* => */"foo"); + } + + @Test + public void testSingleStarPatternWithNamedChild() throws Exception { + assertGlobMatches("*/bar", /* => */"foo/bar"); + } + + @Test + public void testSingleStarPatternWithChildGlob() throws Exception { + assertGlobMatches("*/bar*", /* => */ + "foo/bar", "foo/barnacle", "food/barnacle", "fool/barnacle"); + } + + @Test + public void testSingleStarAsChildGlob() throws Exception { + assertGlobMatches("foo/*/wiz", /* => */"foo/bar/wiz", "foo/barnacle/wiz"); + } + + @Test + public void testNoAsteriskAndFilesDontExist() throws Exception { + // Note un-UNIX like semantics: + assertGlobMatches("ceci/n'est/pas/une/globbe" /* => nothing */); + } + + @Test + public void testSingleAsteriskUnderNonexistentDirectory() throws Exception { + // Note un-UNIX like semantics: + assertGlobMatches("not-there/*" /* => nothing */); + } + + @Test + public void testGlobWithNonExistentBase() throws Exception { + Collection<Path> globResult = UnixGlob.forPath(fs.getPath("/does/not/exist")) + .addPattern("*.txt") + .globInterruptible(); + assertEquals(0, globResult.size()); + } + + @Test + public void testGlobUnderFile() throws Exception { + assertGlobMatches("foo/bar/wiz/file/*" /* => nothing */); + } + + @Test + public void testSingleFileExclude() throws Exception { + assertGlobWithExcludeMatches("*", "food", "foo", "fool"); + } + + @Test + public void testExcludeAll() throws Exception { + assertGlobWithExcludeMatches("*", "*"); + } + + @Test + public void testExcludeAllButNoMatches() throws Exception { + assertGlobWithExcludeMatches("not-there", "*"); + } + + @Test + public void testSingleFileExcludeDoesntMatch() throws Exception { + assertGlobWithExcludeMatches("food", "foo", "food"); + } + + @Test + public void testSingleFileExcludeForDirectoryWithChildGlob() + throws Exception { + assertGlobWithExcludeMatches("foo/*", "foo", "foo/bar", "foo/barnacle"); + } + + @Test + public void testChildGlobWithChildExclude() + throws Exception { + assertGlobWithExcludeMatches("foo/*", "foo/*"); + assertGlobWithExcludeMatches("foo/bar", "foo/*"); + assertGlobWithExcludeMatches("foo/bar", "foo/bar"); + assertGlobWithExcludeMatches("foo/bar", "*/bar"); + assertGlobWithExcludeMatches("foo/bar", "*/*"); + assertGlobWithExcludeMatches("foo/bar/wiz", "*/*/*"); + assertGlobWithExcludeMatches("foo/bar/wiz", "foo/*/*"); + assertGlobWithExcludeMatches("foo/bar/wiz", "foo/bar/*"); + assertGlobWithExcludeMatches("foo/bar/wiz", "foo/bar/wiz"); + assertGlobWithExcludeMatches("foo/bar/wiz", "*/bar/wiz"); + assertGlobWithExcludeMatches("foo/bar/wiz", "*/*/wiz"); + assertGlobWithExcludeMatches("foo/bar/wiz", "foo/*/wiz"); + } + + private void assertGlobMatches(String pattern, String... expecteds) + throws Exception { + assertGlobWithExcludesMatches( + Collections.singleton(pattern), Collections.<String>emptyList(), + expecteds); + } + + private void assertGlobMatches(Collection<String> pattern, + String... expecteds) + throws Exception { + assertGlobWithExcludesMatches(pattern, Collections.<String>emptyList(), + expecteds); + } + + private void assertGlobWithExcludeMatches(String pattern, String exclude, + String... expecteds) + throws Exception { + assertGlobWithExcludesMatches( + Collections.singleton(pattern), Collections.singleton(exclude), + expecteds); + } + + private void assertGlobWithExcludesMatches(Collection<String> pattern, + Collection<String> excludes, + String... expecteds) + throws Exception { + MoreAsserts.assertSameContents(resolvePaths(expecteds), + new UnixGlob.Builder(tmpPath) + .addPatterns(pattern) + .addExcludes(excludes) + .globInterruptible()); + } + + private Set<Path> resolvePaths(String... relativePaths) { + Set<Path> expectedFiles = new HashSet<>(); + for (String expected : relativePaths) { + Path file = expected.equals(".") + ? tmpPath + : tmpPath.getRelative(expected); + expectedFiles.add(file); + } + return expectedFiles; + } + + @Test + public void testGlobWithoutWildcardsDoesNotCallReaddir() throws Exception { + UnixGlob.FilesystemCalls syscalls = new UnixGlob.FilesystemCalls() { + @Override + public FileStatus statNullable(Path path, Symlinks symlinks) { + return UnixGlob.DEFAULT_SYSCALLS.statNullable(path, symlinks); + } + + @Override + public Collection<Dirent> readdir(Path path, Symlinks symlinks) { + throw new IllegalStateException(); + } + }; + + MoreAsserts.assertSameContents(ImmutableList.of(tmpPath.getRelative("foo/bar/wiz/file")), + new UnixGlob.Builder(tmpPath) + .addPattern("foo/bar/wiz/file") + .setFilesystemCalls(new AtomicReference<>(syscalls)) + .glob()); + } + + @Test + public void testIllegalPatterns() throws Exception { + assertIllegalPattern("(illegal) pattern"); + assertIllegalPattern("[illegal pattern"); + assertIllegalPattern("}illegal pattern"); + assertIllegalPattern("foo**bar"); + assertIllegalPattern(""); + assertIllegalPattern("."); + assertIllegalPattern("/foo"); + assertIllegalPattern("./foo"); + assertIllegalPattern("foo/"); + assertIllegalPattern("foo/./bar"); + assertIllegalPattern("../foo/bar"); + assertIllegalPattern("foo//bar"); + } + + /** + * Tests that globs can contain Java regular expression special characters + */ + @Test + public void testSpecialRegexCharacter() throws Exception { + Path tmpPath2 = fs.getPath("/globtmp2"); + FileSystemUtils.createDirectoryAndParents(tmpPath2); + Path aDotB = tmpPath2.getChild("a.b"); + FileSystemUtils.createEmptyFile(aDotB); + FileSystemUtils.createEmptyFile(tmpPath2.getChild("aab")); + // Note: this contains two asterisks because otherwise a RE is not built, + // as an optimization. + assertThat(UnixGlob.forPath(tmpPath2).addPattern("*a.b*").globInterruptible()).containsExactly( + aDotB); + } + + @Test + public void testMatchesCallWithNoCache() { + assertTrue(UnixGlob.matches("*a*b", "CaCb", null)); + } + + @Test + public void testMultiplePatterns() throws Exception { + assertGlobMatches(Lists.newArrayList("foo", "fool"), "foo", "fool"); + } + + @Test + public void testMultiplePatternsWithExcludes() throws Exception { + assertGlobWithExcludesMatches(Lists.newArrayList("foo", "foo?"), + Lists.newArrayList("fool"), "foo", "food"); + } + + @Test + public void testMultiplePatternsWithOverlap() throws Exception { + assertGlobMatchesAnyOrder(Lists.newArrayList("food", "foo?"), + "food", "fool"); + assertGlobMatchesAnyOrder(Lists.newArrayList("food", "?ood", "f??d"), + "food"); + assertThat(resolvePaths("food", "fool", "foo")).containsExactlyElementsIn( + new UnixGlob.Builder(tmpPath).addPatterns("food", "xxx", "*").glob()); + + } + + private void assertGlobMatchesAnyOrder(ArrayList<String> patterns, + String... paths) throws Exception { + assertThat(resolvePaths(paths)).containsExactlyElementsIn( + new UnixGlob.Builder(tmpPath).addPatterns(patterns).globInterruptible()); + } + + /** + * Tests that a glob returns files in sorted order. + */ + @Test + public void testGlobEntriesAreSorted() throws Exception { + Collection<Path> directoryEntries = tmpPath.getDirectoryEntries(); + List<Path> globResult = new UnixGlob.Builder(tmpPath) + .addPattern("*") + .setExcludeDirectories(false) + .globInterruptible(); + assertThat(Ordering.natural().sortedCopy(directoryEntries)).containsExactlyElementsIn( + globResult).inOrder(); + } + + private void assertIllegalPattern(String pattern) throws Exception { + try { + new UnixGlob.Builder(tmpPath) + .addPattern(pattern) + .globInterruptible(); + fail(); + } catch (IllegalArgumentException e) { + MoreAsserts.assertContainsRegex("in glob pattern", e.getMessage()); + } + } + + @Test + public void testHiddenFiles() throws Exception { + for (String dir : ImmutableList.of(".hidden", "..also.hidden", "not.hidden")) { + FileSystemUtils.createDirectoryAndParents(tmpPath.getRelative(dir)); + } + // Note that these are not in the result: ".", ".." + assertGlobMatches("*", "not.hidden", "foo", "fool", "food", ".hidden", "..also.hidden"); + assertGlobMatches("*.hidden", "not.hidden"); + } + + @Test + public void testCheckCanBeInterrupted() throws Exception { + final Thread mainThread = Thread.currentThread(); + final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); + + Predicate<Path> interrupterPredicate = new Predicate<Path>() { + @Override + public boolean apply(Path input) { + mainThread.interrupt(); + return true; + } + }; + + try { + new UnixGlob.Builder(tmpPath) + .addPattern("**") + .setDirectoryFilter(interrupterPredicate) + .setThreadPool(executor) + .globInterruptible(); + fail(); // Should have received InterruptedException + } catch (InterruptedException e) { + // good + } + + assertFalse(executor.isShutdown()); + executor.shutdown(); + assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + + @Test + public void testCheckCannotBeInterrupted() throws Exception { + final Thread mainThread = Thread.currentThread(); + final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); + final AtomicBoolean sentInterrupt = new AtomicBoolean(false); + + Predicate<Path> interrupterPredicate = new Predicate<Path>() { + @Override + public boolean apply(Path input) { + if (!sentInterrupt.getAndSet(true)) { + mainThread.interrupt(); + } + return true; + } + }; + + List<Path> result = new UnixGlob.Builder(tmpPath) + .addPatterns("**", "*") + .setDirectoryFilter(interrupterPredicate).setThreadPool(executor).glob(); + + // In the non-interruptible case, the interrupt bit should be set, but the + // glob should return the correct set of full results. + assertTrue(Thread.interrupted()); + MoreAsserts.assertSameContents(resolvePaths(".", "foo", "foo/bar", "foo/bar/wiz", + "foo/bar/wiz/file", "foo/barnacle", "foo/barnacle/wiz", "food", "food/barnacle", + "food/barnacle/wiz", "fool", "fool/barnacle", "fool/barnacle/wiz"), result); + + assertFalse(executor.isShutdown()); + executor.shutdown(); + assertTrue(executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/JavaIoFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/JavaIoFileSystemTest.java new file mode 100644 index 0000000000..fdb6283d20 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/JavaIoFileSystemTest.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for the {@link JavaIoFileSystem}. That file system by itself is not + * capable of creating symlinks; use the unix one to create them, so that the + * test can check that the file system handles their existence correctly. + */ +@RunWith(JUnit4.class) +public class JavaIoFileSystemTest extends SymlinkAwareFileSystemTest { + + @Override + public FileSystem getFreshFileSystem() { + return new JavaIoFileSystem(); + } + + // The tests are just inherited from the FileSystemTest + + // JavaIoFileSystem incorrectly throws a FileNotFoundException for all IO errors. This means that + // statIfFound incorrectly suppresses those errors. + @Override + @Test + public void testBadPermissionsThrowsExceptionOnStatIfFound() {} +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/ModifiedFileSetTest.java b/src/test/java/com/google/devtools/build/lib/vfs/ModifiedFileSetTest.java new file mode 100644 index 0000000000..96001dfa2f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/ModifiedFileSetTest.java @@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.collect.ImmutableList; +import com.google.common.testing.EqualsTester; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link ModifiedFileSet}. + */ +@RunWith(JUnit4.class) +public class ModifiedFileSetTest { + + @Test + public void testHashCodeAndEqualsContract() throws Exception { + PathFragment fragA = new PathFragment("a"); + PathFragment fragB = new PathFragment("b"); + + ModifiedFileSet empty1 = ModifiedFileSet.NOTHING_MODIFIED; + ModifiedFileSet empty2 = ModifiedFileSet.builder().build(); + ModifiedFileSet empty3 = ModifiedFileSet.builder().modifyAll( + ImmutableList.<PathFragment>of()).build(); + + ModifiedFileSet nonEmpty1 = ModifiedFileSet.builder().modifyAll( + ImmutableList.of(fragA, fragB)).build(); + ModifiedFileSet nonEmpty2 = ModifiedFileSet.builder().modifyAll( + ImmutableList.of(fragB, fragA)).build(); + ModifiedFileSet nonEmpty3 = ModifiedFileSet.builder().modify(fragA).modify(fragB).build(); + ModifiedFileSet nonEmpty4 = ModifiedFileSet.builder().modify(fragB).modify(fragA).build(); + + ModifiedFileSet everythingModified = ModifiedFileSet.EVERYTHING_MODIFIED; + + new EqualsTester() + .addEqualityGroup(empty1, empty2, empty3) + .addEqualityGroup(nonEmpty1, nonEmpty2, nonEmpty3, nonEmpty4) + .addEqualityGroup(everythingModified) + .testEquals(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentTest.java new file mode 100644 index 0000000000..9ab9bfa9f3 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentTest.java @@ -0,0 +1,481 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.testing.EqualsTester; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +/** + * This class tests the functionality of the PathFragment. + */ +@RunWith(JUnit4.class) +public class PathFragmentTest { + @Test + public void testMergeFourPathsWithAbsolute() { + assertEquals(new PathFragment("x/y/z/a/b/c/d/e"), + new PathFragment(new PathFragment("x/y"), new PathFragment("z/a"), + new PathFragment("/b/c"), // absolute! + new PathFragment("d/e"))); + } + + @Test + public void testEqualsAndHashCode() { + InMemoryFileSystem filesystem = new InMemoryFileSystem(); + + new EqualsTester() + .addEqualityGroup(new PathFragment("../relative/path"), + new PathFragment("../relative/path"), + new PathFragment(new File("../relative/path"))) + .addEqualityGroup(new PathFragment("something/else")) + .addEqualityGroup(new PathFragment("/something/else")) + .addEqualityGroup(new PathFragment("/"), + new PathFragment("//////")) + .addEqualityGroup(new PathFragment("")) + .addEqualityGroup(filesystem.getRootDirectory()) // A Path object. + .testEquals(); + } + + @Test + public void testHashCodeCache() { + PathFragment relativePath = new PathFragment("../relative/path"); + PathFragment rootPath = new PathFragment("/"); + + int oldResult = relativePath.hashCode(); + int rootResult = rootPath.hashCode(); + assertEquals(oldResult, relativePath.hashCode()); + assertEquals(rootResult, rootPath.hashCode()); + } + + private void checkRelativeTo(String path, String base) { + PathFragment relative = new PathFragment(path).relativeTo(base); + assertEquals(new PathFragment(path), new PathFragment(base).getRelative(relative).normalize()); + } + + @Test + public void testRelativeTo() { + assertPath("bar/baz", new PathFragment("foo/bar/baz").relativeTo("foo")); + assertPath("bar/baz", new PathFragment("/foo/bar/baz").relativeTo("/foo")); + assertPath("baz", new PathFragment("foo/bar/baz").relativeTo("foo/bar")); + assertPath("baz", new PathFragment("/foo/bar/baz").relativeTo("/foo/bar")); + assertPath("foo", new PathFragment("/foo").relativeTo("/")); + assertPath("foo", new PathFragment("foo").relativeTo("")); + assertPath("foo/bar", new PathFragment("foo/bar").relativeTo("")); + + checkRelativeTo("foo/bar/baz", "foo"); + checkRelativeTo("/foo/bar/baz", "/foo"); + checkRelativeTo("foo/bar/baz", "foo/bar"); + checkRelativeTo("/foo/bar/baz", "/foo/bar"); + checkRelativeTo("/foo", "/"); + checkRelativeTo("foo", ""); + checkRelativeTo("foo/bar", ""); + } + + @Test + public void testIsAbsolute() { + assertTrue(new PathFragment("/absolute/test").isAbsolute()); + assertFalse(new PathFragment("relative/test").isAbsolute()); + assertTrue(new PathFragment(new File("/absolute/test")).isAbsolute()); + assertFalse(new PathFragment(new File("relative/test")).isAbsolute()); + } + + @Test + public void testIsNormalized() { + assertTrue(new PathFragment("/absolute/path").isNormalized()); + assertTrue(new PathFragment("some//path").isNormalized()); + assertFalse(new PathFragment("some/./path").isNormalized()); + assertFalse(new PathFragment("../some/path").isNormalized()); + assertFalse(new PathFragment("some/other/../path").isNormalized()); + assertTrue(new PathFragment("some/other//tricky..path..").isNormalized()); + assertTrue(new PathFragment("/some/other//tricky..path..").isNormalized()); + } + + @Test + public void testRootNodeReturnsRootString() { + PathFragment rootFragment = new PathFragment("/"); + assertEquals("/", rootFragment.getPathString()); + } + + @Test + public void testGetPathFragmentDoesNotNormalize() { + String nonCanonicalPath = "/a/weird/noncanonical/../path/."; + assertEquals(nonCanonicalPath, + new PathFragment(nonCanonicalPath).getPathString()); + } + + @Test + public void testGetRelative() { + assertEquals("a/b", new PathFragment("a").getRelative("b").getPathString()); + assertEquals("a/b/c/d", new PathFragment("a/b").getRelative("c/d").getPathString()); + assertEquals("/a/b", new PathFragment("c/d").getRelative("/a/b").getPathString()); + assertEquals("a", new PathFragment("a").getRelative("").getPathString()); + assertEquals("/", new PathFragment("/").getRelative("").getPathString()); + } + + @Test + public void testGetChildWorks() { + PathFragment pf = new PathFragment("../some/path"); + assertEquals(new PathFragment("../some/path/hi"), pf.getChild("hi")); + } + + @Test + public void testGetChildRejectsInvalidBaseNames() { + PathFragment pf = new PathFragment("../some/path"); + assertGetChildFails(pf, "."); + assertGetChildFails(pf, ".."); + assertGetChildFails(pf, "x/y"); + assertGetChildFails(pf, "/y"); + assertGetChildFails(pf, "y/"); + assertGetChildFails(pf, ""); + } + + private void assertGetChildFails(PathFragment pf, String baseName) { + try { + pf.getChild(baseName); + fail(); + } catch (Exception e) { /* Expected. */ } + } + + // Tests after here test the canonicalization + private void assertRegular(String expected, String actual) { + assertEquals(expected, new PathFragment(actual).getPathString()); // compare string forms + assertEquals(new PathFragment(expected), new PathFragment(actual)); // compare fragment forms + } + + @Test + public void testEmptyPathToEmptyPath() { + assertRegular("/", "/"); + assertRegular("", ""); + } + + @Test + public void testRedundantSlashes() { + assertRegular("/", "///"); + assertRegular("/foo/bar", "/foo///bar"); + assertRegular("/foo/bar", "////foo//bar"); + } + + @Test + public void testSimpleNameToSimpleName() { + assertRegular("/foo", "/foo"); + assertRegular("foo", "foo"); + } + + @Test + public void testSimplePathToSimplePath() { + assertRegular("/foo/bar", "/foo/bar"); + assertRegular("foo/bar", "foo/bar"); + } + + @Test + public void testStripsTrailingSlash() { + assertRegular("/foo/bar", "/foo/bar/"); + } + + @Test + public void testGetParentDirectory() { + PathFragment fooBarWiz = new PathFragment("foo/bar/wiz"); + PathFragment fooBar = new PathFragment("foo/bar"); + PathFragment foo = new PathFragment("foo"); + PathFragment empty = new PathFragment(""); + assertEquals(fooBar, fooBarWiz.getParentDirectory()); + assertEquals(foo, fooBar.getParentDirectory()); + assertEquals(empty, foo.getParentDirectory()); + assertNull(empty.getParentDirectory()); + + PathFragment fooBarWizAbs = new PathFragment("/foo/bar/wiz"); + PathFragment fooBarAbs = new PathFragment("/foo/bar"); + PathFragment fooAbs = new PathFragment("/foo"); + PathFragment rootAbs = new PathFragment("/"); + assertEquals(fooBarAbs, fooBarWizAbs.getParentDirectory()); + assertEquals(fooAbs, fooBarAbs.getParentDirectory()); + assertEquals(rootAbs, fooAbs.getParentDirectory()); + assertNull(rootAbs.getParentDirectory()); + + // Note, this is surprising but correct behavior: + assertEquals(fooBarAbs, + new PathFragment("/foo/bar/..").getParentDirectory()); + } + + @Test + public void testSegmentsCount() { + assertEquals(2, new PathFragment("foo/bar").segmentCount()); + assertEquals(2, new PathFragment("/foo/bar").segmentCount()); + assertEquals(2, new PathFragment("foo//bar").segmentCount()); + assertEquals(2, new PathFragment("/foo//bar").segmentCount()); + assertEquals(1, new PathFragment("foo/").segmentCount()); + assertEquals(1, new PathFragment("/foo/").segmentCount()); + assertEquals(1, new PathFragment("foo").segmentCount()); + assertEquals(1, new PathFragment("/foo").segmentCount()); + assertEquals(0, new PathFragment("/").segmentCount()); + assertEquals(0, new PathFragment("").segmentCount()); + } + + + @Test + public void testGetSegment() { + assertEquals("foo", new PathFragment("foo/bar").getSegment(0)); + assertEquals("bar", new PathFragment("foo/bar").getSegment(1)); + assertEquals("foo", new PathFragment("/foo/bar").getSegment(0)); + assertEquals("bar", new PathFragment("/foo/bar").getSegment(1)); + assertEquals("foo", new PathFragment("foo/").getSegment(0)); + assertEquals("foo", new PathFragment("/foo/").getSegment(0)); + assertEquals("foo", new PathFragment("foo").getSegment(0)); + assertEquals("foo", new PathFragment("/foo").getSegment(0)); + } + + @Test + public void testBasename() throws Exception { + assertEquals("bar", new PathFragment("foo/bar").getBaseName()); + assertEquals("bar", new PathFragment("/foo/bar").getBaseName()); + assertEquals("foo", new PathFragment("foo/").getBaseName()); + assertEquals("foo", new PathFragment("/foo/").getBaseName()); + assertEquals("foo", new PathFragment("foo").getBaseName()); + assertEquals("foo", new PathFragment("/foo").getBaseName()); + assertEquals("", new PathFragment("/").getBaseName()); + assertEquals("", new PathFragment("").getBaseName()); + } + + private static void assertPath(String expected, PathFragment actual) { + assertEquals(expected, actual.getPathString()); + } + + @Test + public void testReplaceName() throws Exception { + assertPath("foo/baz", new PathFragment("foo/bar").replaceName("baz")); + assertPath("/foo/baz", new PathFragment("/foo/bar").replaceName("baz")); + assertPath("foo", new PathFragment("foo/bar").replaceName("")); + assertPath("baz", new PathFragment("foo/").replaceName("baz")); + assertPath("/baz", new PathFragment("/foo/").replaceName("baz")); + assertPath("baz", new PathFragment("foo").replaceName("baz")); + assertPath("/baz", new PathFragment("/foo").replaceName("baz")); + assertEquals(null, new PathFragment("/").replaceName("baz")); + assertEquals(null, new PathFragment("/").replaceName("")); + assertEquals(null, new PathFragment("").replaceName("baz")); + assertEquals(null, new PathFragment("").replaceName("")); + + assertPath("foo/bar/baz", new PathFragment("foo/bar").replaceName("bar/baz")); + assertPath("foo/bar/baz", new PathFragment("foo/bar").replaceName("bar/baz/")); + + // Absolute path arguments will clobber the original path. + assertPath("/absolute", new PathFragment("foo/bar").replaceName("/absolute")); + assertPath("/", new PathFragment("foo/bar").replaceName("/")); + } + @Test + public void testSubFragment() throws Exception { + assertPath("/foo/bar/baz", + new PathFragment("/foo/bar/baz").subFragment(0, 3)); + assertPath("foo/bar/baz", + new PathFragment("foo/bar/baz").subFragment(0, 3)); + assertPath("/foo/bar", + new PathFragment("/foo/bar/baz").subFragment(0, 2)); + assertPath("bar/baz", + new PathFragment("/foo/bar/baz").subFragment(1, 3)); + assertPath("/foo", + new PathFragment("/foo/bar/baz").subFragment(0, 1)); + assertPath("bar", + new PathFragment("/foo/bar/baz").subFragment(1, 2)); + assertPath("baz", new PathFragment("/foo/bar/baz").subFragment(2, 3)); + assertPath("/", new PathFragment("/foo/bar/baz").subFragment(0, 0)); + assertPath("", new PathFragment("foo/bar/baz").subFragment(0, 0)); + assertPath("", new PathFragment("foo/bar/baz").subFragment(1, 1)); + try { + fail("unexpectedly succeeded: " + new PathFragment("foo/bar/baz").subFragment(3, 2)); + } catch (IndexOutOfBoundsException e) { /* Expected. */ } + try { + fail("unexpectedly succeeded: " + new PathFragment("foo/bar/baz").subFragment(4, 4)); + } catch (IndexOutOfBoundsException e) { /* Expected. */ } + } + + @Test + public void testStartsWith() { + PathFragment foobar = new PathFragment("/foo/bar"); + PathFragment foobarRelative = new PathFragment("foo/bar"); + + // (path, prefix) => true + assertTrue(foobar.startsWith(foobar)); + assertTrue(foobar.startsWith(new PathFragment("/"))); + assertTrue(foobar.startsWith(new PathFragment("/foo"))); + assertTrue(foobar.startsWith(new PathFragment("/foo/"))); + assertTrue(foobar.startsWith(new PathFragment("/foo/bar/"))); // Includes trailing slash. + + // (prefix, path) => false + assertFalse(new PathFragment("/foo").startsWith(foobar)); + assertFalse(new PathFragment("/").startsWith(foobar)); + + // (absolute, relative) => false + assertFalse(foobar.startsWith(foobarRelative)); + assertFalse(foobarRelative.startsWith(foobar)); + + // (relative path, relative prefix) => true + assertTrue(foobarRelative.startsWith(foobarRelative)); + assertTrue(foobarRelative.startsWith(new PathFragment("foo"))); + assertTrue(foobarRelative.startsWith(new PathFragment(""))); + + // (path, sibling) => false + assertFalse(new PathFragment("/foo/wiz").startsWith(foobar)); + assertFalse(foobar.startsWith(new PathFragment("/foo/wiz"))); + + // Does not normalize. + PathFragment foodotbar = new PathFragment("foo/./bar"); + assertTrue(foodotbar.startsWith(foodotbar)); + assertTrue(foodotbar.startsWith(new PathFragment("foo/."))); + assertTrue(foodotbar.startsWith(new PathFragment("foo/./"))); + assertTrue(foodotbar.startsWith(new PathFragment("foo/./bar"))); + assertFalse(foodotbar.startsWith(new PathFragment("foo/bar"))); + } + + @Test + public void testEndsWith() { + PathFragment foobar = new PathFragment("/foo/bar"); + PathFragment foobarRelative = new PathFragment("foo/bar"); + + // (path, suffix) => true + assertTrue(foobar.endsWith(foobar)); + assertTrue(foobar.endsWith(new PathFragment("bar"))); + assertTrue(foobar.endsWith(new PathFragment("foo/bar"))); + assertTrue(foobar.endsWith(new PathFragment("/foo/bar"))); + assertFalse(foobar.endsWith(new PathFragment("/bar"))); + + // (prefix, path) => false + assertFalse(new PathFragment("/foo").endsWith(foobar)); + assertFalse(new PathFragment("/").endsWith(foobar)); + + // (suffix, path) => false + assertFalse(new PathFragment("/bar").endsWith(foobar)); + assertFalse(new PathFragment("bar").endsWith(foobar)); + assertFalse(new PathFragment("").endsWith(foobar)); + + // (absolute, relative) => true + assertTrue(foobar.endsWith(foobarRelative)); + + // (relative, absolute) => false + assertFalse(foobarRelative.endsWith(foobar)); + + // (relative path, relative prefix) => true + assertTrue(foobarRelative.endsWith(foobarRelative)); + assertTrue(foobarRelative.endsWith(new PathFragment("bar"))); + assertTrue(foobarRelative.endsWith(new PathFragment(""))); + + // (path, sibling) => false + assertFalse(new PathFragment("/foo/wiz").endsWith(foobar)); + assertFalse(foobar.endsWith(new PathFragment("/foo/wiz"))); + } + + static List<PathFragment> toPaths(List<String> strs) { + List<PathFragment> paths = Lists.newArrayList(); + for (String s : strs) { + paths.add(new PathFragment(s)); + } + return paths; + } + + @Test + public void testCompareTo() throws Exception { + List<String> pathStrs = ImmutableList.of( + "", "/", "//", ".", "/./", "foo/.//bar", "foo", "/foo", "foo/bar", "foo/Bar", "Foo/bar"); + List<PathFragment> paths = toPaths(pathStrs); + // First test that compareTo is self-consistent. + for (PathFragment x : paths) { + for (PathFragment y : paths) { + for (PathFragment z : paths) { + // Anti-symmetry + assertEquals(Integer.signum(x.compareTo(y)), + -1 * Integer.signum(y.compareTo(x))); + // Transitivity + if (x.compareTo(y) > 0 && y.compareTo(z) > 0) { + MoreAsserts.assertGreaterThan(0, x.compareTo(z)); + } + // "Substitutability" + if (x.compareTo(y) == 0) { + assertEquals(Integer.signum(x.compareTo(z)), Integer.signum(y.compareTo(z))); + } + // Consistency with equals + assertEquals((x.compareTo(y) == 0), x.equals(y)); + } + } + } + // Now test that compareTo does what we expect. The exact ordering here doesn't matter much, + // but there are three things to notice: 1. absolute < relative, 2. comparison is lexicographic + // 3. repeated slashes are ignored. (PathFragment("//") prints as "/"). + Collections.shuffle(paths); + Collections.sort(paths); + List<PathFragment> expectedOrder = toPaths(ImmutableList.of( + "/", "//", "/./", "/foo", "", ".", "Foo/bar", "foo", "foo/.//bar", "foo/Bar", "foo/bar")); + assertEquals(expectedOrder, paths); + } + + @Test + public void testGetSafePathString() { + assertEquals("/", new PathFragment("/").getSafePathString()); + assertEquals("/abc", new PathFragment("/abc").getSafePathString()); + assertEquals(".", new PathFragment("").getSafePathString()); + assertEquals(".", PathFragment.EMPTY_FRAGMENT.getSafePathString()); + assertEquals("abc/def", new PathFragment("abc/def").getSafePathString()); + } + + @Test + public void testNormalize() { + assertEquals(new PathFragment("/a/b"), new PathFragment("/a/b").normalize()); + assertEquals(new PathFragment("/a/b"), new PathFragment("/a/./b").normalize()); + assertEquals(new PathFragment("/b"), new PathFragment("/a/../b").normalize()); + assertEquals(new PathFragment("a/b"), new PathFragment("a/b").normalize()); + assertEquals(new PathFragment("../b"), new PathFragment("a/../../b").normalize()); + assertEquals(new PathFragment(".."), new PathFragment("a/../..").normalize()); + assertEquals(new PathFragment("b"), new PathFragment("a/../b").normalize()); + assertEquals(new PathFragment("a/b"), new PathFragment("a/b/../b").normalize()); + assertEquals(new PathFragment("/.."), new PathFragment("/..").normalize()); + } + + @Test + public void testSerializationSimple() throws Exception { + checkSerialization("a", 91); + } + + @Test + public void testSerializationAbsolute() throws Exception { + checkSerialization("/foo", 94); + } + + @Test + public void testSerializationNested() throws Exception { + checkSerialization("foo/bar/baz", 101); + } + + private void checkSerialization(String pathFragmentString, int expectedSize) throws Exception { + PathFragment a = new PathFragment(pathFragmentString); + byte[] sa = TestUtils.serializeObject(a); + assertEquals(expectedSize, sa.length); + + PathFragment a2 = (PathFragment) TestUtils.deserializeObject(sa); + assertEquals(a, a2); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentWindowsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentWindowsTest.java new file mode 100644 index 0000000000..43c94d4b8f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/PathFragmentWindowsTest.java @@ -0,0 +1,218 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; + +/** + * This class tests the functionality of the PathFragment. + */ +@RunWith(JUnit4.class) +public class PathFragmentWindowsTest { + + @Test + public void testWindowsSeparator() { + assertEquals("bar/baz", new PathFragment("bar\\baz").toString()); + assertEquals("C:/bar/baz", new PathFragment("c:\\bar\\baz").toString()); + } + + @Test + public void testIsAbsoluteWindows() { + assertTrue(new PathFragment("C:/").isAbsolute()); + assertTrue(new PathFragment("C:/").isAbsolute()); + assertTrue(new PathFragment("C:/foo").isAbsolute()); + assertTrue(new PathFragment("d:/foo/bar").isAbsolute()); + + assertFalse(new PathFragment("*:/").isAbsolute()); + + // C: is not an absolute path, it points to the current active directory on drive C:. + assertFalse(new PathFragment("C:").isAbsolute()); + assertFalse(new PathFragment("C:foo").isAbsolute()); + } + + @Test + public void testIsAbsoluteWindowsBackslash() { + assertTrue(new PathFragment(new File("C:\\blah")).isAbsolute()); + assertTrue(new PathFragment(new File("C:\\")).isAbsolute()); + assertTrue(new PathFragment(new File("\\blah")).isAbsolute()); + assertTrue(new PathFragment(new File("\\")).isAbsolute()); + } + + @Test + public void testIsNormalizedWindows() { + assertTrue(new PathFragment("C:/").isNormalized()); + assertTrue(new PathFragment("C:/absolute/path").isNormalized()); + assertFalse(new PathFragment("C:/absolute/./path").isNormalized()); + assertFalse(new PathFragment("C:/absolute/../path").isNormalized()); + } + + @Test + public void testRootNodeReturnsRootStringWindows() { + PathFragment rootFragment = new PathFragment("C:/"); + assertEquals("C:/", rootFragment.getPathString()); + } + + @Test + public void testGetRelativeWindows() { + assertEquals("C:/a/b", new PathFragment("C:/a").getRelative("b").getPathString()); + assertEquals("C:/a/b/c/d", new PathFragment("C:/a/b").getRelative("c/d").getPathString()); + assertEquals("C:/b", new PathFragment("C:/a").getRelative("C:/b").getPathString()); + assertEquals("C:/c/d", new PathFragment("C:/a/b").getRelative("C:/c/d").getPathString()); + assertEquals("C:/b", new PathFragment("a").getRelative("C:/b").getPathString()); + assertEquals("C:/c/d", new PathFragment("a/b").getRelative("C:/c/d").getPathString()); + } + + @Test + public void testGetRelativeMixed() { + assertEquals("/b", new PathFragment("C:/a").getRelative("/b").getPathString()); + assertEquals("C:/b", new PathFragment("/a").getRelative("C:/b").getPathString()); + } + + @Test + public void testGetChildWorks() { + PathFragment pf = new PathFragment("../some/path"); + assertEquals(new PathFragment("../some/path/hi"), pf.getChild("hi")); + } + + // Tests after here test the canonicalization + private void assertRegular(String expected, String actual) { + assertEquals(expected, new PathFragment(actual).getPathString()); // compare string forms + assertEquals(new PathFragment(expected), new PathFragment(actual)); // compare fragment forms + } + + @Test + public void testEmptyPathToEmptyPathWindows() { + assertRegular("C:/", "C:/"); + } + + @Test + public void testEmptyRelativePathToEmptyPathWindows() { + assertRegular("C:", "C:"); + } + + @Test + public void testWindowsVolumeUppercase() { + assertRegular("C:/", "c:/"); + } + + @Test + public void testRedundantSlashesWindows() { + assertRegular("C:/", "C:///"); + assertRegular("C:/foo/bar", "C:/foo///bar"); + assertRegular("C:/foo/bar", "C:////foo//bar"); + } + + @Test + public void testSimpleNameToSimpleNameWindows() { + assertRegular("C:/foo", "C:/foo"); + } + + @Test + public void testStripsTrailingSlashWindows() { + assertRegular("C:/foo/bar", "C:/foo/bar/"); + } + + @Test + public void testGetParentDirectoryWindows() { + PathFragment fooBarWizAbs = new PathFragment("C:/foo/bar/wiz"); + PathFragment fooBarAbs = new PathFragment("C:/foo/bar"); + PathFragment fooAbs = new PathFragment("C:/foo"); + PathFragment rootAbs = new PathFragment("C:/"); + assertEquals(fooBarAbs, fooBarWizAbs.getParentDirectory()); + assertEquals(fooAbs, fooBarAbs.getParentDirectory()); + assertEquals(rootAbs, fooAbs.getParentDirectory()); + assertNull(rootAbs.getParentDirectory()); + + // Note, this is suprising but correct behaviour: + assertEquals(fooBarAbs, + new PathFragment("C:/foo/bar/..").getParentDirectory()); + } + + @Test + public void testSegmentsCountWindows() { + assertEquals(1, new PathFragment("C:/foo").segmentCount()); + assertEquals(0, new PathFragment("C:/").segmentCount()); + } + + @Test + public void testGetSegmentWindows() { + assertEquals("foo", new PathFragment("C:/foo/bar").getSegment(0)); + assertEquals("bar", new PathFragment("C:/foo/bar").getSegment(1)); + assertEquals("foo", new PathFragment("C:/foo/").getSegment(0)); + assertEquals("foo", new PathFragment("C:/foo").getSegment(0)); + } + + @Test + public void testBasenameWindows() throws Exception { + assertEquals("bar", new PathFragment("C:/foo/bar").getBaseName()); + assertEquals("foo", new PathFragment("C:/foo").getBaseName()); + // Never return the drive name as a basename. + assertEquals("", new PathFragment("C:/").getBaseName()); + } + + private static void assertPath(String expected, PathFragment actual) { + assertEquals(expected, actual.getPathString()); + } + + @Test + public void testReplaceNameWindows() throws Exception { + assertPath("C:/foo/baz", new PathFragment("C:/foo/bar").replaceName("baz")); + assertEquals(null, new PathFragment("C:/").replaceName("baz")); + } + + @Test + public void testStartsWithWindows() { + assertTrue(new PathFragment("C:/foo/bar").startsWith(new PathFragment("C:/foo"))); + assertTrue(new PathFragment("C:/foo/bar").startsWith(new PathFragment("C:/"))); + assertTrue(new PathFragment("C:foo/bar").startsWith(new PathFragment("C:"))); + assertTrue(new PathFragment("C:/").startsWith(new PathFragment("C:/"))); + assertTrue(new PathFragment("C:").startsWith(new PathFragment("C:"))); + + // The first path is absolute, the second is not. + assertFalse(new PathFragment("C:/foo/bar").startsWith(new PathFragment("C:"))); + assertFalse(new PathFragment("C:/").startsWith(new PathFragment("C:"))); + } + + @Test + public void testEndsWithWindows() { + assertTrue(new PathFragment("C:/foo/bar").endsWith(new PathFragment("bar"))); + assertTrue(new PathFragment("C:/foo/bar").endsWith(new PathFragment("foo/bar"))); + assertTrue(new PathFragment("C:/foo/bar").endsWith(new PathFragment("C:/foo/bar"))); + assertTrue(new PathFragment("C:/").endsWith(new PathFragment("C:/"))); + } + + @Test + public void testGetSafePathStringWindows() { + assertEquals("C:/", new PathFragment("C:/").getSafePathString()); + assertEquals("C:/abc", new PathFragment("C:/abc").getSafePathString()); + assertEquals("C:/abc/def", new PathFragment("C:/abc/def").getSafePathString()); + } + + @Test + public void testNormalizeWindows() { + assertEquals(new PathFragment("C:/a/b"), new PathFragment("C:/a/b").normalize()); + assertEquals(new PathFragment("C:/a/b"), new PathFragment("C:/a/./b").normalize()); + assertEquals(new PathFragment("C:/b"), new PathFragment("C:/a/../b").normalize()); + assertEquals(new PathFragment("C:/../b"), new PathFragment("C:/../b").normalize()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathTest.java new file mode 100644 index 0000000000..738e454e43 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/PathTest.java @@ -0,0 +1,312 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Lists; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.GcFinalization; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.ref.WeakReference; +import java.util.Collections; +import java.util.List; + +/** + * A test for {@link Path}. + */ +@RunWith(JUnit4.class) +public class PathTest { + private FileSystem filesystem; + private Path root; + + @Before + public void setUp() throws Exception { + filesystem = new InMemoryFileSystem(BlazeClock.instance()); + root = filesystem.getRootDirectory(); + Path first = root.getChild("first"); + first.createDirectory(); + } + + @Test + public void testStartsWithWorksForSelf() { + assertStartsWithReturns(true, "/first/child", "/first/child"); + } + + @Test + public void testStartsWithWorksForChild() { + assertStartsWithReturns(true, + "/first/child", "/first/child/grandchild"); + } + + @Test + public void testStartsWithWorksForDeepDescendant() { + assertStartsWithReturns(true, + "/first/child", "/first/child/grandchild/x/y/z"); + } + + @Test + public void testStartsWithFailsForParent() { + assertStartsWithReturns(false, "/first/child", "/first"); + } + + @Test + public void testStartsWithFailsForSibling() { + assertStartsWithReturns(false, "/first/child", "/first/child2"); + } + + @Test + public void testStartsWithFailsForLinkToDescendant() + throws Exception { + Path linkTarget = filesystem.getPath("/first/linked_to"); + FileSystemUtils.createEmptyFile(linkTarget); + Path second = filesystem.getPath("/second/"); + second.createDirectory(); + second.getChild("child_link").createSymbolicLink(linkTarget); + assertStartsWithReturns(false, "/first", "/second/child_link"); + } + + @Test + public void testStartsWithFailsForNullPrefix() { + try { + filesystem.getPath("/first").startsWith(null); + fail(); + } catch (Exception e) { + } + } + + private void assertStartsWithReturns(boolean expected, + String ancestor, + String descendant) { + Path parent = filesystem.getPath(ancestor); + Path child = filesystem.getPath(descendant); + assertEquals(expected, child.startsWith(parent)); + } + + @Test + public void testGetChildWorks() { + assertGetChildWorks("second"); + assertGetChildWorks("..."); + assertGetChildWorks("...."); + } + + private void assertGetChildWorks(String childName) { + assertEquals(filesystem.getPath("/first/" + childName), + filesystem.getPath("/first").getChild(childName)); + } + + @Test + public void testGetChildFailsForChildWithSlashes() { + assertGetChildFails("second/third"); + assertGetChildFails("./third"); + assertGetChildFails("../third"); + assertGetChildFails("second/.."); + assertGetChildFails("second/."); + assertGetChildFails("/third"); + assertGetChildFails("third/"); + } + + private void assertGetChildFails(String childName) { + try { + filesystem.getPath("/first").getChild(childName); + fail(); + } catch (IllegalArgumentException e) { + // Expected. + } + } + + @Test + public void testGetChildFailsForDotAndDotDot() { + assertGetChildFails("."); + assertGetChildFails(".."); + } + + @Test + public void testGetChildFailsForEmptyString() { + assertGetChildFails(""); + } + + @Test + public void testRelativeToWorks() { + assertRelativeToWorks("apple", "/fruit/apple", "/fruit"); + assertRelativeToWorks("apple/jonagold", "/fruit/apple/jonagold", "/fruit"); + } + + @Test + public void testGetRelativeWithStringWorks() { + assertGetRelativeWorks("/first/x/y", "y"); + assertGetRelativeWorks("/y", "/y"); + assertGetRelativeWorks("/first/x/x", "./x"); + assertGetRelativeWorks("/first/y", "../y"); + assertGetRelativeWorks("/", "../../../../.."); + } + + @Test + public void testAsFragmentWorks() { + assertAsFragmentWorks("/"); + assertAsFragmentWorks("//"); + assertAsFragmentWorks("/first"); + assertAsFragmentWorks("/first/x/y"); + assertAsFragmentWorks("/first/x/y.foo"); + } + + @Test + public void testGetRelativeWithFragmentWorks() { + Path dir = filesystem.getPath("/first/x"); + assertEquals("/first/x/y", + dir.getRelative(new PathFragment("y")).toString()); + assertEquals("/first/x/x", + dir.getRelative(new PathFragment("./x")).toString()); + assertEquals("/first/y", + dir.getRelative(new PathFragment("../y")).toString()); + + } + + @Test + public void testGetRelativeWithAbsoluteFragmentWorks() { + Path root = filesystem.getPath("/first/x"); + assertEquals("/x/y", + root.getRelative(new PathFragment("/x/y")).toString()); + } + + @Test + public void testGetRelativeWithAbsoluteStringWorks() { + Path root = filesystem.getPath("/first/x"); + assertEquals("/x/y", root.getRelative("/x/y").toString()); + } + + @Test + public void testComparableSortOrder() { + Path zzz = filesystem.getPath("/zzz"); + Path ZZZ = filesystem.getPath("/ZZZ"); + Path abc = filesystem.getPath("/abc"); + Path aBc = filesystem.getPath("/aBc"); + Path AbC = filesystem.getPath("/AbC"); + Path ABC = filesystem.getPath("/ABC"); + List<Path> list = Lists.newArrayList(zzz, ZZZ, ABC, aBc, AbC, abc); + Collections.sort(list); + assertThat(list).containsExactly(ABC, AbC, ZZZ, aBc, abc, zzz).inOrder(); + } + + @Test + public void testParentOfRootIsRoot() { + assertSame(root, root.getRelative("..")); + + assertSame(root.getRelative("dots"), + root.getRelative("broken/../../dots")); + } + + @Test + public void testSingleSegmentEquivalence() { + assertSame( + root.getRelative("aSingleSegment"), + root.getRelative("aSingleSegment")); + } + + @Test + public void testSiblingNonEquivalenceString() { + assertNotSame( + root.getRelative("aSingleSegment"), + root.getRelative("aDifferentSegment")); + } + + @Test + public void testSiblingNonEquivalenceFragment() { + assertNotSame( + root.getRelative(new PathFragment("aSingleSegment")), + root.getRelative(new PathFragment("aDifferentSegment"))); + } + + @Test + public void testHashCodeStableAcrossGarbageCollections() { + Path parent = filesystem.getPath("/a"); + PathFragment childFragment = new PathFragment("b"); + Path child = parent.getRelative(childFragment); + WeakReference<Path> childRef = new WeakReference<>(child); + int childHashCode1 = childRef.get().hashCode(); + assertEquals(childHashCode1, parent.getRelative(childFragment).hashCode()); + child = null; + GcFinalization.awaitClear(childRef); + int childHashCode2 = parent.getRelative(childFragment).hashCode(); + assertEquals(childHashCode1, childHashCode2); + } + + @Test + public void testSerialization() throws Exception { + FileSystem oldFileSystem = Path.getFileSystemForSerialization(); + try { + Path.setFileSystemForSerialization(filesystem); + Path root = filesystem.getPath("/"); + Path p1 = filesystem.getPath("/foo"); + Path p2 = filesystem.getPath("/foo/bar"); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bos); + + oos.writeObject(root); + oos.writeObject(p1); + oos.writeObject(p2); + + ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); + ObjectInputStream ois = new ObjectInputStream(bis); + + Path dsRoot = (Path) ois.readObject(); + Path dsP1 = (Path) ois.readObject(); + Path dsP2 = (Path) ois.readObject(); + + new EqualsTester() + .addEqualityGroup(root, dsRoot) + .addEqualityGroup(p1, dsP1) + .addEqualityGroup(p2, dsP2) + .testEquals(); + + assertTrue(p2.startsWith(p1)); + assertTrue(p2.startsWith(dsP1)); + assertTrue(dsP2.startsWith(p1)); + assertTrue(dsP2.startsWith(dsP1)); + } finally { + Path.setFileSystemForSerialization(oldFileSystem); + } + } + + private void assertAsFragmentWorks(String expected) { + assertEquals(new PathFragment(expected), filesystem.getPath(expected).asFragment()); + } + + private void assertGetRelativeWorks(String expected, String relative) { + assertEquals(filesystem.getPath(expected), + filesystem.getPath("/first/x").getRelative(relative)); + } + + private void assertRelativeToWorks(String expected, String relative, String original) { + assertEquals(new PathFragment(expected), + filesystem.getPath(relative).relativeTo(filesystem.getPath(original))); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/PathWindowsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/PathWindowsTest.java new file mode 100644 index 0000000000..c92fc2b634 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/PathWindowsTest.java @@ -0,0 +1,98 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * A test for windows aspects of {@link Path}. + */ +@RunWith(JUnit4.class) +public class PathWindowsTest { + private FileSystem filesystem; + private Path root; + + @Before + public void setUp() throws Exception { + filesystem = new InMemoryFileSystem(BlazeClock.instance()); + root = filesystem.getRootDirectory(); + Path first = root.getChild("first"); + first.createDirectory(); + } + + private void assertAsFragmentWorks(String expected) { + assertEquals(new PathFragment(expected), filesystem.getPath(expected).asFragment()); + } + + @Test + public void testWindowsPath() { + Path p = filesystem.getPath("C:/foo/bar"); + assertEquals("C:/foo/bar", p.getPathString()); + assertEquals("C:/foo/bar", p.toString()); + } + + @Test + public void testAsFragmentWindows() { + assertAsFragmentWorks("C:/"); + assertAsFragmentWorks("C://"); + assertAsFragmentWorks("C:/first"); + assertAsFragmentWorks("C:/first/x/y"); + assertAsFragmentWorks("C:/first/x/y.foo"); + } + + @Test + public void testGetRelativeWithFragmentWindows() { + Path dir = filesystem.getPath("C:/first/x"); + assertEquals("C:/first/x/y", + dir.getRelative(new PathFragment("y")).toString()); + assertEquals("C:/first/x/x", + dir.getRelative(new PathFragment("./x")).toString()); + assertEquals("C:/first/y", + dir.getRelative(new PathFragment("../y")).toString()); + assertEquals("C:/first/y", + dir.getRelative(new PathFragment("../y")).toString()); + assertEquals("C:/y", + dir.getRelative(new PathFragment("../../../y")).toString()); + } + + @Test + public void testGetRelativeWithAbsoluteFragmentWindows() { + Path root = filesystem.getPath("C:/first/x"); + assertEquals("C:/x/y", + root.getRelative(new PathFragment("C:/x/y")).toString()); + } + + @Test + public void testGetRelativeWithAbsoluteStringWorksWindows() { + Path root = filesystem.getPath("C:/first/x"); + assertEquals("C:/x/y", root.getRelative("C:/x/y").toString()); + } + + @Test + public void testParentOfRootIsRootWindows() { + assertSame(root, root.getRelative("..")); + + assertSame(root.getRelative("dots"), + root.getRelative("broken/../../dots")); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/RecursiveGlobTest.java b/src/test/java/com/google/devtools/build/lib/vfs/RecursiveGlobTest.java new file mode 100644 index 0000000000..5e0012ac08 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/RecursiveGlobTest.java @@ -0,0 +1,227 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.testutil.MoreAsserts.assertSameContents; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Tests {@link UnixGlob} recursive globs. + */ +@RunWith(JUnit4.class) +public class RecursiveGlobTest { + + private Path tmpPath; + private FileSystem fileSystem; + + @Before + public void setUp() throws Exception { + fileSystem = new InMemoryFileSystem(BlazeClock.instance()); + tmpPath = fileSystem.getPath("/rglobtmp"); + for (String dir : ImmutableList.of("foo/bar/wiz", + "foo/baz/wiz", + "foo/baz/quip/wiz", + "food/baz/wiz", + "fool/baz/wiz")) { + FileSystemUtils.createDirectoryAndParents(tmpPath.getRelative(dir)); + } + FileSystemUtils.createEmptyFile(tmpPath.getRelative("foo/bar/wiz/file")); + } + + @Test + public void testDoubleStar() throws Exception { + assertGlobMatches("**", ".", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", "foo/baz/quip", + "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file", "food", "food/baz", + "food/baz/wiz", "fool", "fool/baz", "fool/baz/wiz"); + } + + @Test + public void testDoubleDoubleStar() throws Exception { + assertGlobMatches("**/**", ".", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", "foo/baz/quip", + "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file", "food", "food/baz", + "food/baz/wiz", "fool", "fool/baz", "fool/baz/wiz"); + } + + @Test + public void testDirectoryWithDoubleStar() throws Exception { + assertGlobMatches("foo/**", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", "foo/baz/quip", + "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file"); + } + + @Test + public void testIllegalPatterns() throws Exception { + for (String prefix : Lists.newArrayList("", "*/", "**/", "ba/")) { + String suffix = ("/" + prefix).substring(0, prefix.length()); + for (String pattern : Lists.newArrayList("**fo", "fo**", "**fo**", "fo**fo", "fo**fo**fo")) { + assertIllegalWildcard(prefix + pattern); + assertIllegalWildcard(pattern + suffix); + assertIllegalWildcard("foo", pattern + suffix); + } + } + } + + @Test + public void testDoubleStarPatternWithNamedChild() throws Exception { + assertGlobMatches("**/bar", "foo/bar"); + } + + @Test + public void testDoubleStarPatternWithChildGlob() throws Exception { + assertGlobMatches("**/ba*", + "foo/bar", "foo/baz", "food/baz", "fool/baz"); + } + + @Test + public void testDoubleStarAsChildGlob() throws Exception { + assertGlobMatches("foo/**/wiz", "foo/bar/wiz", "foo/baz/quip/wiz", "foo/baz/wiz"); + } + + @Test + public void testDoubleStarUnderNonexistentDirectory() throws Exception { + assertGlobMatches("not-there/**" /* => nothing */); + } + + @Test + public void testDoubleStarGlobWithNonExistentBase() throws Exception { + Collection<Path> globResult = UnixGlob.forPath(fileSystem.getPath("/does/not/exist")) + .addPattern("**") + .globInterruptible(); + assertEquals(0, globResult.size()); + } + + @Test + public void testDoubleStarUnderFile() throws Exception { + assertGlobMatches("foo/bar/wiz/file/**" /* => nothing */); + } + + @Test + public void testSingleFileExclude() throws Exception { + assertGlobWithExcludeMatches("**", "food", ".", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", + "foo/baz/quip", "foo/baz/quip/wiz", "foo/baz/wiz", + "foo/bar/wiz/file", "food/baz", "food/baz/wiz", "fool", "fool/baz", + "fool/baz/wiz"); + } + + @Test + public void testSingleFileExcludeForDirectoryWithChildGlob() + throws Exception { + assertGlobWithExcludeMatches("foo/**", "foo", "foo/bar", "foo/bar/wiz", "foo/baz", + "foo/baz/quip", "foo/baz/quip/wiz", "foo/baz/wiz", + "foo/bar/wiz/file"); + } + + @Test + public void testGlobExcludeForDirectoryWithChildGlob() + throws Exception { + assertGlobWithExcludeMatches("foo/**", "foo/*", "foo", "foo/bar/wiz", "foo/baz/quip", + "foo/baz/quip/wiz", "foo/baz/wiz", "foo/bar/wiz/file"); + } + + @Test + public void testExcludeAll() throws Exception { + assertGlobWithExcludesMatches(Lists.newArrayList("**"), + Lists.newArrayList("*", "*/*", "*/*/*", "*/*/*/*"), "."); + } + + @Test + public void testManualGlobExcludeForDirectoryWithChildGlob() + throws Exception { + assertGlobWithExcludesMatches(Lists.newArrayList("foo/**"), + Lists.newArrayList("foo", "foo/*", "foo/*/*", "foo/*/*/*")); + } + + private void assertGlobMatches(String pattern, String... expecteds) + throws Exception { + assertGlobWithExcludesMatches( + Collections.singleton(pattern), Collections.<String>emptyList(), + expecteds); + } + + private void assertGlobWithExcludeMatches(String pattern, String exclude, + String... expecteds) + throws Exception { + assertGlobWithExcludesMatches( + Collections.singleton(pattern), Collections.singleton(exclude), + expecteds); + } + + private void assertGlobWithExcludesMatches(Collection<String> pattern, + Collection<String> excludes, + String... expecteds) throws Exception { + assertSameContents(resolvePaths(expecteds), + new UnixGlob.Builder(tmpPath) + .addPatterns(pattern) + .addExcludes(excludes) + .globInterruptible()); + } + + private Set<Path> resolvePaths(String... relativePaths) { + Set<Path> expectedFiles = new HashSet<>(); + for (String expected : relativePaths) { + Path file = expected.equals(".") + ? tmpPath + : tmpPath.getRelative(expected); + expectedFiles.add(file); + } + return expectedFiles; + } + + /** + * Tests that a recursive glob returns files in sorted order. + */ + @Test + public void testGlobEntriesAreSorted() throws Exception { + List<Path> globResult = new UnixGlob.Builder(tmpPath) + .addPattern("**") + .setExcludeDirectories(false) + .globInterruptible(); + + assertThat(Ordering.natural().sortedCopy(globResult)).containsExactlyElementsIn(globResult) + .inOrder(); + } + + private void assertIllegalWildcard(String pattern, String... excludePatterns) + throws Exception { + try { + new UnixGlob.Builder(tmpPath) + .addPattern(pattern) + .addExcludes(excludePatterns) + .globInterruptible(); + fail(); + } catch (IllegalArgumentException e) { + MoreAsserts.assertContainsRegex("recursive wildcard must be its own segment", e.getMessage()); + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/RootedPathTest.java b/src/test/java/com/google/devtools/build/lib/vfs/RootedPathTest.java new file mode 100644 index 0000000000..46d286dcdc --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/RootedPathTest.java @@ -0,0 +1,56 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import com.google.common.testing.EqualsTester; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link RootedPath}. + */ +@RunWith(JUnit4.class) +public class RootedPathTest { + private FileSystem filesystem; + private Path root; + + @Before + public void setUp() throws Exception { + filesystem = new InMemoryFileSystem(BlazeClock.instance()); + root = filesystem.getRootDirectory(); + } + + @Test + public void testEqualsAndHashCodeContract() throws Exception { + Path pkgRoot1 = root.getRelative("pkgroot1"); + Path pkgRoot2 = root.getRelative("pkgroot2"); + RootedPath rootedPathA1 = RootedPath.toRootedPath(pkgRoot1, new PathFragment("foo/bar")); + RootedPath rootedPathA2 = RootedPath.toRootedPath(pkgRoot1, new PathFragment("foo/bar")); + RootedPath absolutePath1 = RootedPath.toRootedPath(root, new PathFragment("pkgroot1/foo/bar")); + RootedPath rootedPathB1 = RootedPath.toRootedPath(pkgRoot2, new PathFragment("foo/bar")); + RootedPath rootedPathB2 = RootedPath.toRootedPath(pkgRoot2, new PathFragment("foo/bar")); + RootedPath absolutePath2 = RootedPath.toRootedPath(root, new PathFragment("pkgroot2/foo/bar")); + new EqualsTester() + .addEqualityGroup(rootedPathA1, rootedPathA2) + .addEqualityGroup(rootedPathB1, rootedPathB2) + .addEqualityGroup(absolutePath1) + .addEqualityGroup(absolutePath2) + .testEquals(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystemTest.java new file mode 100644 index 0000000000..6c8071f075 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/ScopeEscapableFileSystemTest.java @@ -0,0 +1,806 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Collection; + +/** + * Generic tests for any file system that implements {@link ScopeEscapableFileSystem}, + * i.e. any file system that supports symlinks that escape its scope. + * + * Each suitable file system test should inherit from this class, thereby obtaining + * all the tests. + */ +public abstract class ScopeEscapableFileSystemTest extends SymlinkAwareFileSystemTest { + + /** + * Trivial FileSystem implementation that can record the last path passed to each method + * and read/write to a unified "state" variable (which can then be checked by tests) for + * each data type this class manipulates. + * + * The default implementation of each method throws an exception. Each test case should + * selectively override the methods it expects to be invoked. + */ + private static class TestDelegator extends FileSystem { + protected Path lastPath; + protected boolean booleanState; + protected long longState; + protected Object objectState; + + public void setState(boolean state) { booleanState = state; } + public void setState(long state) { longState = state; } + public void setState(Object state) { objectState = state; } + + public boolean booleanState() { return booleanState; } + public long longState() { return longState; } + public Object objectState() { return objectState; } + + public PathFragment lastPath() { + Path ans = lastPath; + // Clear this out to protect against accidental matches when testing the same path multiple + // consecutive times. + lastPath = null; + return ans != null ? ans.asFragment() : null; + } + + @Override public boolean supportsModifications() { return true; } + @Override public boolean supportsSymbolicLinks() { return true; } + + private static RuntimeException re() { + return new RuntimeException("This method should not be called in this context"); + } + + @Override protected boolean isReadable(Path path) { throw re(); } + @Override protected boolean isWritable(Path path) { throw re(); } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { throw re(); } + @Override protected boolean isFile(Path path, boolean followSymlinks) { throw re(); } + @Override protected boolean isExecutable(Path path) { throw re(); } + @Override protected boolean exists(Path path, boolean followSymlinks) {throw re(); } + @Override protected boolean isSymbolicLink(Path path) { throw re(); } + @Override protected boolean createDirectory(Path path) { throw re(); } + @Override protected boolean delete(Path path) { throw re(); } + + @Override protected long getFileSize(Path path, boolean followSymlinks) { throw re(); } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { throw re(); } + + @Override protected void setWritable(Path path, boolean writable) { throw re(); } + @Override protected void setExecutable(Path path, boolean executable) { throw re(); } + @Override protected void setReadable(Path path, boolean readable) { throw re(); } + @Override protected void setLastModifiedTime(Path path, long newTime) { throw re(); } + @Override protected void renameTo(Path sourcePath, Path targetPath) { throw re(); } + @Override protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) { + throw re(); + } + + @Override protected PathFragment readSymbolicLink(Path path) { throw re(); } + @Override protected InputStream getInputStream(Path path) { throw re(); } + @Override protected Collection<Path> getDirectoryEntries(Path path) { throw re(); } + @Override protected OutputStream getOutputStream(Path path, boolean append) { throw re(); } + @Override + protected FileStatus statIfFound(Path path, boolean followSymlinks) throws IOException { + throw re(); + } + } + + protected static final PathFragment SCOPE_ROOT = new PathFragment("/fs/root"); + + private Path fileLink; + private PathFragment fileLinkTarget; + private Path dirLink; + private PathFragment dirLinkTarget; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + Preconditions.checkState(testFS instanceof ScopeEscapableFileSystem, + "Not ScopeEscapable: " + testFS); + ((ScopeEscapableFileSystem) testFS).enableScopeChecking(false); + for (int i = 1; i <= SCOPE_ROOT.segmentCount(); i++) { + testFS.getPath(SCOPE_ROOT.subFragment(0, i)).createDirectory(); + } + + fileLink = testFS.getPath(SCOPE_ROOT.getRelative("link")); + fileLinkTarget = new PathFragment("/should/be/delegated/fileLinkTarget"); + testFS.createSymbolicLink(fileLink, fileLinkTarget); + + dirLink = testFS.getPath(SCOPE_ROOT.getRelative("dirlink")); + dirLinkTarget = new PathFragment("/should/be/delegated/dirLinkTarget"); + testFS.createSymbolicLink(dirLink, dirLinkTarget); + } + + /** + * Returns the file system supplied by {@link #getFreshFileSystem}, cast to + * a {@link ScopeEscapableFileSystem}. Also enables scope checking within + * the file system (which we keep disabled for inherited tests that aren't + * intended to test scope boundaries). + */ + private ScopeEscapableFileSystem scopedFS() { + ScopeEscapableFileSystem fs = (ScopeEscapableFileSystem) testFS; + fs.enableScopeChecking(true); + return fs; + } + + // Checks that the semi-resolved path passed to the delegator matches the expected value. + private void checkPath(TestDelegator delegator, PathFragment expectedDelegatedPath) { + assertTrue(expectedDelegatedPath.equals(delegator.lastPath())); + } + + // Asserts that the condition is false and checks that the expected path was delegated. + private void assertFalseWithPathCheck(boolean result, TestDelegator delegator, + PathFragment expectedDelegatedPath) { + assertFalse(result); + checkPath(delegator, expectedDelegatedPath); + } + + // Asserts that the condition is true and checks that the expected path was delegated. + private void assertTrueWithPathCheck(boolean result, TestDelegator delegator, + PathFragment expectedDelegatedPath) { + assertTrue(result); + checkPath(delegator, expectedDelegatedPath); + } + + ///////////////////////////////////////////////////////////////////////////// + // Tests: + ///////////////////////////////////////////////////////////////////////////// + + @Test + public void testIsReadableCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean isReadable(Path path) { + lastPath = path; + return booleanState(); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(fileLink.isReadable(), delegator, fileLinkTarget); + assertFalseWithPathCheck(dirLink.getRelative("a").isReadable(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(fileLink.isReadable(), delegator, fileLinkTarget); + assertTrueWithPathCheck(dirLink.getRelative("a").isReadable(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testIsWritableCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean isWritable(Path path) { + lastPath = path; + return booleanState(); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(fileLink.isWritable(), delegator, fileLinkTarget); + assertFalseWithPathCheck(dirLink.getRelative("a").isWritable(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(fileLink.isWritable(), delegator, fileLinkTarget); + assertTrueWithPathCheck(dirLink.getRelative("a").isWritable(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testisExecutableCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean isExecutable(Path path) { + lastPath = path; + return booleanState(); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(fileLink.isExecutable(), delegator, fileLinkTarget); + assertFalseWithPathCheck(dirLink.getRelative("a").isExecutable(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(fileLink.isExecutable(), delegator, fileLinkTarget); + assertTrueWithPathCheck(dirLink.getRelative("a").isExecutable(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testIsDirectoryCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { + lastPath = path; + return booleanState(); + } + @Override protected boolean exists(Path path, boolean followSymlinks) { return true; } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(fileLink.isDirectory(), delegator, fileLinkTarget); + assertFalseWithPathCheck(dirLink.getRelative("a").isDirectory(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(fileLink.isDirectory(), delegator, fileLinkTarget); + assertTrueWithPathCheck(dirLink.getRelative("a").isDirectory(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testIsFileCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean isFile(Path path, boolean followSymlinks) { + lastPath = path; + return booleanState(); + } + @Override protected boolean exists(Path path, boolean followSymlinks) { return true; } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(fileLink.isFile(), delegator, fileLinkTarget); + assertFalseWithPathCheck(dirLink.getRelative("a").isFile(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(fileLink.isFile(), delegator, fileLinkTarget); + assertTrueWithPathCheck(dirLink.getRelative("a").isFile(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testIsSymbolicLinkCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean isSymbolicLink(Path path) { + lastPath = path; + return booleanState(); + } + @Override protected boolean exists(Path path, boolean followSymlinks) { return true; } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + // We shouldn't follow final-segment links, so they should never invoke the delegator. + delegator.setState(false); + assertTrue(fileLink.isSymbolicLink()); + assertTrue(delegator.lastPath() == null); + + assertFalseWithPathCheck(dirLink.getRelative("a").isSymbolicLink(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(dirLink.getRelative("a").isSymbolicLink(), delegator, + dirLinkTarget.getRelative("a")); + } + + /** + * Returns a test delegator that reflects info passed to Path.exists() calls. + */ + private TestDelegator newExistsDelegator() { + return new TestDelegator() { + @Override protected boolean exists(Path path, boolean followSymlinks) { + lastPath = path; + return booleanState(); + } + @Override protected FileStatus stat(Path path, boolean followSymlinks) throws IOException { + if (!exists(path, followSymlinks)) { + throw new IOException("Expected exception on stat of non-existent file"); + } + return super.stat(path, followSymlinks); + } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; } + }; + } + + @Test + public void testExistsCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = newExistsDelegator(); + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(fileLink.exists(), delegator, fileLinkTarget); + assertFalseWithPathCheck(dirLink.getRelative("a").exists(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(fileLink.exists(), delegator, fileLinkTarget); + assertTrueWithPathCheck(dirLink.getRelative("a").exists(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testCreateDirectoryCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean createDirectory(Path path) { + lastPath = path; + return booleanState(); + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertFalseWithPathCheck(dirLink.getRelative("a").createDirectory(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(dirLink.getRelative("a").createDirectory(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testDeleteCallOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected boolean delete(Path path) { + lastPath = path; + return booleanState(); + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + assertTrue(fileLink.delete()); + assertTrue(delegator.lastPath() == null); // Deleting a link shouldn't require delegation. + assertFalseWithPathCheck(dirLink.getRelative("a").delete(), delegator, + dirLinkTarget.getRelative("a")); + + delegator.setState(true); + assertTrueWithPathCheck(dirLink.getRelative("a").delete(), delegator, + dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallGetFileSizeOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected long getFileSize(Path path, boolean followSymlinks) { + lastPath = path; + return longState(); + } + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { return 0; } + }; + scopedFS().setDelegator(delegator); + + final int state1 = 10; + delegator.setState(state1); + assertEquals(state1, fileLink.getFileSize()); + checkPath(delegator, fileLinkTarget); + assertEquals(state1, dirLink.getRelative("a").getFileSize()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + + final int state2 = 10; + delegator.setState(state2); + assertEquals(state2, fileLink.getFileSize()); + checkPath(delegator, fileLinkTarget); + assertEquals(state2, dirLink.getRelative("a").getFileSize()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallGetLastModifiedTimeOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected long getLastModifiedTime(Path path, boolean followSymlinks) { + lastPath = path; + return longState(); + } + }; + scopedFS().setDelegator(delegator); + + final int state1 = 10; + delegator.setState(state1); + assertEquals(state1, fileLink.getLastModifiedTime()); + checkPath(delegator, fileLinkTarget); + assertEquals(state1, dirLink.getRelative("a").getLastModifiedTime()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + + final int state2 = 10; + delegator.setState(state2); + assertEquals(state2, fileLink.getLastModifiedTime()); + checkPath(delegator, fileLinkTarget); + assertEquals(state2, dirLink.getRelative("a").getLastModifiedTime()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallSetReadableOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected void setReadable(Path path, boolean readable) { + lastPath = path; + setState(readable); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + fileLink.setReadable(true); + assertTrue(delegator.booleanState()); + checkPath(delegator, fileLinkTarget); + fileLink.setReadable(false); + assertFalse(delegator.booleanState()); + checkPath(delegator, fileLinkTarget); + + delegator.setState(false); + dirLink.getRelative("a").setReadable(true); + assertTrue(delegator.booleanState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + dirLink.getRelative("a").setReadable(false); + assertFalse(delegator.booleanState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallSetWritableOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected void setWritable(Path path, boolean writable) { + lastPath = path; + setState(writable); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + fileLink.setWritable(true); + assertTrue(delegator.booleanState()); + checkPath(delegator, fileLinkTarget); + fileLink.setWritable(false); + assertFalse(delegator.booleanState()); + checkPath(delegator, fileLinkTarget); + + delegator.setState(false); + dirLink.getRelative("a").setWritable(true); + assertTrue(delegator.booleanState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + dirLink.getRelative("a").setWritable(false); + assertFalse(delegator.booleanState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallSetExecutableOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected void setReadable(Path path, boolean readable) { + lastPath = path; + setState(readable); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(false); + fileLink.setReadable(true); + assertTrue(delegator.booleanState()); + checkPath(delegator, fileLinkTarget); + fileLink.setReadable(false); + assertFalse(delegator.booleanState()); + checkPath(delegator, fileLinkTarget); + + delegator.setState(false); + dirLink.getRelative("a").setReadable(true); + assertTrue(delegator.booleanState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + dirLink.getRelative("a").setReadable(false); + assertFalse(delegator.booleanState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallSetLastModifiedTimeOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected void setLastModifiedTime(Path path, long newTime) { + lastPath = path; + setState(newTime); + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(0); + fileLink.setLastModifiedTime(10); + assertEquals(10, delegator.longState()); + checkPath(delegator, fileLinkTarget); + fileLink.setLastModifiedTime(15); + assertEquals(15, delegator.longState()); + checkPath(delegator, fileLinkTarget); + + dirLink.getRelative("a").setLastModifiedTime(20); + assertEquals(20, delegator.longState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + dirLink.getRelative("a").setLastModifiedTime(25); + assertEquals(25, delegator.longState()); + checkPath(delegator, dirLinkTarget.getRelative("a")); + } + + @Test + public void testCallRenameToOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected void renameTo(Path sourcePath, Path targetPath) { + lastPath = sourcePath; + setState(targetPath); + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + // Renaming a link should work fine. + delegator.setState(null); + fileLink.renameTo(testFS.getPath(SCOPE_ROOT).getRelative("newname")); + assertEquals(null, delegator.lastPath()); // Renaming a link shouldn't require delegation. + assertEquals(null, delegator.objectState()); + + // Renaming an out-of-scope path to an in-scope path should fail due to filesystem mismatch + // errors. + Path newPath = testFS.getPath(SCOPE_ROOT.getRelative("blah")); + try { + dirLink.getRelative("a").renameTo(newPath); + fail("This is an attempt at a cross-filesystem renaming, which should fail"); + } catch (IOException e) { + // Expected. + } + + // Renaming an out-of-scope path to another out-of-scope path can be valid. + newPath = dirLink.getRelative("b"); + dirLink.getRelative("a").renameTo(newPath); + assertEquals(dirLinkTarget.getRelative("a"), delegator.lastPath()); + assertEquals(dirLinkTarget.getRelative("b"), ((Path) delegator.objectState()).asFragment()); + } + + @Test + public void testCallCreateSymbolicLinkOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected void createSymbolicLink(Path linkPath, PathFragment targetFragment) { + lastPath = linkPath; + setState(targetFragment); + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + PathFragment newLinkTarget = new PathFragment("/something/else"); + dirLink.getRelative("a").createSymbolicLink(newLinkTarget); + assertEquals(dirLinkTarget.getRelative("a"), delegator.lastPath()); + assertSame(newLinkTarget, delegator.objectState()); + } + + @Test + public void testCallReadSymbolicLinkOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected PathFragment readSymbolicLink(Path path) { + lastPath = path; + return (PathFragment) objectState; + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + // Since we're not following the link, this shouldn't invoke delegation. + delegator.setState(new PathFragment("whatever")); + PathFragment p = fileLink.readSymbolicLink(); + assertEquals(null, delegator.lastPath()); + assertNotSame(delegator.objectState(), p); + + // This should. + p = dirLink.getRelative("a").readSymbolicLink(); + assertEquals(dirLinkTarget.getRelative("a"), delegator.lastPath()); + assertSame(delegator.objectState(), p); + } + + @Test + public void testCallGetInputStreamOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected InputStream getInputStream(Path path) { + lastPath = path; + return (InputStream) objectState; + } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(new ByteArrayInputStream("blah".getBytes())); + InputStream is = fileLink.getInputStream(); + assertEquals(fileLinkTarget, delegator.lastPath()); + assertSame(delegator.objectState(), is); + + delegator.setState(new ByteArrayInputStream("blah2".getBytes())); + is = dirLink.getInputStream(); + assertEquals(dirLinkTarget, delegator.lastPath()); + assertSame(delegator.objectState(), is); + } + + @Test + public void testCallGetOutputStreamOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected OutputStream getOutputStream(Path path, boolean append) { + lastPath = path; + return (OutputStream) objectState; + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(new ByteArrayOutputStream()); + OutputStream os = fileLink.getOutputStream(); + assertEquals(fileLinkTarget, delegator.lastPath()); + assertSame(delegator.objectState(), os); + + delegator.setState(new ByteArrayOutputStream()); + os = dirLink.getOutputStream(); + assertEquals(dirLinkTarget, delegator.lastPath()); + assertSame(delegator.objectState(), os); + } + + @Test + public void testCallGetDirectoryEntriesOnEscapingSymlink() throws Exception { + TestDelegator delegator = new TestDelegator() { + @Override protected Collection<Path> getDirectoryEntries(Path path) { + lastPath = path; + return ImmutableList.of((Path) objectState); + } + @Override protected boolean isDirectory(Path path, boolean followSymlinks) { return true; } + }; + scopedFS().setDelegator(delegator); + + delegator.setState(testFS.getPath("/anything")); + Collection<Path> entries = dirLink.getDirectoryEntries(); + assertEquals(dirLinkTarget, delegator.lastPath()); + assertEquals(1, entries.size()); + assertSame(delegator.objectState(), entries.iterator().next()); + } + + /** + * Asserts that "link" is an in-scope link that doesn't result in an out-of-FS + * delegation. If link is relative, its path is relative to SCOPE_ROOT. + * + * Note that we don't actually check that the canonicalized target path matches + * the link's target value. Such testing should be covered by + * SymlinkAwareFileSystemTest. + */ + private void assertInScopeLink(String link, String target, TestDelegator d) throws IOException { + Path l = testFS.getPath(SCOPE_ROOT.getRelative(link)); + testFS.createSymbolicLink(l, new PathFragment(target)); + l.exists(); + assertNull(d.lastPath()); + } + + /** + * Asserts that "link" is an out-of-scope link and that the re-delegated path + * matches expectedPath. If link is relative, its path is relative to SCOPE_ROOT. + */ + private void assertOutOfScopeLink(String link, String target, String expectedPath, + TestDelegator d) throws IOException { + Path l = testFS.getPath(SCOPE_ROOT.getRelative(link)); + testFS.createSymbolicLink(l, new PathFragment(target)); + l.exists(); + assertEquals(expectedPath, d.lastPath().getPathString()); + } + + /** + * Returns the scope root with the final n segments chopped off (or a 0-segment path + * if n > SCOPE_ROOT.segmentCount()). + */ + private String chopScopeRoot(int n) { + return SCOPE_ROOT + .subFragment(0, n > SCOPE_ROOT.segmentCount() ? 0 : SCOPE_ROOT.segmentCount() - n) + .getPathString(); + } + + /** + * Tests that absolute symlinks with ".." and "." segments are delegated to + * the expected paths. + */ + @Test + public void testAbsoluteSymlinksWithParentReferences() throws Exception { + TestDelegator d = newExistsDelegator(); + scopedFS().setDelegator(d); + testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir"))); + String scopeRoot = SCOPE_ROOT.getPathString(); + String scopeBase = SCOPE_ROOT.getBaseName(); + + // Symlinks that should never escape our scope. + assertInScopeLink("ilink1", scopeRoot, d); + assertInScopeLink("ilink2", scopeRoot + "/target", d); + assertInScopeLink("ilink3", scopeRoot + "/dir/../target", d); + assertInScopeLink("ilink4", scopeRoot + "/dir/../dir/dir2/../target", d); + assertInScopeLink("ilink5", scopeRoot + "/./dir/.././target", d); + assertInScopeLink("ilink6", scopeRoot + "/../" + scopeBase + "/target", d); + assertInScopeLink("ilink7", "/some/path/../.." + scopeRoot + "/target", d); + + // Symlinks that should escape our scope. + assertOutOfScopeLink("olink1", scopeRoot + "/../target", chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("olink2", "/some/other/path", "/some/other/path", d); + assertOutOfScopeLink("olink3", scopeRoot + "/../target", chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("olink4", chopScopeRoot(1) + "/target", chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("olink5", scopeRoot + "/../../../../target", "/target", d); + + // In-scope symlink that's not the final segment in a query. + Path iDirLink = testFS.getPath(SCOPE_ROOT.getRelative("ilinkdir")); + testFS.createSymbolicLink(iDirLink, SCOPE_ROOT.getRelative("dir")); + iDirLink.getRelative("file").exists(); + assertNull(d.lastPath()); + + // Out-of-scope symlink that's not the final segment in a query. + Path oDirLink = testFS.getPath(SCOPE_ROOT.getRelative("olinkdir")); + testFS.createSymbolicLink(oDirLink, new PathFragment("/some/other/dir")); + oDirLink.getRelative("file").exists(); + assertEquals("/some/other/dir/file", d.lastPath().getPathString()); + } + + /** + * Tests that relative symlinks with ".." and "." segments are delegated to + * the expected paths. + */ + @Test + public void testRelativeSymlinksWithParentReferences() throws Exception { + TestDelegator d = newExistsDelegator(); + scopedFS().setDelegator(d); + testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir"))); + testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2"))); + testFS.createDirectory(testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2/dir3"))); + String scopeRoot = SCOPE_ROOT.getPathString(); + String scopeBase = SCOPE_ROOT.getBaseName(); + + // Symlinks that should never escape our scope. + assertInScopeLink("ilink1", "target", d); + assertInScopeLink("ilink2", "dir/../otherdir/target", d); + assertInScopeLink("dir/ilink3", "../target", d); + assertInScopeLink("dir/dir2/ilink4", "../../target", d); + assertInScopeLink("dir/dir2/ilink5", ".././../dir/./target", d); + assertInScopeLink("dir/dir2/ilink6", "../dir2/../../dir/dir2/dir3/../../../target", d); + + // Symlinks that should escape our scope. + assertOutOfScopeLink("olink1", "../target", chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("dir/olink2", "../../target", chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("olink3", "../" + scopeBase + "/target", scopeRoot + "/target", d); + assertOutOfScopeLink("dir/dir2/olink5", "../../../target", chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("dir/dir2/olink6", "../dir2/../../dir/dir2/../../../target", + chopScopeRoot(1) + "/target", d); + assertOutOfScopeLink("dir/olink7", "../../../target", chopScopeRoot(2) + "target", d); + assertOutOfScopeLink("olink8", "../../../../../target", "/target", d); + + // In-scope symlink that's not the final segment in a query. + Path iDirLink = testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2/ilinkdir")); + testFS.createSymbolicLink(iDirLink, new PathFragment("../../dir")); + iDirLink.getRelative("file").exists(); + assertNull(d.lastPath()); + + // Out-of-scope symlink that's not the final segment in a query. + Path oDirLink = testFS.getPath(SCOPE_ROOT.getRelative("dir/dir2/olinkdir")); + testFS.createSymbolicLink(oDirLink, new PathFragment("../../../other/dir")); + oDirLink.getRelative("file").exists(); + assertEquals(chopScopeRoot(1) + "/other/dir/file", d.lastPath().getPathString()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/SymlinkAwareFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/SymlinkAwareFileSystemTest.java new file mode 100644 index 0000000000..a728c88796 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/SymlinkAwareFileSystemTest.java @@ -0,0 +1,717 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.vfs.FileSystem.NotASymlinkException; + +import org.junit.Before; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Collection; + +/** + * This class handles the generic tests that any filesystem must pass. + * + * <p>Each filesystem-test should inherit from this class, thereby obtaining + * all the tests. + */ +public abstract class SymlinkAwareFileSystemTest extends FileSystemTest { + + protected Path xLinkToFile; + protected Path xLinkToLinkToFile; + protected Path xLinkToDirectory; + protected Path xDanglingLink; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + + // % ls -lR + // -rw-rw-r-- xFile + // drwxrwxr-x xNonEmptyDirectory + // -rw-rw-r-- xNonEmptyDirectory/foo + // drwxrwxr-x xEmptyDirectory + // lrwxrwxr-x xLinkToFile -> xFile + // lrwxrwxr-x xLinkToDirectory -> xEmptyDirectory + // lrwxrwxr-x xLinkToLinkToFile -> xLinkToFile + // lrwxrwxr-x xDanglingLink -> xNothing + + xLinkToFile = absolutize("xLinkToFile"); + xLinkToLinkToFile = absolutize("xLinkToLinkToFile"); + xLinkToDirectory = absolutize("xLinkToDirectory"); + xDanglingLink = absolutize("xDanglingLink"); + + createSymbolicLink(xLinkToFile, xFile); + createSymbolicLink(xLinkToLinkToFile, xLinkToFile); + createSymbolicLink(xLinkToDirectory, xEmptyDirectory); + createSymbolicLink(xDanglingLink, xNothing); + } + + @Test + public void testCreateLinkToFile() throws IOException { + Path newPath = xEmptyDirectory.getChild("new-file"); + FileSystemUtils.createEmptyFile(newPath); + + Path linkPath = xEmptyDirectory.getChild("some-link"); + + createSymbolicLink(linkPath, newPath); + + assertTrue(linkPath.isSymbolicLink()); + + assertTrue(linkPath.isFile()); + assertFalse(linkPath.isFile(Symlinks.NOFOLLOW)); + assertTrue(linkPath.isFile(Symlinks.FOLLOW)); + + assertFalse(linkPath.isDirectory()); + assertFalse(linkPath.isDirectory(Symlinks.NOFOLLOW)); + assertFalse(linkPath.isDirectory(Symlinks.FOLLOW)); + + if (supportsSymlinks) { + assertEquals(newPath.toString().length(), linkPath.getFileSize(Symlinks.NOFOLLOW)); + assertEquals(newPath.getFileSize(Symlinks.NOFOLLOW), linkPath.getFileSize()); + } + assertEquals(2, + linkPath.getParentDirectory().getDirectoryEntries().size()); + assertThat(linkPath.getParentDirectory().getDirectoryEntries()).containsExactly(newPath, + linkPath); + } + + @Test + public void testCreateLinkToDirectory() throws IOException { + Path newPath = xEmptyDirectory.getChild("new-file"); + newPath.createDirectory(); + + Path linkPath = xEmptyDirectory.getChild("some-link"); + + createSymbolicLink(linkPath, newPath); + + assertTrue(linkPath.isSymbolicLink()); + assertFalse(linkPath.isFile()); + assertTrue(linkPath.isDirectory()); + assertEquals(2, + linkPath.getParentDirectory().getDirectoryEntries().size()); + assertThat(linkPath.getParentDirectory(). + getDirectoryEntries()).containsExactly(newPath, linkPath); + } + + @Test + public void testFileCanonicalPath() throws IOException { + Path newPath = absolutize("new-file"); + FileSystemUtils.createEmptyFile(newPath); + newPath = newPath.resolveSymbolicLinks(); + + Path link1 = absolutize("some-link"); + Path link2 = absolutize("some-link2"); + + createSymbolicLink(link1, newPath); + createSymbolicLink(link2, link1); + + assertCanonicalPathsMatch(newPath, link1, link2); + } + + @Test + public void testDirectoryCanonicalPath() throws IOException { + Path newPath = absolutize("new-folder"); + newPath.createDirectory(); + newPath = newPath.resolveSymbolicLinks(); + + Path newFile = newPath.getChild("file"); + FileSystemUtils.createEmptyFile(newFile); + + Path link1 = absolutize("some-link"); + Path link2 = absolutize("some-link2"); + + createSymbolicLink(link1, newPath); + createSymbolicLink(link2, link1); + + Path linkFile1 = link1.getChild("file"); + Path linkFile2 = link2.getChild("file"); + + assertCanonicalPathsMatch(newFile, linkFile1, linkFile2); + } + + private void assertCanonicalPathsMatch(Path newPath, Path link1, Path link2) + throws IOException { + assertEquals(newPath, link1.resolveSymbolicLinks()); + assertEquals(newPath, link2.resolveSymbolicLinks()); + } + + // + // createDirectory + // + + @Test + public void testCreateDirectoryWhereDanglingSymlinkAlreadyExists() { + try { + xDanglingLink.createDirectory(); + fail(); + } catch (IOException e) { + assertEquals(xDanglingLink + " (File exists)", e.getMessage()); + } + assertTrue(xDanglingLink.isSymbolicLink()); // still a symbolic link + assertFalse(xDanglingLink.isDirectory(Symlinks.FOLLOW)); // link still dangles + } + + @Test + public void testCreateDirectoryWhereSymlinkAlreadyExists() { + try { + xLinkToDirectory.createDirectory(); + fail(); + } catch (IOException e) { + assertEquals(xLinkToDirectory + " (File exists)", e.getMessage()); + } + assertTrue(xLinkToDirectory.isSymbolicLink()); // still a symbolic link + assertTrue(xLinkToDirectory.isDirectory(Symlinks.FOLLOW)); // link still points to dir + } + + // createSymbolicLink(PathFragment) + + @Test + public void testCreateSymbolicLinkFromFragment() throws IOException { + String[] linkTargets = { + "foo", + "foo/bar", + ".", + "..", + "../foo", + "../../foo", + "../../../../../../../../../../../../../../../../../../../../../foo", + "/foo", + "/foo/bar", + "/..", + "/foo/../bar", + }; + Path linkPath = absolutize("link"); + for (String linkTarget : linkTargets) { + PathFragment relative = new PathFragment(linkTarget); + linkPath.delete(); + createSymbolicLink(linkPath, relative); + if (supportsSymlinks) { + assertEquals(linkTarget.length(), linkPath.getFileSize(Symlinks.NOFOLLOW)); + assertEquals(relative, linkPath.readSymbolicLink()); + } + } + } + + @Test + public void testLinkToRootResolvesCorrectly() throws IOException { + Path rootPath = testFS.getPath("/"); + Path linkPath = absolutize("link"); + createSymbolicLink(linkPath, rootPath); + + // resolveSymbolicLinks requires an existing path: + try { + linkPath.getRelative("test").resolveSymbolicLinks(); + fail(); + } catch (FileNotFoundException e) { /* ok */ } + + // The path may not be a symlink, neither on Darwin nor on Linux. + Path rootChild = testFS.getPath("/sbin"); + if (!rootChild.isDirectory()) { + rootChild.createDirectory(); + } + assertEquals(rootChild, linkPath.getRelative("sbin").resolveSymbolicLinks()); + } + + @Test + public void testLinkToFragmentContainingLinkResolvesCorrectly() throws IOException { + Path link1 = absolutize("link1"); + PathFragment link1target = new PathFragment("link2/foo"); + Path link2 = absolutize("link2"); + Path link2target = xNonEmptyDirectory; + + createSymbolicLink(link1, link1target); // ln -s link2/foo link1 + createSymbolicLink(link2, link2target); // ln -s xNonEmptyDirectory link2 + // link1 --> xNonEmptyDirectory/foo + assertEquals(link1.resolveSymbolicLinks(), link2target.getRelative("foo")); + } + + // + // readSymbolicLink / resolveSymbolicLinks + // + + @Test + public void testRecursiveSymbolicLink() throws IOException { + Path link = absolutize("recursive-link"); + createSymbolicLink(link, link); + + if (supportsSymlinks) { + try { + link.resolveSymbolicLinks(); + fail(); + } catch (IOException e) { + assertEquals(link + " (Too many levels of symbolic links)", + e.getMessage()); + } + } + } + + @Test + public void testMutuallyRecursiveSymbolicLinks() throws IOException { + Path link1 = absolutize("link1"); + Path link2 = absolutize("link2"); + createSymbolicLink(link2, link1); + createSymbolicLink(link1, link2); + + if (supportsSymlinks) { + try { + link1.resolveSymbolicLinks(); + fail(); + } catch (IOException e) { + assertEquals(link1 + " (Too many levels of symbolic links)", e.getMessage()); + } + } + } + + @Test + public void testResolveSymbolicLinksENOENT() { + if (supportsSymlinks) { + try { + xDanglingLink.resolveSymbolicLinks(); + fail(); + } catch (IOException e) { + assertEquals(xNothing + " (No such file or directory)", e.getMessage()); + } + } + } + + @Test + public void testResolveSymbolicLinksENOTDIR() throws IOException { + if (supportsSymlinks) { + Path badLinkTarget = xFile.getChild("bad"); // parent is not a directory! + Path badLink = absolutize("badLink"); + createSymbolicLink(badLink, badLinkTarget); + try { + badLink.resolveSymbolicLinks(); + fail(); + } catch (IOException e) { + // ok. Ideally we would assert "(Not a directory)" in the error + // message, but that would require yet another stat in the + // implementation. + } + } + } + + @Test + public void testResolveSymbolicLinksWithUplevelRefs() throws IOException { + if (supportsSymlinks) { + // Create a series of links that refer to xFile as ./xFile, + // ./../foo/xFile, ./../../bar/foo/xFile, etc. They should all resolve + // to xFile. + Path ancestor = xFile; + String prefix = "./"; + while ((ancestor = ancestor.getParentDirectory()) != null) { + xLinkToFile.delete(); + createSymbolicLink(xLinkToFile, new PathFragment(prefix + xFile.relativeTo(ancestor))); + assertEquals(xFile, xLinkToFile.resolveSymbolicLinks()); + + prefix += "../"; + } + } + } + + @Test + public void testReadSymbolicLink() throws IOException { + if (supportsSymlinks) { + assertEquals(xNothing.toString(), + xDanglingLink.readSymbolicLink().toString()); + } + + assertEquals(xFile.toString(), + xLinkToFile.readSymbolicLink().toString()); + + assertEquals(xEmptyDirectory.toString(), + xLinkToDirectory.readSymbolicLink().toString()); + + try { + xFile.readSymbolicLink(); // not a link + fail(); + } catch (NotASymlinkException e) { + assertEquals(xFile.toString(), e.getMessage()); + } + + try { + xNothing.readSymbolicLink(); // nothing there + fail(); + } catch (IOException e) { + assertEquals(xNothing + " (No such file or directory)", e.getMessage()); + } + } + + @Test + public void testCannotCreateSymbolicLinkWithReadOnlyParent() + throws IOException { + xEmptyDirectory.setWritable(false); + Path xChildOfReadonlyDir = xEmptyDirectory.getChild("x"); + if (supportsSymlinks) { + try { + xChildOfReadonlyDir.createSymbolicLink(xNothing); + fail(); + } catch (IOException e) { + assertEquals(xChildOfReadonlyDir + " (Permission denied)", e.getMessage()); + } + } + } + + // + // createSymbolicLink + // + + @Test + public void testCanCreateDanglingLink() throws IOException { + Path newPath = absolutize("non-existing-dir/new-file"); + Path someLink = absolutize("dangling-link"); + createSymbolicLink(someLink, newPath); + assertTrue(someLink.isSymbolicLink()); + assertTrue(someLink.exists(Symlinks.NOFOLLOW)); // the link itself exists + assertFalse(someLink.exists()); // ...but the referent doesn't + if (supportsSymlinks) { + try { + someLink.resolveSymbolicLinks(); + } catch (FileNotFoundException e) { + assertEquals(newPath.getParentDirectory() + + " (No such file or directory)", e.getMessage()); + } + } + } + + @Test + public void testCannotCreateSymbolicLinkWithoutParent() throws IOException { + Path xChildOfMissingDir = xNothing.getChild("x"); + if (supportsSymlinks) { + try { + xChildOfMissingDir.createSymbolicLink(xFile); + fail(); + } catch (FileNotFoundException e) { + MoreAsserts.assertEndsWith(" (No such file or directory)", e.getMessage()); + } + } + } + + @Test + public void testCreateSymbolicLinkWhereNothingExists() throws IOException { + createSymbolicLink(xNothing, xFile); + assertTrue(xNothing.isSymbolicLink()); + } + + @Test + public void testCreateSymbolicLinkWhereDirectoryAlreadyExists() { + try { + createSymbolicLink(xEmptyDirectory, xFile); + fail(); + } catch (IOException e) { // => couldn't be created + assertEquals(xEmptyDirectory + " (File exists)", e.getMessage()); + } + assertTrue(xEmptyDirectory.isDirectory(Symlinks.NOFOLLOW)); + } + + @Test + public void testCreateSymbolicLinkWhereFileAlreadyExists() { + try { + createSymbolicLink(xFile, xEmptyDirectory); + fail(); + } catch (IOException e) { // => couldn't be created + assertEquals(xFile + " (File exists)", e.getMessage()); + } + assertTrue(xFile.isFile(Symlinks.NOFOLLOW)); + } + + @Test + public void testCreateSymbolicLinkWhereDanglingSymlinkAlreadyExists() { + try { + createSymbolicLink(xDanglingLink, xFile); + fail(); + } catch (IOException e) { + assertEquals(xDanglingLink + " (File exists)", e.getMessage()); + } + assertTrue(xDanglingLink.isSymbolicLink()); // still a symbolic link + assertFalse(xDanglingLink.isDirectory()); // link still dangles + } + + @Test + public void testCreateSymbolicLinkWhereSymlinkAlreadyExists() { + try { + createSymbolicLink(xLinkToDirectory, xNothing); + fail(); + } catch (IOException e) { + assertEquals(xLinkToDirectory + " (File exists)", e.getMessage()); + } + assertTrue(xLinkToDirectory.isSymbolicLink()); // still a symbolic link + assertTrue(xLinkToDirectory.isDirectory()); // link still points to dir + } + + @Test + public void testDeleteLink() throws IOException { + Path newPath = xEmptyDirectory.getChild("new-file"); + Path someLink = xEmptyDirectory.getChild("a-link"); + FileSystemUtils.createEmptyFile(newPath); + createSymbolicLink(someLink, newPath); + + assertEquals(xEmptyDirectory.getDirectoryEntries().size(), 2); + + assertTrue(someLink.delete()); + assertEquals(xEmptyDirectory.getDirectoryEntries().size(), 1); + + assertThat(xEmptyDirectory.getDirectoryEntries()).containsExactly(newPath); + } + + // Testing the links + @Test + public void testLinkFollowedToDirectory() throws IOException { + Path theDirectory = absolutize("foo/"); + assertTrue(theDirectory.createDirectory()); + Path newPath1 = absolutize("foo/new-file-1"); + Path newPath2 = absolutize("foo/new-file-2"); + Path newPath3 = absolutize("foo/new-file-3"); + + FileSystemUtils.createEmptyFile(newPath1); + FileSystemUtils.createEmptyFile(newPath2); + FileSystemUtils.createEmptyFile(newPath3); + + Path linkPath = absolutize("link"); + createSymbolicLink(linkPath, theDirectory); + + Path resultPath1 = absolutize("link/new-file-1"); + Path resultPath2 = absolutize("link/new-file-2"); + Path resultPath3 = absolutize("link/new-file-3"); + assertThat(linkPath.getDirectoryEntries()).containsExactly(resultPath1, resultPath2, + resultPath3); + } + + @Test + public void testDanglingLinkIsNoFile() throws IOException { + Path newPath1 = absolutize("new-file-1"); + Path newPath2 = absolutize("new-file-2"); + FileSystemUtils.createEmptyFile(newPath1); + assertTrue(newPath2.createDirectory()); + + Path linkPath1 = absolutize("link1"); + Path linkPath2 = absolutize("link2"); + createSymbolicLink(linkPath1, newPath1); + createSymbolicLink(linkPath2, newPath2); + + newPath1.delete(); + newPath2.delete(); + + assertFalse(linkPath1.isFile()); + assertFalse(linkPath2.isDirectory()); + } + + @Test + public void testWriteOnLinkChangesFile() throws IOException { + Path testFile = absolutize("test-file"); + FileSystemUtils.createEmptyFile(testFile); + String testData = "abc19"; + + Path testLink = absolutize("a-link"); + createSymbolicLink(testLink, testFile); + + FileSystemUtils.writeContentAsLatin1(testLink, testData); + String resultData = + new String(FileSystemUtils.readContentAsLatin1(testFile)); + + assertEquals(testData,resultData); + } + + // + // Symlink tests: + // + + @Test + public void testExistsWithSymlinks() throws IOException { + Path a = absolutize("a"); + Path b = absolutize("b"); + FileSystemUtils.createEmptyFile(b); + createSymbolicLink(a, b); // ln -sf "b" "a" + assertTrue(a.exists()); // = exists(FOLLOW) + assertTrue(b.exists()); // = exists(FOLLOW) + assertTrue(a.exists(Symlinks.FOLLOW)); + assertTrue(b.exists(Symlinks.FOLLOW)); + assertTrue(a.exists(Symlinks.NOFOLLOW)); + assertTrue(b.exists(Symlinks.NOFOLLOW)); + b.delete(); // "a" is now a dangling link + assertFalse(a.exists()); // = exists(FOLLOW) + assertFalse(b.exists()); // = exists(FOLLOW) + assertFalse(a.exists(Symlinks.FOLLOW)); + assertFalse(b.exists(Symlinks.FOLLOW)); + + assertTrue(a.exists(Symlinks.NOFOLLOW)); // symlink still exists + assertFalse(b.exists(Symlinks.NOFOLLOW)); + } + + @Test + public void testIsDirectoryWithSymlinks() throws IOException { + Path a = absolutize("a"); + Path b = absolutize("b"); + b.createDirectory(); + createSymbolicLink(a, b); // ln -sf "b" "a" + assertTrue(a.isDirectory()); // = isDirectory(FOLLOW) + assertTrue(b.isDirectory()); // = isDirectory(FOLLOW) + assertTrue(a.isDirectory(Symlinks.FOLLOW)); + assertTrue(b.isDirectory(Symlinks.FOLLOW)); + assertFalse(a.isDirectory(Symlinks.NOFOLLOW)); // it's a link! + assertTrue(b.isDirectory(Symlinks.NOFOLLOW)); + b.delete(); // "a" is now a dangling link + assertFalse(a.isDirectory()); // = isDirectory(FOLLOW) + assertFalse(b.isDirectory()); // = isDirectory(FOLLOW) + assertFalse(a.isDirectory(Symlinks.FOLLOW)); + assertFalse(b.isDirectory(Symlinks.FOLLOW)); + assertFalse(a.isDirectory(Symlinks.NOFOLLOW)); + assertFalse(b.isDirectory(Symlinks.NOFOLLOW)); + } + + @Test + public void testIsFileWithSymlinks() throws IOException { + Path a = absolutize("a"); + Path b = absolutize("b"); + FileSystemUtils.createEmptyFile(b); + createSymbolicLink(a, b); // ln -sf "b" "a" + assertTrue(a.isFile()); // = isFile(FOLLOW) + assertTrue(b.isFile()); // = isFile(FOLLOW) + assertTrue(a.isFile(Symlinks.FOLLOW)); + assertTrue(b.isFile(Symlinks.FOLLOW)); + assertFalse(a.isFile(Symlinks.NOFOLLOW)); // it's a link! + assertTrue(b.isFile(Symlinks.NOFOLLOW)); + b.delete(); // "a" is now a dangling link + assertFalse(a.isFile()); // = isFile() + assertFalse(b.isFile()); // = isFile() + assertFalse(a.isFile()); + assertFalse(b.isFile()); + assertFalse(a.isFile(Symlinks.NOFOLLOW)); + assertFalse(b.isFile(Symlinks.NOFOLLOW)); + } + + @Test + public void testGetDirectoryEntriesOnLinkToDirectory() throws Exception { + Path fooAlias = xNothing.getChild("foo"); + createSymbolicLink(xNothing, xNonEmptyDirectory); + Collection<Path> dirents = xNothing.getDirectoryEntries(); + assertThat(dirents).containsExactly(fooAlias); + } + + @Test + public void testFilesOfLinkedDirectories() throws Exception { + Path child = xEmptyDirectory.getChild("child"); + Path aliasToChild = xLinkToDirectory.getChild("child"); + + assertFalse(aliasToChild.exists()); + FileSystemUtils.createEmptyFile(child); + assertTrue(aliasToChild.exists()); + assertTrue(aliasToChild.isFile()); + assertFalse(aliasToChild.isDirectory()); + + validateLinkedReferenceObeysReadOnly(child, aliasToChild); + validateLinkedReferenceObeysExecutable(child, aliasToChild); + } + + @Test + public void testDirectoriesOfLinkedDirectories() throws Exception { + Path childDir = xEmptyDirectory.getChild("childDir"); + Path linkToChildDir = xLinkToDirectory.getChild("childDir"); + + assertFalse(linkToChildDir.exists()); + childDir.createDirectory(); + assertTrue(linkToChildDir.exists()); + assertTrue(linkToChildDir.isDirectory()); + assertFalse(linkToChildDir.isFile()); + + validateLinkedReferenceObeysReadOnly(childDir, linkToChildDir); + validateLinkedReferenceObeysExecutable(childDir, linkToChildDir); + } + + @Test + public void testDirectoriesOfLinkedDirectoriesOfLinkedDirectories() throws Exception { + Path childDir = xEmptyDirectory.getChild("childDir"); + Path linkToLinkToDirectory = absolutize("xLinkToLinkToDirectory"); + createSymbolicLink(linkToLinkToDirectory, xLinkToDirectory); + Path linkToChildDir = linkToLinkToDirectory.getChild("childDir"); + + assertFalse(linkToChildDir.exists()); + childDir.createDirectory(); + assertTrue(linkToChildDir.exists()); + assertTrue(linkToChildDir.isDirectory()); + assertFalse(linkToChildDir.isFile()); + + validateLinkedReferenceObeysReadOnly(childDir, linkToChildDir); + validateLinkedReferenceObeysExecutable(childDir, linkToChildDir); + } + + private void validateLinkedReferenceObeysReadOnly(Path path, Path link) throws IOException { + path.setWritable(false); + assertFalse(path.isWritable()); + assertFalse(link.isWritable()); + path.setWritable(true); + assertTrue(path.isWritable()); + assertTrue(link.isWritable()); + path.setWritable(false); + assertFalse(path.isWritable()); + assertFalse(link.isWritable()); + } + + private void validateLinkedReferenceObeysExecutable(Path path, Path link) throws IOException { + path.setExecutable(true); + assertTrue(path.isExecutable()); + assertTrue(link.isExecutable()); + path.setExecutable(false); + assertFalse(path.isExecutable()); + assertFalse(link.isExecutable()); + path.setExecutable(true); + assertTrue(path.isExecutable()); + assertTrue(link.isExecutable()); + } + + @Test + public void testReadingFileFromLinkedDirectory() throws Exception { + Path linkedTo = absolutize("linkedTo"); + linkedTo.createDirectory(); + Path child = linkedTo.getChild("child"); + FileSystemUtils.createEmptyFile(child); + + byte[] outputData = "This is a test".getBytes(); + FileSystemUtils.writeContent(child, outputData); + + Path link = absolutize("link"); + createSymbolicLink(link, linkedTo); + Path linkedChild = link.getChild("child"); + byte[] inputData = FileSystemUtils.readContent(linkedChild); + assertArrayEquals(outputData, inputData); + } + + @Test + public void testCreatingFileInLinkedDirectory() throws Exception { + Path linkedTo = absolutize("linkedTo"); + linkedTo.createDirectory(); + Path child = linkedTo.getChild("child"); + + Path link = absolutize("link"); + createSymbolicLink(link, linkedTo); + Path linkedChild = link.getChild("child"); + byte[] outputData = "This is a test".getBytes(); + FileSystemUtils.writeContent(linkedChild, outputData); + + byte[] inputData = FileSystemUtils.readContent(child); + assertArrayEquals(outputData, inputData); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnionFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnionFileSystemTest.java new file mode 100644 index 0000000000..396a9f8441 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/UnionFileSystemTest.java @@ -0,0 +1,330 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Tests for the UnionFileSystem, both of generic FileSystem functionality + * (inherited) and tests of UnionFileSystem-specific behavior. + */ +@RunWith(JUnit4.class) +public class UnionFileSystemTest extends SymlinkAwareFileSystemTest { + private XAttrInMemoryFs inDelegate; + private XAttrInMemoryFs outDelegate; + private XAttrInMemoryFs defaultDelegate; + private UnionFileSystem unionfs; + + private static final String XATTR_VAL = "SOME_XATTR_VAL"; + private static final String XATTR_KEY = "SOME_XATTR_KEY"; + + private void setupDelegateFileSystems() { + inDelegate = new XAttrInMemoryFs(BlazeClock.instance()); + outDelegate = new XAttrInMemoryFs(BlazeClock.instance()); + defaultDelegate = new XAttrInMemoryFs(BlazeClock.instance()); + + unionfs = createDefaultUnionFileSystem(); + } + + private UnionFileSystem createDefaultUnionFileSystem() { + return createDefaultUnionFileSystem(false); + } + + private UnionFileSystem createDefaultUnionFileSystem(boolean readOnly) { + return new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of( + new PathFragment("/in"), inDelegate, + new PathFragment("/out"), outDelegate), + defaultDelegate, readOnly); + } + + @Override + protected FileSystem getFreshFileSystem() { + // Executed with each new test because it is called by super.setUp(). + setupDelegateFileSystems(); + return unionfs; + } + + @Override + public void destroyFileSystem(FileSystem fileSystem) { + // Nothing. + } + + // Tests of UnionFileSystem-specific behavior below. + + @Test + public void testBasicDelegation() throws Exception { + unionfs = createDefaultUnionFileSystem(); + Path fooPath = unionfs.getPath("/foo"); + Path inPath = unionfs.getPath("/in"); + Path outPath = unionfs.getPath("/out/in.txt"); + assertSame(inDelegate, unionfs.getDelegate(inPath)); + assertSame(outDelegate, unionfs.getDelegate(outPath)); + assertSame(defaultDelegate, unionfs.getDelegate(fooPath)); + } + + @Test + public void testBasicXattr() throws Exception { + Path fooPath = unionfs.getPath("/foo"); + Path inPath = unionfs.getPath("/in"); + Path outPath = unionfs.getPath("/out/in.txt"); + + assertArrayEquals(XATTR_VAL.getBytes(UTF_8), inPath.getxattr(XATTR_KEY)); + assertArrayEquals(XATTR_VAL.getBytes(UTF_8), outPath.getxattr(XATTR_KEY)); + assertArrayEquals(XATTR_VAL.getBytes(UTF_8), fooPath.getxattr(XATTR_KEY)); + assertNull(inPath.getxattr("not_key")); + assertNull(outPath.getxattr("not_key")); + assertNull(fooPath.getxattr("not_key")); + } + + @Test + public void testDefaultFileSystemRequired() throws Exception { + try { + new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of(), null); + fail("Able to create a UnionFileSystem with no default!"); + } catch (NullPointerException expected) { + // OK - should fail in this case. + } + } + + // Check for appropriate registration and lookup of delegate filesystems based + // on path prefixes, including non-canonical paths. + @Test + public void testPrefixDelegation() throws Exception { + unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of( + new PathFragment("/foo"), inDelegate, + new PathFragment("/foo/bar"), outDelegate), defaultDelegate); + + assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/foo/foo.txt"))); + assertSame(outDelegate, unionfs.getDelegate(unionfs.getPath("/foo/bar/foo.txt"))); + assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/foo/bar/../foo.txt"))); + assertSame(defaultDelegate, unionfs.getDelegate(unionfs.getPath("/bar/foo.txt"))); + assertSame(defaultDelegate, unionfs.getDelegate(unionfs.getPath("/foo/bar/../.."))); + } + + // Checks that files cannot be modified when the filesystem is created + // read-only, even if the delegate filesystems are read/write. + @Test + public void testModificationFlag() throws Exception { + assertTrue(unionfs.supportsModifications()); + Path outPath = unionfs.getPath("/out/foo.txt"); + assertTrue(unionfs.createDirectory(outPath.getParentDirectory())); + OutputStream outFile = unionfs.getOutputStream(outPath); + outFile.write('b'); + outFile.close(); + + unionfs.setExecutable(outPath, true); + + // Note that this does not destroy the underlying filesystems; + // UnionFileSystem is just a view. + unionfs = createDefaultUnionFileSystem(true); + assertFalse(unionfs.supportsModifications()); + + InputStream outFileInput = unionfs.getInputStream(outPath); + int outFileByte = outFileInput.read(); + outFileInput.close(); + assertEquals('b', outFileByte); + + assertTrue(unionfs.isExecutable(outPath)); + + // Modifying files through the unionfs isn't permitted, even if the + // delegates are read/write. + try { + unionfs.setExecutable(outPath, false); + fail("Modification to a read-only UnionFileSystem succeeded."); + } catch (UnsupportedOperationException expected) { + // OK - should fail. + } + } + + // Checks that roots of delegate filesystems are created outside of the + // delegate filesystems; i.e. they can be seen from the filesystem of the parent. + @Test + public void testDelegateRootDirectoryCreation() throws Exception { + Path foo = unionfs.getPath("/foo"); + Path bar = unionfs.getPath("/bar"); + Path out = unionfs.getPath("/out"); + assertTrue(unionfs.createDirectory(foo)); + assertTrue(unionfs.createDirectory(bar)); + assertTrue(unionfs.createDirectory(out)); + Path outFile = unionfs.getPath("/out/in"); + FileSystemUtils.writeContentAsLatin1(outFile, "Out"); + + // FileSystemTest.setUp() silently creates the test root on the filesystem... + Path testDirUnderRoot = unionfs.getPath(workingDir.asFragment().subFragment(0, 1)); + assertThat(unionfs.getDirectoryEntries(unionfs.getRootDirectory())).containsExactly(foo, bar, + out, testDirUnderRoot); + assertThat(unionfs.getDirectoryEntries(out)).containsExactly(outFile); + + assertSame(unionfs.getDelegate(foo), defaultDelegate); + assertEquals(foo.asFragment(), unionfs.adjustPath(foo, defaultDelegate).asFragment()); + assertSame(unionfs.getDelegate(bar), defaultDelegate); + assertSame(unionfs.getDelegate(outFile), outDelegate); + assertSame(unionfs.getDelegate(out), outDelegate); + + // As a fragment (i.e. without filesystem or root info), the path name should be preserved. + assertEquals(outFile.asFragment(), unionfs.adjustPath(outFile, outDelegate).asFragment()); + } + + // Ensure that the right filesystem is still chosen when paths contain "..". + @Test + public void testDelegationOfUpLevelReferences() throws Exception { + assertSame(defaultDelegate, unionfs.getDelegate(unionfs.getPath("/in/../foo.txt"))); + assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/out/../in"))); + assertSame(outDelegate, unionfs.getDelegate(unionfs.getPath("/out/../in/../out/foo.txt"))); + assertSame(inDelegate, unionfs.getDelegate(unionfs.getPath("/in/./foo.txt"))); + } + + // Basic *explicit* cross-filesystem symlink check. + // Note: This does not work implicitly yet, as the next test illustrates. + @Test + public void testCrossDeviceSymlinks() throws Exception { + assertTrue(unionfs.createDirectory(unionfs.getPath("/out"))); + + // Create an "/in" directory directly on the output delegate to bypass the + // UnionFileSystem's mapping. + assertTrue(inDelegate.getPath("/in").createDirectory()); + OutputStream outStream = inDelegate.getPath("/in/bar.txt").getOutputStream(); + outStream.write('i'); + outStream.close(); + + Path outFoo = unionfs.getPath("/out/foo"); + unionfs.createSymbolicLink(outFoo, new PathFragment("../in/bar.txt")); + assertTrue(unionfs.stat(outFoo, false).isSymbolicLink()); + + try { + unionfs.stat(outFoo, true).isFile(); + fail("Stat on cross-device symlink succeeded!"); + } catch (FileNotFoundException expected) { + // OK + } + + Path resolved = unionfs.resolveSymbolicLinks(outFoo); + assertSame(unionfs, resolved.getFileSystem()); + InputStream barInput = resolved.getInputStream(); + int barChar = barInput.read(); + barInput.close(); + assertEquals('i', barChar); + } + + @Test + public void testNoDelegateLeakage() throws Exception { + assertSame(unionfs, unionfs.getPath("/in/foo.txt").getFileSystem()); + assertSame(unionfs, unionfs.getPath("/in/foo/bar").getParentDirectory().getFileSystem()); + unionfs.createDirectory(unionfs.getPath("/out")); + unionfs.createDirectory(unionfs.getPath("/out/foo")); + unionfs.createDirectory(unionfs.getPath("/out/foo/bar")); + assertSame(unionfs, Iterables.getOnlyElement(unionfs.getDirectoryEntries( + unionfs.getPath("/out/foo"))).getParentDirectory().getFileSystem()); + } + + // Prefix mappings can apply to files starting with a prefix within a directory. + @Test + public void testWithinDirectoryMapping() throws Exception { + unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of( + new PathFragment("/fruit/a"), inDelegate, + new PathFragment("/fruit/b"), outDelegate), defaultDelegate); + assertTrue(unionfs.createDirectory(unionfs.getPath("/fruit"))); + assertTrue(defaultDelegate.getPath("/fruit").isDirectory()); + assertTrue(inDelegate.getPath("/fruit").createDirectory()); + assertTrue(outDelegate.getPath("/fruit").createDirectory()); + + Path apple = unionfs.getPath("/fruit/apple"); + Path banana = unionfs.getPath("/fruit/banana"); + Path cherry = unionfs.getPath("/fruit/cherry"); + unionfs.createDirectory(apple); + unionfs.createDirectory(banana); + assertSame(inDelegate, unionfs.getDelegate(apple)); + assertSame(outDelegate, unionfs.getDelegate(banana)); + assertSame(defaultDelegate, unionfs.getDelegate(cherry)); + + FileSystemUtils.writeContentAsLatin1(apple.getRelative("table"), "penny"); + FileSystemUtils.writeContentAsLatin1(banana.getRelative("nana"), "nanana"); + FileSystemUtils.writeContentAsLatin1(cherry, "garcia"); + + assertEquals("penny", new String( + FileSystemUtils.readContentAsLatin1(inDelegate.getPath("/fruit/apple/table")))); + assertEquals("nanana", new String( + FileSystemUtils.readContentAsLatin1(outDelegate.getPath("/fruit/banana/nana")))); + assertEquals("garcia", new String( + FileSystemUtils.readContentAsLatin1(defaultDelegate.getPath("/fruit/cherry")))); + } + + // Write using the VFS through a UnionFileSystem and check that the file can + // be read back in the same location using standard Java IO. + // There is a similar test in UnixFileSystem, but this is essential to ensure + // that paths aren't being remapped in some nasty way on the underlying FS. + @Test + public void testDelegateOperationsReflectOnLocalFilesystem() throws Exception { + unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of( + workingDir.getParentDirectory().asFragment(), new UnixFileSystem()), + defaultDelegate, false); + // This is a child of the current tmpdir, and doesn't exist on its own. + // It would be created in setup(), but of course, that didn't use a UnixFileSystem. + unionfs.createDirectory(workingDir); + Path testFile = unionfs.getPath(workingDir.getRelative("test_file").asFragment()); + assertTrue(testFile.asFragment().startsWith(workingDir.asFragment())); + String testString = "This is a test file"; + FileSystemUtils.writeContentAsLatin1(testFile, testString); + try { + assertEquals(testString, new String(FileSystemUtils.readContentAsLatin1(testFile))); + } finally { + testFile.delete(); + assertTrue(unionfs.delete(workingDir)); + } + } + + // Regression test for [UnionFS: Directory creation across mapping fails.] + @Test + public void testCreateParentsAcrossMapping() throws Exception { + unionfs = new UnionFileSystem(ImmutableMap.<PathFragment, FileSystem>of( + new PathFragment("/out/dir"), outDelegate), defaultDelegate, false); + Path outDir = unionfs.getPath("/out/dir/biz/bang"); + FileSystemUtils.createDirectoryAndParents(outDir); + assertTrue(outDir.isDirectory()); + } + + private static class XAttrInMemoryFs extends InMemoryFileSystem { + public XAttrInMemoryFs(Clock clock) { + super(clock); + } + + @Override + protected byte[] getxattr(Path path, String name, boolean followSymlinks) { + assertSame(this, path.getFileSystem()); + return (name.equals(XATTR_KEY)) ? XATTR_VAL.getBytes(UTF_8) : null; + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixFileSystemTest.java new file mode 100644 index 0000000000..0ded4048a6 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixFileSystemTest.java @@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; + +/** + * Tests for the {@link UnixFileSystem} class. + */ +@RunWith(JUnit4.class) +public class UnixFileSystemTest extends SymlinkAwareFileSystemTest { + + @Override + protected FileSystem getFreshFileSystem() { + return new UnixFileSystem(); + } + + @Override + public void destroyFileSystem(FileSystem fileSystem) { + // Nothing. + } + + @Override + protected void expectNotFound(Path path) throws IOException { + assertNull(path.statIfFound()); + } + + // Most tests are just inherited from FileSystemTest. + + @Test + public void testCircularSymlinkFound() throws Exception { + Path linkA = absolutize("link-a"); + Path linkB = absolutize("link-b"); + linkA.createSymbolicLink(linkB); + linkB.createSymbolicLink(linkA); + assertFalse(linkA.exists(Symlinks.FOLLOW)); + try { + linkA.statIfFound(Symlinks.FOLLOW); + fail(); + } catch (IOException expected) { + // Expected. + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixPathEqualityTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathEqualityTest.java new file mode 100644 index 0000000000..f5f58e2c7e --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathEqualityTest.java @@ -0,0 +1,118 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * This tests how canonical paths and non-canonical paths are equal with each + * other, and also how paths from different filesystems behave with each other. + */ +@RunWith(JUnit4.class) +public class UnixPathEqualityTest { + + private FileSystem otherUnixFs; + private FileSystem unixFs; + + @Before + public void setUp() throws Exception { + unixFs = new UnixFileSystem(); + otherUnixFs = new UnixFileSystem(); + assertTrue(unixFs != otherUnixFs); + } + + private void assertTwoWayEquals(Object obj1, Object obj2) { + assertTrue(obj1.equals(obj2)); + assertTrue(obj2.equals(obj1)); + assertEquals(obj1.hashCode(), obj2.hashCode()); + } + + private void assertTwoWayNotEquals(Object obj1, Object obj2) { + assertFalse(obj1.equals(obj2)); + assertFalse(obj2.equals(obj1)); + } + + @Test + public void testPathsAreEqualEvenIfNotCanonical() { + // This path is already canonical, so there's no difference between + // the canonical / nonCanonical path, as far as equals is concerned + Path nonCanonical = unixFs.getPath("/a/canonical/unix/path"); + Path canonical = unixFs.getPath("/a/canonical/unix/path"); + assertTwoWayEquals(nonCanonical, canonical); + } + + @Test + public void testPathsAreNeverEqualWithStrings() { + // Make sure that paths aren't equal to plain old strings + Path nonCanonical = unixFs.getPath("/a/non/../canonical/unix/path"); + Path canonical = unixFs.getPath("/a/non/../canonical/unix/path"); + assertTwoWayNotEquals(nonCanonical, "/a/non/../canonical/unix/path"); + assertTwoWayNotEquals(canonical, "/a/non/../canonical/unix/path"); + } + + @Test + public void testCanonicalPathsFromDifferentFileSystemsAreNeverEqual() { + Path canonical = unixFs.getPath("/canonical/path"); + Path otherCanonical = otherUnixFs.getPath("/canonical/path"); + assertTwoWayNotEquals(canonical, otherCanonical); + } + + @Test + public void testNonCanonicalPathsFromDifferentFileSystemsAreNeverEqual() { + Path nonCanonical = unixFs.getPath("/non/canonical/path"); + Path otherNonCanonical = otherUnixFs.getPath("/non/canonical/path"); + assertTwoWayNotEquals(nonCanonical, otherNonCanonical); + } + + @Test + public void testCrossFilesystemStartsWithReturnsFalse() { + assertFalse(unixFs.getPath("/a").startsWith(otherUnixFs.getPath("/b"))); + } + + @Test + public void testCrossFilesystemOperationsForbidden() throws Exception { + Path a = unixFs.getPath("/a"); + Path b = otherUnixFs.getPath("/b"); + + try { + a.renameTo(b); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage()).contains("different filesystems"); + } + + try { + a.relativeTo(b); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage()).contains("different filesystems"); + } + + try { + a.createSymbolicLink(b); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage()).contains("different filesystems"); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixPathGetParentTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathGetParentTest.java new file mode 100644 index 0000000000..0f679c3e78 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathGetParentTest.java @@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; + +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.vfs.util.FileSystems; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; + +/** + * A test for {@link Path} in the context of {@link UnixFileSystem}. + */ +@RunWith(JUnit4.class) +public class UnixPathGetParentTest { + + private FileSystem unixFs; + private Path testRoot; + + @Before + public void setUp() throws Exception { + unixFs = FileSystems.initDefaultAsNative(); + testRoot = unixFs.getPath(TestUtils.tmpDir()).getRelative("UnixPathGetParentTest"); + FileSystemUtils.createDirectoryAndParents(testRoot); + } + + @After + public void tearDown() throws Exception { + FileSystemUtils.deleteTree(testRoot); // (comment out during debugging) + } + + private Path getParent(String path) { + return unixFs.getPath(path).getParentDirectory(); + } + + @Test + public void testAbsoluteRootHasNoParent() { + assertEquals(null, getParent("/")); + } + + @Test + public void testParentOfSimpleDirectory() { + assertEquals("/foo", getParent("/foo/bar").getPathString()); + } + + @Test + public void testParentOfDotDotInMiddleOfPathname() { + assertEquals("/", getParent("/foo/../bar").getPathString()); + } + + @Test + public void testGetPathDoesNormalizationWithoutIO() throws IOException { + Path tmp = testRoot.getChild("tmp"); + Path tmpWiz = tmp.getChild("wiz"); + + tmp.createDirectory(); + + // ln -sf /tmp /tmp/wiz + tmpWiz.createSymbolicLink(tmp); + + assertEquals(testRoot, tmp.getParentDirectory()); + + assertEquals(tmp, tmpWiz.getParentDirectory()); + + // Under UNIX, inode(/tmp/wiz/..) == inode(/). However getPath() does not + // perform I/O, only string operations, so it disagrees: + assertEquals(tmp, tmp.getRelative(new PathFragment("wiz/.."))); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/UnixPathTest.java b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathTest.java new file mode 100644 index 0000000000..b59336725a --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/UnixPathTest.java @@ -0,0 +1,279 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.vfs.util.FileSystems; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +/** + * Tests for {@link Path}. + */ +@RunWith(JUnit4.class) +public class UnixPathTest { + + private FileSystem unixFs; + private File aDirectory; + private File aFile; + private File anotherFile; + private File tmpDir; + + protected FileSystem getUnixFileSystem() { + return FileSystems.initDefaultAsNative(); + } + + @Before + public void setUp() throws Exception { + unixFs = getUnixFileSystem(); + tmpDir = new File(TestUtils.tmpDir(), "tmpDir"); + tmpDir.mkdirs(); + aDirectory = new File(tmpDir, "a_directory"); + aDirectory.mkdirs(); + aFile = new File(tmpDir, "a_file"); + new FileWriter(aFile).close(); + anotherFile = new File(aDirectory, "another_file.txt"); + new FileWriter(anotherFile).close(); + } + + @Test + public void testExists() { + assertTrue(unixFs.getPath(aDirectory.getPath()).exists()); + assertTrue(unixFs.getPath(aFile.getPath()).exists()); + assertFalse(unixFs.getPath("/does/not/exist").exists()); + } + + @Test + public void testDirectoryEntriesForDirectory() throws IOException { + Collection<Path> entries = + unixFs.getPath(tmpDir.getPath()).getDirectoryEntries(); + List<Path> expectedEntries = Arrays.asList( + unixFs.getPath(tmpDir.getPath() + "/a_file"), + unixFs.getPath(tmpDir.getPath() + "/a_directory")); + + assertEquals(new HashSet<Object>(expectedEntries), + new HashSet<Object>(entries)); + } + + @Test + public void testDirectoryEntriesForFileThrowsException() { + try { + unixFs.getPath(aFile.getPath()).getDirectoryEntries(); + fail("No exception thrown."); + } catch (IOException x) { + // The expected result. + } + } + + @Test + public void testIsFileIsTrueForFile() { + assertTrue(unixFs.getPath(aFile.getPath()).isFile()); + } + + @Test + public void testIsFileIsFalseForDirectory() { + assertFalse(unixFs.getPath(aDirectory.getPath()).isFile()); + } + + @Test + public void testBaseName() { + assertEquals("base", unixFs.getPath("/foo/base").getBaseName()); + } + + @Test + public void testBaseNameRunsAfterDotDotInterpretation() { + assertEquals("base", unixFs.getPath("/base/foo/..").getBaseName()); + } + + @Test + public void testParentOfRootIsRoot() { + assertEquals(unixFs.getPath("/"), unixFs.getPath("/..")); + assertEquals(unixFs.getPath("/"), unixFs.getPath("/../../../../../..")); + assertEquals(unixFs.getPath("/foo"), unixFs.getPath("/../../../foo")); + } + + @Test + public void testIsDirectory() { + assertTrue(unixFs.getPath(aDirectory.getPath()).isDirectory()); + assertFalse(unixFs.getPath(aFile.getPath()).isDirectory()); + assertFalse(unixFs.getPath("/does/not/exist").isDirectory()); + } + + @Test + public void testListNonExistingDirectoryThrowsException() { + try { + unixFs.getPath("/does/not/exist").getDirectoryEntries(); + fail("No exception thrown."); + } catch (IOException ex) { + // success! + } + } + + private void assertPathSet(Collection<Path> actual, String... expected) { + List<String> actualStrings = Lists.newArrayListWithCapacity(actual.size()); + + for (Path path : actual) { + actualStrings.add(path.getPathString()); + } + + assertThat(actualStrings).containsExactlyElementsIn(Arrays.asList(expected)); + } + + @Test + public void testGlob() throws Exception { + Collection<Path> textFiles = UnixGlob.forPath(unixFs.getPath(tmpDir.getPath())) + .addPattern("*/*.txt") + .globInterruptible(); + assertEquals(1, textFiles.size()); + Path onlyFile = textFiles.iterator().next(); + assertEquals(unixFs.getPath(anotherFile.getPath()), onlyFile); + + Collection<Path> onlyFiles = + UnixGlob.forPath(unixFs.getPath(tmpDir.getPath())) + .addPattern("*") + .setExcludeDirectories(true) + .globInterruptible(); + assertPathSet(onlyFiles, aFile.getPath()); + + Collection<Path> directoriesToo = + UnixGlob.forPath(unixFs.getPath(tmpDir.getPath())) + .addPattern("*") + .setExcludeDirectories(false) + .globInterruptible(); + assertPathSet(directoriesToo, aFile.getPath(), aDirectory.getPath()); + } + + @Test + public void testGetRelative() { + Path relative = unixFs.getPath("/foo").getChild("bar"); + Path expected = unixFs.getPath("/foo/bar"); + assertEquals(expected, relative); + } + + @Test + public void testEqualsAndHash() { + Path path = unixFs.getPath("/foo/bar"); + Path equalPath = unixFs.getPath("/foo/bar"); + Path differentPath = unixFs.getPath("/foo/bar/baz"); + Object differentType = new Object(); + + assertEquals(path.hashCode(), equalPath.hashCode()); + assertEquals(path, equalPath); + assertFalse(path.equals(differentPath)); + assertFalse(path.equals(differentType)); + } + + @Test + public void testLatin1ReadAndWrite() throws IOException { + char[] allLatin1Chars = new char[256]; + for (int i = 0; i < 256; i++) { + allLatin1Chars[i] = (char) i; + } + Path path = unixFs.getPath(aFile.getPath()); + String latin1String = new String(allLatin1Chars); + FileSystemUtils.writeContentAsLatin1(path, latin1String); + String fileContent = new String(FileSystemUtils.readContentAsLatin1(path)); + assertEquals(fileContent, latin1String); + } + + /** + * Verify that the encoding implemented by + * {@link FileSystemUtils#writeContentAsLatin1(Path, String)} + * really is 8859-1 (latin1). + */ + @Test + public void testVerifyLatin1() throws IOException { + char[] allLatin1Chars = new char[256]; + for( int i = 0; i < 256; i++) { + allLatin1Chars[i] = (char)i; + } + Path path = unixFs.getPath(aFile.getPath()); + String latin1String = new String(allLatin1Chars); + FileSystemUtils.writeContentAsLatin1(path, latin1String); + byte[] bytes = FileSystemUtils.readContent(path); + assertEquals(new String(bytes, "ISO-8859-1"), latin1String); + } + + @Test + public void testBytesReadAndWrite() throws IOException { + byte[] bytes = new byte[] { (byte) 0xdeadbeef, (byte) 0xdeadbeef>>8, + (byte) 0xdeadbeef>>16, (byte) 0xdeadbeef>>24 }; + Path path = unixFs.getPath(aFile.getPath()); + FileSystemUtils.writeContent(path, bytes); + byte[] content = FileSystemUtils.readContent(path); + assertEquals(bytes.length, content.length); + for (int i = 0; i < bytes.length; i++) { + assertEquals(bytes[i], content[i]); + } + } + + @Test + public void testInputOutputStreams() throws IOException { + Path path = unixFs.getPath(aFile.getPath()); + OutputStream out = path.getOutputStream(); + for (int i = 0; i < 256; i++) { + out.write(i); + } + out.close(); + InputStream in = path.getInputStream(); + for (int i = 0; i < 256; i++) { + assertEquals(i, in.read()); + } + assertEquals(-1, in.read()); + in.close(); + } + + @Test + public void testAbsolutePathRoot() { + assertEquals("/", new Path(null).toString()); + } + + @Test + public void testAbsolutePath() { + Path segment = new Path(null, "bar.txt", + new Path(null, "foo", new Path(null))); + assertEquals("/foo/bar.txt", segment.toString()); + } + + @Test + public void testDerivedSegmentEquality() { + Path absoluteSegment = new Path(null); + + Path derivedNode = absoluteSegment.getChild("derivedSegment"); + Path otherDerivedNode = absoluteSegment.getChild("derivedSegment"); + + assertSame(derivedNode, otherDerivedNode); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/ZipFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/ZipFileSystemTest.java new file mode 100644 index 0000000000..9dc12764ae --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/ZipFileSystemTest.java @@ -0,0 +1,233 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.Lists; +import com.google.common.io.CharStreams; +import com.google.devtools.build.lib.testutil.BlazeTestUtils; +import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.vfs.util.FileSystems; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@RunWith(JUnit4.class) +public class ZipFileSystemTest { + + /** + * Expected listing of sample zip files, in alpha sorted order + */ + private static final String[] LISTING = { + "/dir1", + "/dir1/file1a", + "/dir1/file1b", + "/dir2", + "/dir2/dir3", + "/dir2/dir3/dir4", + "/dir2/dir3/dir4/file4", + "/dir2/file2", + "/file0", + }; + + private FileSystem zipFS1; + private FileSystem zipFS2; + + @Before + public void setUp() throws Exception { + FileSystem unixFs = FileSystems.initDefaultAsNative(); + Path testdataDir = unixFs.getPath(BlazeTestUtils.runfilesDir()).getRelative( + TestConstants.JAVATESTS_ROOT + "/com/google/devtools/build/lib/vfs"); + Path zPath1 = testdataDir.getChild("sample_with_dirs.zip"); + Path zPath2 = testdataDir.getChild("sample_without_dirs.zip"); + zipFS1 = new ZipFileSystem(zPath1); + zipFS2 = new ZipFileSystem(zPath2); + } + + private void checkExists(FileSystem fs) { + assertTrue(fs.getPath("/dir2/dir3/dir4").exists()); + assertTrue(fs.getPath("/dir2/dir3/dir4/file4").exists()); + assertFalse(fs.getPath("/dir2/dir3/dir4/bogus").exists()); + } + + @Test + public void testExists() { + checkExists(zipFS1); + checkExists(zipFS2); + } + + private void checkIsFile(FileSystem fs) { + assertFalse(fs.getPath("/dir2/dir3/dir4").isFile()); + assertTrue(fs.getPath("/dir2/dir3/dir4/file4").isFile()); + assertFalse(fs.getPath("/dir2/dir3/dir4/bogus").isFile()); + } + + @Test + public void testIsFile() { + checkIsFile(zipFS1); + checkIsFile(zipFS2); + } + + private void checkIsDir(FileSystem fs) { + assertTrue(fs.getPath("/dir2/dir3/dir4").isDirectory()); + assertFalse(fs.getPath("/dir2/dir3/dir4/file4").isDirectory()); + assertFalse(fs.getPath("/bogus/mobogus").isDirectory()); + assertFalse(fs.getPath("/bogus").isDirectory()); + } + + @Test + public void testIsDir() { + checkIsDir(zipFS1); + checkIsDir(zipFS2); + } + + /** + * Recursively add the contents of a given path, rendered as strings, into a + * given list. + */ + private static void listChildren(Path p, List<String> list) + throws IOException { + for (Path c : p.getDirectoryEntries()) { + list.add(c.getPathString()); + if (c.isDirectory()) { + listChildren(c, list); + } + } + } + + private void checkListing(FileSystem fs) throws Exception { + List<String> list = new ArrayList<>(); + listChildren(fs.getRootDirectory(), list); + Collections.sort(list); + assertEquals(Lists.newArrayList(LISTING), list); + } + + @Test + public void testListing() throws Exception { + checkListing(zipFS1); + checkListing(zipFS2); + + // Regression test for: creation of a path (i.e. a file *name*) + // must not affect the result of getDirectoryEntries(). + zipFS1.getPath("/dir1/notthere"); + checkListing(zipFS1); + } + + private void checkFileSize(FileSystem fs, String name, long expectedSize) + throws IOException { + assertEquals(expectedSize, fs.getPath(name).getFileSize()); + } + + @Test + public void testCanReadRoot() { + Path rootDirectory = zipFS1.getRootDirectory(); + assertTrue(rootDirectory.isDirectory()); + } + + @Test + public void testFileSize() throws IOException { + checkFileSize(zipFS1, "/dir1/file1a", 5); + checkFileSize(zipFS2, "/dir1/file1a", 5); + checkFileSize(zipFS1, "/dir2/dir3/dir4/file4", 5000); + checkFileSize(zipFS2, "/dir2/dir3/dir4/file4", 5000); + } + + private void checkCantGetFileSize(FileSystem fs, String name) { + try { + fs.getPath(name).getFileSize(); + fail(); + } catch (IOException expected) { + // expected + } + } + + @Test + public void testCantGetFileSize() { + checkCantGetFileSize(zipFS1, "/dir2/dir3/dir4/bogus"); + checkCantGetFileSize(zipFS2, "/dir2/dir3/dir4/bogus"); + } + + private void checkOpenFile(FileSystem fs, String name, int expectedSize) + throws Exception { + InputStream is = fs.getPath(name).getInputStream(); + List<String> lines = CharStreams.readLines(new InputStreamReader(is, "ISO-8859-1")); + assertEquals(expectedSize, lines.size()); + for (int i = 0; i < expectedSize; i++) { + assertEquals("body", lines.get(i)); + } + } + + @Test + public void testOpenSmallFile() throws Exception { + checkOpenFile(zipFS1, "/dir1/file1a", 1); + checkOpenFile(zipFS2, "/dir1/file1a", 1); + } + + @Test + public void testOpenBigFile() throws Exception { + checkOpenFile(zipFS1, "/dir2/dir3/dir4/file4", 1000); + checkOpenFile(zipFS2, "/dir2/dir3/dir4/file4", 1000); + } + + private void checkCantOpenFile(FileSystem fs, String name) { + try { + fs.getPath(name).getInputStream(); + fail(); + } catch (IOException expected) { + // expected + } + } + + @Test + public void testCantOpenFile() throws Exception { + checkCantOpenFile(zipFS1, "/dir2/dir3/dir4/bogus"); + checkCantOpenFile(zipFS2, "/dir2/dir3/dir4/bogus"); + } + + private void checkCantCreateAnything(FileSystem fs, String name) { + Path p = fs.getPath(name); + try { + p.createDirectory(); + fail(); + } catch (Exception expected) {} + try { + FileSystemUtils.createEmptyFile(p); + fail(); + } catch (Exception expected) {} + try { + p.createSymbolicLink(p); + fail(); + } catch (Exception expected) {} + } + + @Test + public void testCantCreateAnything() throws Exception { + checkCantCreateAnything(zipFS1, "/dir2/dir3/dir4/new"); + checkCantCreateAnything(zipFS2, "/dir2/dir3/dir4/new"); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfoTest.java b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfoTest.java new file mode 100644 index 0000000000..dbdd64a328 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryContentInfoTest.java @@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import static org.junit.Assert.fail; + +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class InMemoryContentInfoTest { + + private Clock clock; + + @Before + public void setUp() throws Exception { + clock = BlazeClock.instance(); + } + + @Test + public void testDirectoryCannotAddNullChild() { + InMemoryDirectoryInfo directory = new InMemoryDirectoryInfo(clock); + + try { + directory.addChild("bar", null); + fail("NullPointerException not thrown."); + } catch (NullPointerException e) { + // success. + } + } + + @Test + public void testDirectoryCannotAddChildTwice() { + InMemoryDirectoryInfo directory = new InMemoryDirectoryInfo(clock); + InMemoryFileInfo otherFile = new InMemoryFileInfo(clock); + directory.addChild("bar", otherFile); + + try { + directory.addChild("bar", otherFile); + fail("IllegalArgumentException not thrown."); + } catch (IllegalArgumentException e) { + // success. + } + } + + @Test + public void testDirectoryRemoveNonExistingChild() { + InMemoryDirectoryInfo directory = new InMemoryDirectoryInfo(clock); + try { + directory.removeChild("bar"); + fail(); + } catch (IllegalArgumentException e) { + // success + } + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystemTest.java b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystemTest.java new file mode 100644 index 0000000000..65ea6f6557 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/inmemoryfs/InMemoryFileSystemTest.java @@ -0,0 +1,414 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.inmemoryfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.ScopeEscapableFileSystemTest; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Tests for {@link InMemoryFileSystem}. Note that most tests are inherited + * from {@link ScopeEscapableFileSystemTest} and ancestors. This specific + * file focuses only on concurrency tests. + * + */ +@RunWith(JUnit4.class) +public class InMemoryFileSystemTest extends ScopeEscapableFileSystemTest { + + @Override + public FileSystem getFreshFileSystem() { + return new InMemoryFileSystem(BlazeClock.instance(), SCOPE_ROOT); + } + + @Override + public void destroyFileSystem(FileSystem fileSystem) { + // Nothing. + } + + private static final int NUM_THREADS_FOR_CONCURRENCY_TESTS = 10; + private static final String TEST_FILE_DATA = "data"; + + /** + * Writes the given data to the given file. + */ + private static void writeToFile(Path path, String data) throws IOException { + OutputStream out = path.getOutputStream(); + out.write(data.getBytes(Charset.defaultCharset())); + out.close(); + } + + /** + * Tests concurrent creation of a substantial tree hierarchy including + * files, directories, symlinks, file contents, and permissions. + */ + @Test + public void testConcurrentTreeConstruction() throws Exception { + final int NUM_TO_WRITE = 10000; + final AtomicInteger baseSelector = new AtomicInteger(); + + // 1) Define the intended path structure. + class PathCreator extends TestThread { + @Override + public void runTest() throws Exception { + Path base = testFS.getPath("/base" + baseSelector.getAndIncrement()); + base.createDirectory(); + + for (int i = 0; i < NUM_TO_WRITE; i++) { + Path subdir1 = base.getRelative("subdir1_" + i); + subdir1.createDirectory(); + Path subdir2 = base.getRelative("subdir2_" + i); + subdir2.createDirectory(); + + Path file = base.getRelative("somefile" + i); + writeToFile(file, TEST_FILE_DATA); + + subdir1.setReadable(true); + subdir2.setReadable(false); + file.setReadable(true); + + subdir1.setWritable(false); + subdir2.setWritable(true); + file.setWritable(false); + + subdir1.setExecutable(false); + subdir2.setExecutable(true); + file.setExecutable(false); + + subdir1.setLastModifiedTime(100); + subdir2.setLastModifiedTime(200); + file.setLastModifiedTime(300); + + Path symlink = base.getRelative("symlink" + i); + symlink.createSymbolicLink(file); + } + } + } + + // 2) Construct the tree. + Collection<TestThread> threads = + Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS); + for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + TestThread thread = new PathCreator(); + thread.start(); + threads.add(thread); + } + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + + // 3) Define the validation logic. + class PathValidator extends TestThread { + @Override + public void runTest() throws Exception { + Path base = testFS.getPath("/base" + baseSelector.getAndIncrement()); + assertTrue(base.exists()); + assertFalse(base.getRelative("notreal").exists()); + + for (int i = 0; i < NUM_TO_WRITE; i++) { + Path subdir1 = base.getRelative("subdir1_" + i); + assertTrue(subdir1.exists()); + assertTrue(subdir1.isDirectory()); + assertTrue(subdir1.isReadable()); + assertFalse(subdir1.isWritable()); + assertFalse(subdir1.isExecutable()); + assertEquals(100, subdir1.getLastModifiedTime()); + + Path subdir2 = base.getRelative("subdir2_" + i); + assertTrue(subdir2.exists()); + assertTrue(subdir2.isDirectory()); + assertFalse(subdir2.isReadable()); + assertTrue(subdir2.isWritable()); + assertTrue(subdir2.isExecutable()); + assertEquals(200, subdir2.getLastModifiedTime()); + + Path file = base.getRelative("somefile" + i); + assertTrue(file.exists()); + assertTrue(file.isFile()); + assertTrue(file.isReadable()); + assertFalse(file.isWritable()); + assertFalse(file.isExecutable()); + assertEquals(300, file.getLastModifiedTime()); + BufferedReader reader = new BufferedReader( + new InputStreamReader(file.getInputStream(), Charset.defaultCharset())); + assertEquals(TEST_FILE_DATA, reader.readLine()); + assertEquals(null, reader.readLine()); + + Path symlink = base.getRelative("symlink" + i); + assertTrue(symlink.exists()); + assertTrue(symlink.isSymbolicLink()); + assertEquals(file.asFragment(), symlink.readSymbolicLink()); + } + } + } + + // 4) Validate the results. + baseSelector.set(0); + threads = Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS); + for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + TestThread thread = new PathValidator(); + thread.start(); + threads.add(thread); + } + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + } + + /** + * Tests concurrent creation of many files, all within the same directory. + */ + @Test + public void testConcurrentDirectoryConstruction() throws Exception { + final int NUM_TO_WRITE = 10000; + final AtomicInteger baseSelector = new AtomicInteger(); + + // 1) Define the intended path structure. + class PathCreator extends TestThread { + @Override + public void runTest() throws Exception { + final int threadId = baseSelector.getAndIncrement(); + Path base = testFS.getPath("/common_dir"); + base.createDirectory(); + + for (int i = 0; i < NUM_TO_WRITE; i++) { + Path file = base.getRelative("somefile_" + threadId + "_" + i); + writeToFile(file, TEST_FILE_DATA); + file.setReadable(i % 2 == 0); + file.setWritable(i % 3 == 0); + file.setExecutable(i % 4 == 0); + file.setLastModifiedTime(i); + Path symlink = base.getRelative("symlink_" + threadId + "_" + i); + symlink.createSymbolicLink(file); + } + } + } + + // 2) Create the files. + Collection<TestThread> threads = + Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS); + for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + TestThread thread = new PathCreator(); + thread.start(); + threads.add(thread); + } + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + + // 3) Define the validation logic. + class PathValidator extends TestThread { + @Override + public void runTest() throws Exception { + final int threadId = baseSelector.getAndIncrement(); + Path base = testFS.getPath("/common_dir"); + assertTrue(base.exists()); + + for (int i = 0; i < NUM_TO_WRITE; i++) { + Path file = base.getRelative("somefile_" + threadId + "_" + i); + assertTrue(file.exists()); + assertTrue(file.isFile()); + assertEquals(i % 2 == 0, file.isReadable()); + assertEquals(i % 3 == 0, file.isWritable()); + assertEquals(i % 4 == 0, file.isExecutable()); + assertEquals(i, file.getLastModifiedTime()); + if (file.isReadable()) { + BufferedReader reader = new BufferedReader( + new InputStreamReader(file.getInputStream(), Charset.defaultCharset())); + assertEquals(TEST_FILE_DATA, reader.readLine()); + assertEquals(null, reader.readLine()); + } + + Path symlink = base.getRelative("symlink_" + threadId + "_" + i); + assertTrue(symlink.exists()); + assertTrue(symlink.isSymbolicLink()); + assertEquals(file.asFragment(), symlink.readSymbolicLink()); + } + } + } + + // 4) Validate the results. + baseSelector.set(0); + threads = Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS); + for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + TestThread thread = new PathValidator(); + thread.start(); + threads.add(thread); + } + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + } + + /** + * Tests concurrent file deletion. + */ + @Test + public void testConcurrentDeletion() throws Exception { + final int NUM_TO_WRITE = 10000; + final AtomicInteger baseSelector = new AtomicInteger(); + + final Path base = testFS.getPath("/base"); + base.createDirectory(); + + // 1) Create a bunch of files. + for (int i = 0; i < NUM_TO_WRITE; i++) { + writeToFile(base.getRelative("file" + i), TEST_FILE_DATA); + } + + // 2) Define our deletion strategy. + class FileDeleter extends TestThread { + @Override + public void runTest() throws Exception { + for (int i = 0; i < NUM_TO_WRITE / NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + int whichFile = baseSelector.getAndIncrement(); + Path file = base.getRelative("file" + whichFile); + if (whichFile % 25 != 0) { + assertTrue(file.delete()); + } else { + // Throw another concurrent access point into the mix. + file.setExecutable(whichFile % 2 == 0); + } + assertFalse(base.getRelative("doesnotexist" + whichFile).delete()); + } + } + } + + // 3) Delete some files. + Collection<TestThread> threads = + Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS); + for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + TestThread thread = new FileDeleter(); + thread.start(); + threads.add(thread); + } + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + + // 4) Check the results. + for (int i = 0; i < NUM_TO_WRITE; i++) { + Path file = base.getRelative("file" + i); + if (i % 25 != 0) { + assertFalse(file.exists()); + } else { + assertTrue(file.exists()); + assertEquals(i % 2 == 0, file.isExecutable()); + } + } + } + + /** + * Tests concurrent file renaming. + */ + @Test + public void testConcurrentRenaming() throws Exception { + final int NUM_TO_WRITE = 10000; + final AtomicInteger baseSelector = new AtomicInteger(); + + final Path base = testFS.getPath("/base"); + base.createDirectory(); + + // 1) Create a bunch of files. + for (int i = 0; i < NUM_TO_WRITE; i++) { + writeToFile(base.getRelative("file" + i), TEST_FILE_DATA); + } + + // 2) Define our renaming strategy. + class FileDeleter extends TestThread { + @Override + public void runTest() throws Exception { + for (int i = 0; i < NUM_TO_WRITE / NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + int whichFile = baseSelector.getAndIncrement(); + Path file = base.getRelative("file" + whichFile); + if (whichFile % 25 != 0) { + Path newName = base.getRelative("newname" + whichFile); + file.renameTo(newName); + } else { + // Throw another concurrent access point into the mix. + file.setExecutable(whichFile % 2 == 0); + } + assertFalse(base.getRelative("doesnotexist" + whichFile).delete()); + } + } + } + + // 3) Rename some files. + Collection<TestThread> threads = + Lists.newArrayListWithCapacity(NUM_THREADS_FOR_CONCURRENCY_TESTS); + for (int i = 0; i < NUM_THREADS_FOR_CONCURRENCY_TESTS; i++) { + TestThread thread = new FileDeleter(); + thread.start(); + threads.add(thread); + } + for (TestThread thread : threads) { + thread.joinAndAssertState(0); + } + + // 4) Check the results. + for (int i = 0; i < NUM_TO_WRITE; i++) { + Path file = base.getRelative("file" + i); + if (i % 25 != 0) { + assertFalse(file.exists()); + assertTrue(base.getRelative("newname" + i).exists()); + } else { + assertTrue(file.exists()); + assertEquals(i % 2 == 0, file.isExecutable()); + } + } + } + + @Test + public void testEloop() throws Exception { + Path a = testFS.getPath("/a"); + Path b = testFS.getPath("/b"); + a.createSymbolicLink(new PathFragment("b")); + b.createSymbolicLink(new PathFragment("a")); + try { + a.stat(); + } catch (IOException e) { + assertEquals("/a (Too many levels of symbolic links)", e.getMessage()); + } + } + + @Test + public void testEloopSelf() throws Exception { + Path a = testFS.getPath("/a"); + a.createSymbolicLink(new PathFragment("a")); + try { + a.stat(); + } catch (IOException e) { + assertEquals("/a (Too many levels of symbolic links)", e.getMessage()); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/sample_with_dirs.zip b/src/test/java/com/google/devtools/build/lib/vfs/sample_with_dirs.zip Binary files differnew file mode 100644 index 0000000000..22ff63cd32 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/sample_with_dirs.zip diff --git a/src/test/java/com/google/devtools/build/lib/vfs/sample_without_dirs.zip b/src/test/java/com/google/devtools/build/lib/vfs/sample_without_dirs.zip Binary files differnew file mode 100644 index 0000000000..f3ec5ab792 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/sample_without_dirs.zip diff --git a/src/test/java/com/google/devtools/build/lib/vfs/util/FileSystems.java b/src/test/java/com/google/devtools/build/lib/vfs/util/FileSystems.java new file mode 100644 index 0000000000..6a79a2b948 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/util/FileSystems.java @@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.util; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.UnionFileSystem; +import com.google.devtools.build.lib.vfs.UnixFileSystem; +import com.google.devtools.build.lib.vfs.ZipFileSystem; + +import java.io.IOException; +import java.util.Map; + +/** + * This static file system singleton manages access to a single default + * {@link FileSystem} instance created within the methods of this class. + */ +@ThreadSafe +public final class FileSystems { + + private FileSystems() {} + + private static FileSystem defaultFileSystem; + + /** + * Initializes the default {@link FileSystem} instance as a platform native + * (Unix) file system, creating one iff needed, and returns the instance. + * + * <p>This method is idempotent as long as the initialization is of the same + * type (Native/JavaIo/Union). + */ + public static synchronized FileSystem initDefaultAsNative() { + if (!(defaultFileSystem instanceof UnixFileSystem)) { + defaultFileSystem = new UnixFileSystem(); + } + return defaultFileSystem; + } + + /** + * Initializes the default {@link FileSystem} instance as a java.io.File + * file system, creating one iff needed, and returns the instance. + * + * <p>This method is idempotent as long as the initialization is of the same + * type (Native/JavaIo/Union). + */ + public static synchronized FileSystem initDefaultAsJavaIo() { + if (!(defaultFileSystem instanceof JavaIoFileSystem)) { + defaultFileSystem = new JavaIoFileSystem(); + } + return defaultFileSystem; + } + + /** + * Initializes the default {@link FileSystem} instance as a + * {@link UnionFileSystem}, creating one iff needed, + * and returns the instance. + * + * <p>This method is idempotent as long as the initialization is of the same + * type (Native/JavaIo/Union). + * + * @param prefixMapping the desired mapping of path prefixes to delegate file systems + * @param rootFileSystem the default file system for paths that don't match any prefix map + */ + public static synchronized FileSystem initDefaultAsUnion( + Map<PathFragment, FileSystem> prefixMapping, FileSystem rootFileSystem) { + if (!(defaultFileSystem instanceof UnionFileSystem)) { + defaultFileSystem = new UnionFileSystem(prefixMapping, rootFileSystem); + } + return defaultFileSystem; + } + + /** + * Returns a new instance of a simple {@link FileSystem} implementation that + * presents the contents of a zip file as a read-only file system view. + */ + public static FileSystem newZipFileSystem(Path zipFile) throws IOException { + return new ZipFileSystem(zipFile); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/util/FsApparatus.java b/src/test/java/com/google/devtools/build/lib/vfs/util/FsApparatus.java new file mode 100644 index 0000000000..5d93351a8d --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/util/FsApparatus.java @@ -0,0 +1,158 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.util; + +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; + +import junit.framework.AssertionFailedError; + +import java.io.File; +import java.io.IOException; + +/** + * Base class for a testing apparatus for a scratch filesystem. + */ +public class FsApparatus { + + /* ---------- State that the apparatus initializes / operates on --------- */ + protected FileSystem fileSystem = null; + protected Path workingDir = null; + + public static FsApparatus newInMemory() { + return new FsApparatus(); + } + + // TestUtil.getTmpDir is slow, so cache the result here + private static final String TMP_DIR = + new File(TestUtils.tmpDir(), "bs").toString(); + + + /** + * When using a Native file system, absolute paths will be treated as absolute paths on the unix + * file system, as opposed to paths relative to the backing temp directory. So for simplicity, + * you ought to only use relative paths for FsApparatus#file, FsApparatus#dir, and + * FsApparatus#path. Otherwise, be aware of the following issue + * + * Path p1 = scratch.path(...); + * Path p2 = scratch.path(p1.getPathString()); + * + * We'd like the invariant that p1.equals(p2) regardless if scratch is in-memory or not, but this + * does not hold with our usage of Unix filesystems. + */ + public static FsApparatus newNative() { + FileSystem fs = FileSystems.initDefaultAsNative(); + Path wd = fs.getPath(TMP_DIR); + + try { + FileSystemUtils.deleteTree(wd); + } catch (IOException e) { + throw new AssertionFailedError(e.getMessage()); + } + + return new FsApparatus(fs, wd); + } + + private FsApparatus() { + fileSystem = new InMemoryFileSystem(BlazeClock.instance()); + workingDir = fileSystem.getPath("/"); + } + + public FsApparatus(FileSystem fs, Path cwd) { + fileSystem = fs; + workingDir = cwd; + } + + public FsApparatus(FileSystem fs) { + fileSystem = fs; + workingDir = fs.getPath("/"); + } + + public FileSystem fs() { + return fileSystem; + } + + /** + * Initializes this apparatus (if it hasn't been initialized yet), and creates + * a scratch file in the scratch filesystem with the given {@code pathName} + * with {@code lines} being its content. The method returns a Path instance + * for the scratch file. + */ + public Path file(String pathName, String... lines) throws IOException { + Path file = path(pathName); + Path parentDir = file.getParentDirectory(); + if (!parentDir.exists()) { + FileSystemUtils.createDirectoryAndParents(parentDir); + } + if (file.exists()) { + throw new IOException("Could not create scratch file (file exists) " + + file); + } + String fileContent = StringUtilities.joinLines(lines); + FileSystemUtils.writeContentAsLatin1(file, fileContent); + return file; + } + + /** + * Initializes this apparatus (if it hasn't been initialized yet), and creates + * a directory in the scratch filesystem, with the given {@code pathName}. + * Creates parent directories as necessary. + */ + public Path dir(String pathName) throws IOException { + Path dir = path(pathName); + if (!dir.exists()) { + FileSystemUtils.createDirectoryAndParents(dir); + } + if (!dir.isDirectory()) { + throw new IOException("Exists, but is not a directory: " + dir); + } + return dir; + } + + /** + * Initializes this apparatus (if it hasn't been initialized yet), and returns + * a path object describing a file, directory, or symlink pointed at by + * {@code pathName}. Note that this will not create any entity in the + * filesystem; i.e., the file that the object is describing may not exist in + * the filesystem. + */ + public Path path(String pathName) { + return workingDir.getRelative(pathName); + } + + /** + * Create a fresh directory in the system temporary directory, instead of the + * testing directory provided by the testing framework. This path is usually + * shorter than a path starting with TestUtil.getTmpDir(). We care about the + * length because of the path length restriction for Unix local socket files. + * + * Clients are responsible for deleting the directory after tests. + */ + public Path createUnixTempDir() throws IOException { + if (fileSystem instanceof InMemoryFileSystem) { + throw new IOException("Can not create Unix temporary directories in " + + "an in-memory file system"); + } + File file = File.createTempFile("scratch", "tmp"); + final Path path = fileSystem.getPath(file.getAbsolutePath()); + path.delete(); + path.createDirectory(); + return path; + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/AllTests.java b/src/test/java/com/google/devtools/build/skyframe/AllTests.java new file mode 100644 index 0000000000..4ea36910f6 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/AllTests.java @@ -0,0 +1,25 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.testutil.ClasspathSuite; + +import org.junit.runner.RunWith; + +/** + * Automatically collect the tests annotated with @RunWith in this package and all subpackages. + */ +@RunWith(ClasspathSuite.class) +public class AllTests { +} diff --git a/src/test/java/com/google/devtools/build/skyframe/ChainedFunction.java b/src/test/java/com/google/devtools/build/skyframe/ChainedFunction.java new file mode 100644 index 0000000000..4f87bb38ca --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/ChainedFunction.java @@ -0,0 +1,87 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.skyframe.GraphTester.ValueComputer; +import com.google.devtools.build.skyframe.ParallelEvaluator.SkyFunctionEnvironment; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; + +import java.util.concurrent.CountDownLatch; + +import javax.annotation.Nullable; + +/** + * {@link ValueComputer} that can be chained together with others of its type to synchronize the + * order in which builders finish. + */ +final class ChainedFunction implements SkyFunction { + @Nullable private final SkyValue value; + @Nullable private final CountDownLatch notifyStart; + @Nullable private final CountDownLatch waitToFinish; + @Nullable private final CountDownLatch notifyFinish; + private final boolean waitForException; + private final Iterable<SkyKey> deps; + + ChainedFunction(@Nullable CountDownLatch notifyStart, @Nullable CountDownLatch waitToFinish, + @Nullable CountDownLatch notifyFinish, boolean waitForException, + @Nullable SkyValue value, Iterable<SkyKey> deps) { + this.notifyStart = notifyStart; + this.waitToFinish = waitToFinish; + this.notifyFinish = notifyFinish; + this.waitForException = waitForException; + Preconditions.checkState(this.waitToFinish != null || !this.waitForException, value); + this.value = value; + this.deps = deps; + } + + @Override + public SkyValue compute(SkyKey key, SkyFunction.Environment env) throws GenericFunctionException, + InterruptedException { + try { + if (notifyStart != null) { + notifyStart.countDown(); + } + if (waitToFinish != null) { + TrackingAwaiter.waitAndMaybeThrowInterrupt(waitToFinish, + key + " timed out waiting to finish"); + if (waitForException) { + SkyFunctionEnvironment skyEnv = (SkyFunctionEnvironment) env; + TrackingAwaiter.waitAndMaybeThrowInterrupt(skyEnv.getExceptionLatchForTesting(), + key + " timed out waiting for exception"); + } + } + for (SkyKey dep : deps) { + env.getValue(dep); + } + if (value == null) { + throw new GenericFunctionException(new SomeErrorException("oops"), + Transience.PERSISTENT); + } + if (env.valuesMissing()) { + return null; + } + return value; + } finally { + if (notifyFinish != null) { + notifyFinish.countDown(); + } + } + } + + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/CycleDeduperTest.java b/src/test/java/com/google/devtools/build/skyframe/CycleDeduperTest.java new file mode 100644 index 0000000000..592d95fa8f --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/CycleDeduperTest.java @@ -0,0 +1,64 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Simple tests for {@link CycleDeduper}. */ +@RunWith(JUnit4.class) +public class CycleDeduperTest { + + private CycleDeduper<String> cycleDeduper = new CycleDeduper<>(); + + @Test + public void simple() throws Exception { + assertTrue(cycleDeduper.seen(ImmutableList.of("a", "b"))); + assertFalse(cycleDeduper.seen(ImmutableList.of("a", "b"))); + assertFalse(cycleDeduper.seen(ImmutableList.of("b", "a"))); + + assertTrue(cycleDeduper.seen(ImmutableList.of("a", "b", "c"))); + assertFalse(cycleDeduper.seen(ImmutableList.of("b", "c", "a"))); + assertFalse(cycleDeduper.seen(ImmutableList.of("c", "a", "b"))); + assertTrue(cycleDeduper.seen(ImmutableList.of("b", "a", "c"))); + assertFalse(cycleDeduper.seen(ImmutableList.of("c", "b", "a"))); + } + + @Test + public void badCycle_Empty() throws Exception { + try { + cycleDeduper.seen(ImmutableList.<String>of()); + fail(); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void badCycle_NonUniqueMembers() throws Exception { + try { + cycleDeduper.seen(ImmutableList.<String>of("a", "b", "a")); + fail(); + } catch (IllegalStateException e) { + // Expected. + } + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/CyclesReporterTest.java b/src/test/java/com/google/devtools/build/skyframe/CyclesReporterTest.java new file mode 100644 index 0000000000..35bda02f15 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/CyclesReporterTest.java @@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.NullEventHandler; +import com.google.devtools.build.skyframe.CyclesReporter.SingleCycleReporter; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.concurrent.atomic.AtomicBoolean; + +@RunWith(JUnit4.class) +public class CyclesReporterTest { + + private static final SkyKey DUMMY_KEY = new SkyKey(SkyFunctionName.computed("func"), "key"); + + @Test + public void nullEventHandler() { + CyclesReporter cyclesReporter = new CyclesReporter(); + try { + cyclesReporter.reportCycles(ImmutableList.<CycleInfo>of(), DUMMY_KEY, null); + assertThat(false).isTrue(); + } catch (NullPointerException e) { + // Expected. + } + } + + @Test + public void notReportedAssertion() { + SingleCycleReporter singleReporter = new SingleCycleReporter() { + @Override + public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, + boolean alreadyReported, EventHandler eventHandler) { + return false; + } + }; + + CycleInfo cycleInfo = new CycleInfo(ImmutableList.of(DUMMY_KEY)); + CyclesReporter cyclesReporter = new CyclesReporter(singleReporter); + try { + cyclesReporter.reportCycles(ImmutableList.of(cycleInfo), DUMMY_KEY, + NullEventHandler.INSTANCE); + assertThat(false).isTrue(); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void smoke() { + final AtomicBoolean reported = new AtomicBoolean(); + SingleCycleReporter singleReporter = new SingleCycleReporter() { + @Override + public boolean maybeReportCycle(SkyKey topLevelKey, CycleInfo cycleInfo, + boolean alreadyReported, EventHandler eventHandler) { + reported.set(true); + return true; + } + }; + + CycleInfo cycleInfo = new CycleInfo(ImmutableList.of(DUMMY_KEY)); + CyclesReporter cyclesReporter = new CyclesReporter(singleReporter); + cyclesReporter.reportCycles(ImmutableList.of(cycleInfo), DUMMY_KEY, + NullEventHandler.INSTANCE); + assertThat(reported.get()).isTrue(); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/DeterministicInMemoryGraph.java b/src/test/java/com/google/devtools/build/skyframe/DeterministicInMemoryGraph.java new file mode 100644 index 0000000000..8ea81d0002 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/DeterministicInMemoryGraph.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +/** {@link NotifyingInMemoryGraph} that returns reverse deps ordered alphabetically. */ +public class DeterministicInMemoryGraph extends NotifyingInMemoryGraph { + public DeterministicInMemoryGraph(Listener listener) { + super(listener); + } + + public DeterministicInMemoryGraph() { + super(Listener.NULL_LISTENER); + } + + @Override + protected DeterministicValueEntry getEntry(SkyKey key) { + return new DeterministicValueEntry(key); + } + + /** + * This class uses TreeSet to store reverse dependencies of NodeEntry. As a result all values are + * lexicographically sorted. + */ + private class DeterministicValueEntry extends NotifyingNodeEntry { + private DeterministicValueEntry(SkyKey myKey) { + super(myKey); + } + + final Comparator<SkyKey> valueEntryComparator = new Comparator<SkyKey>() { + @Override + public int compare(SkyKey o1, SkyKey o2) { + return o1.toString().compareTo(o2.toString()); + } + }; + @SuppressWarnings("unchecked") + @Override + synchronized Iterable<SkyKey> getReverseDeps() { + TreeSet<SkyKey> result = new TreeSet<SkyKey>(valueEntryComparator); + if (reverseDeps instanceof List) { + result.addAll((Collection<? extends SkyKey>) reverseDeps); + } else { + result.add((SkyKey) reverseDeps); + } + return result; + } + + @Override + synchronized Set<SkyKey> getInProgressReverseDeps() { + TreeSet<SkyKey> result = new TreeSet<SkyKey>(valueEntryComparator); + result.addAll(buildingState.getReverseDepsToSignal()); + return result; + } + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/EagerInvalidatorTest.java b/src/test/java/com/google/devtools/build/skyframe/EagerInvalidatorTest.java new file mode 100644 index 0000000000..f12754a04c --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/EagerInvalidatorTest.java @@ -0,0 +1,616 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.skyframe.GraphTester.CONCATENATE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.testing.GcFinalization; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.skyframe.GraphTester.StringValue; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingInvalidationState; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState; +import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationType; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.lang.ref.WeakReference; +import java.util.HashSet; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * Tests for {@link InvalidatingNodeVisitor}. + */ +@RunWith(Enclosed.class) +public class EagerInvalidatorTest { + protected InMemoryGraph graph; + protected GraphTester tester = new GraphTester(); + protected InvalidationState state = newInvalidationState(); + protected AtomicReference<InvalidatingNodeVisitor> visitor = new AtomicReference<>(); + protected DirtyKeyTrackerImpl dirtyKeyTracker; + + private IntVersion graphVersion = new IntVersion(0); + + // The following three methods should be abstract, but junit4 does not allow us to run inner + // classes in an abstract outer class. Thus, we provide implementations. These methods will never + // be run because only the inner classes, annotated with @RunWith, will actually be executed. + EvaluationProgressReceiver.InvalidationState expectedState() { + throw new UnsupportedOperationException(); + } + + @SuppressWarnings("unused") // Overridden by subclasses. + void invalidate(DirtiableGraph graph, EvaluationProgressReceiver invalidationReceiver, + SkyKey... keys) throws InterruptedException { throw new UnsupportedOperationException(); } + + boolean gcExpected() { throw new UnsupportedOperationException(); } + + private boolean isInvalidated(SkyKey key) { + NodeEntry entry = graph.get(key); + if (gcExpected()) { + return entry == null; + } else { + return entry == null || entry.isDirty(); + } + } + + private void assertChanged(SkyKey key) { + NodeEntry entry = graph.get(key); + if (gcExpected()) { + assertNull(entry); + } else { + assertTrue(entry.isChanged()); + } + } + + private void assertDirtyAndNotChanged(SkyKey key) { + NodeEntry entry = graph.get(key); + if (gcExpected()) { + assertNull(entry); + } else { + assertTrue(entry.isDirty()); + assertFalse(entry.isChanged()); + } + + } + + protected InvalidationState newInvalidationState() { + throw new UnsupportedOperationException("Sublcasses must override"); + } + + protected InvalidationType defaultInvalidationType() { + throw new UnsupportedOperationException("Sublcasses must override"); + } + + // Convenience method for eval-ing a single value. + protected SkyValue eval(boolean keepGoing, SkyKey key) throws InterruptedException { + SkyKey[] keys = { key }; + return eval(keepGoing, keys).get(key); + } + + protected <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, SkyKey... keys) + throws InterruptedException { + Reporter reporter = new Reporter(); + ParallelEvaluator evaluator = new ParallelEvaluator(graph, graphVersion, + ImmutableMap.of(GraphTester.NODE_TYPE, tester.createDelegatingFunction()), + reporter, new MemoizingEvaluator.EmittedEventState(), keepGoing, 200, null, + new DirtyKeyTrackerImpl()); + graphVersion = graphVersion.next(); + return evaluator.eval(ImmutableList.copyOf(keys)); + } + + protected void invalidateWithoutError(@Nullable EvaluationProgressReceiver invalidationReceiver, + SkyKey... keys) throws InterruptedException { + invalidate(graph, invalidationReceiver, keys); + assertTrue(state.isEmpty()); + } + + protected void set(String name, String value) { + tester.set(name, new StringValue(value)); + } + + protected SkyKey skyKey(String name) { + return GraphTester.toSkyKeys(name)[0]; + } + + protected void assertValueValue(String name, String expectedValue) throws InterruptedException { + StringValue value = (StringValue) eval(false, skyKey(name)); + assertEquals(expectedValue, value.getValue()); + } + + @Before + public void setUp() throws Exception { + dirtyKeyTracker = new DirtyKeyTrackerImpl(); + } + + @Test + public void receiverWorks() throws Exception { + final Set<String> invalidated = Sets.newConcurrentHashSet(); + EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + Preconditions.checkState(state == expectedState()); + invalidated.add(((StringValue) value).getValue()); + } + + @Override + public void enqueueing(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + throw new UnsupportedOperationException(); + } + }; + graph = new InMemoryGraph(); + set("a", "a"); + set("b", "b"); + tester.getOrCreate("ab").addDependency("a").addDependency("b") + .setComputedValue(CONCATENATE); + assertValueValue("ab", "ab"); + + set("a", "c"); + invalidateWithoutError(receiver, skyKey("a")); + assertThat(invalidated).containsExactly("a", "ab"); + assertValueValue("ab", "cb"); + set("b", "d"); + invalidateWithoutError(receiver, skyKey("b")); + assertThat(invalidated).containsExactly("a", "ab", "b", "cb"); + } + + @Test + public void receiverIsNotNotifiedAboutValuesInError() throws Exception { + final Set<String> invalidated = Sets.newConcurrentHashSet(); + EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + Preconditions.checkState(state == expectedState()); + invalidated.add(((StringValue) value).getValue()); + } + + @Override + public void enqueueing(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + throw new UnsupportedOperationException(); + } + }; + + graph = new InMemoryGraph(); + set("a", "a"); + tester.getOrCreate("ab").addDependency("a").setHasError(true); + eval(false, skyKey("ab")); + + invalidateWithoutError(receiver, skyKey("a")); + assertThat(invalidated).containsExactly("a").inOrder(); + } + + @Test + public void invalidateValuesNotInGraph() throws Exception { + final Set<String> invalidated = Sets.newConcurrentHashSet(); + EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + Preconditions.checkState(state == InvalidationState.DIRTY); + invalidated.add(((StringValue) value).getValue()); + } + + @Override + public void enqueueing(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + throw new UnsupportedOperationException(); + } + }; + graph = new InMemoryGraph(); + invalidateWithoutError(receiver, skyKey("a")); + assertThat(invalidated).isEmpty(); + set("a", "a"); + assertValueValue("a", "a"); + invalidateWithoutError(receiver, skyKey("b")); + assertThat(invalidated).isEmpty(); + } + + @Test + public void invalidatedValuesAreGCedAsExpected() throws Exception { + SkyKey key = GraphTester.skyKey("a"); + HeavyValue heavyValue = new HeavyValue(); + WeakReference<HeavyValue> weakRef = new WeakReference<>(heavyValue); + tester.set("a", heavyValue); + + graph = new InMemoryGraph(); + eval(false, key); + invalidate(graph, null, key); + + tester = null; + heavyValue = null; + if (gcExpected()) { + GcFinalization.awaitClear(weakRef); + } else { + // Not a reliable check, but better than nothing. + System.gc(); + Thread.sleep(300); + assertNotNull(weakRef.get()); + } + } + + @Test + public void reverseDepsConsistent() throws Exception { + graph = new InMemoryGraph(); + set("a", "a"); + set("b", "b"); + set("c", "c"); + tester.getOrCreate("ab").addDependency("a").addDependency("b").setComputedValue(CONCATENATE); + tester.getOrCreate("bc").addDependency("b").addDependency("c").setComputedValue(CONCATENATE); + tester.getOrCreate("ab_c").addDependency("ab").addDependency("c") + .setComputedValue(CONCATENATE); + eval(false, skyKey("ab_c"), skyKey("bc")); + + assertThat(graph.get(skyKey("a")).getReverseDeps()).containsExactly(skyKey("ab")); + assertThat(graph.get(skyKey("b")).getReverseDeps()).containsExactly(skyKey("ab"), skyKey("bc")); + assertThat(graph.get(skyKey("c")).getReverseDeps()).containsExactly(skyKey("ab_c"), + skyKey("bc")); + + invalidateWithoutError(null, skyKey("ab")); + eval(false); + + // The graph values should be gone. + assertTrue(isInvalidated(skyKey("ab"))); + assertTrue(isInvalidated(skyKey("abc"))); + + // The reverse deps to ab and ab_c should have been removed. + assertThat(graph.get(skyKey("a")).getReverseDeps()).isEmpty(); + assertThat(graph.get(skyKey("b")).getReverseDeps()).containsExactly(skyKey("bc")); + assertThat(graph.get(skyKey("c")).getReverseDeps()).containsExactly(skyKey("bc")); + } + + @Test + public void interruptChild() throws Exception { + graph = new InMemoryGraph(); + int numValues = 50; // More values than the invalidator has threads. + final SkyKey[] family = new SkyKey[numValues]; + final SkyKey child = GraphTester.skyKey("child"); + final StringValue childValue = new StringValue("child"); + tester.set(child, childValue); + family[0] = child; + for (int i = 1; i < numValues; i++) { + SkyKey member = skyKey(Integer.toString(i)); + tester.getOrCreate(member).addDependency(family[i - 1]).setComputedValue(CONCATENATE); + family[i] = member; + } + SkyKey parent = GraphTester.skyKey("parent"); + tester.getOrCreate(parent).addDependency(family[numValues - 1]).setComputedValue(CONCATENATE); + eval(/*keepGoing=*/false, parent); + final Thread mainThread = Thread.currentThread(); + final AtomicReference<SkyValue> badValue = new AtomicReference<>(); + EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + if (value == childValue) { + // Interrupt on the very first invalidate + mainThread.interrupt(); + } else if (!childValue.equals(value)) { + // All other invalidations should be of the same value. + // Exceptions thrown here may be silently dropped, so keep track of errors ourselves. + badValue.set(value); + } + try { + assertTrue(visitor.get().awaitInterruptionForTestingOnly(2, TimeUnit.HOURS)); + } catch (InterruptedException e) { + // We may well have thrown here because by the time we try to await, the main thread is + // already interrupted. + } + } + + @Override + public void enqueueing(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + throw new UnsupportedOperationException(); + } + }; + try { + invalidateWithoutError(receiver, child); + fail(); + } catch (InterruptedException e) { + // Expected. + } + assertNull(badValue.get()); + assertFalse(state.isEmpty()); + final Set<SkyValue> invalidated = Sets.newConcurrentHashSet(); + assertFalse(isInvalidated(parent)); + SkyValue parentValue = graph.getValue(parent); + assertNotNull(parentValue); + receiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + invalidated.add(value); + } + + @Override + public void enqueueing(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + throw new UnsupportedOperationException(); + } + }; + invalidateWithoutError(receiver); + assertTrue(invalidated.contains(parentValue)); + assertThat(state.getInvalidationsForTesting()).isEmpty(); + + // Regression test coverage: + // "all pending values are marked changed on interrupt". + assertTrue(isInvalidated(child)); + assertChanged(child); + for (int i = 1; i < numValues; i++) { + assertDirtyAndNotChanged(family[i]); + } + assertDirtyAndNotChanged(parent); + } + + private SkyKey[] constructLargeGraph(int size) { + Random random = new Random(TestUtils.getRandomSeed()); + SkyKey[] values = new SkyKey[size]; + for (int i = 0; i < size; i++) { + String iString = Integer.toString(i); + SkyKey iKey = GraphTester.toSkyKey(iString); + set(iString, iString); + for (int j = 0; j < i; j++) { + if (random.nextInt(3) == 0) { + tester.getOrCreate(iKey).addDependency(Integer.toString(j)); + } + } + values[i] = iKey; + } + return values; + } + + /** Returns a subset of {@code nodes} that are still valid and so can be invalidated. */ + private Set<Pair<SkyKey, InvalidationType>> getValuesToInvalidate(SkyKey[] nodes) { + Set<Pair<SkyKey, InvalidationType>> result = new HashSet<>(); + Random random = new Random(TestUtils.getRandomSeed()); + for (SkyKey node : nodes) { + if (!isInvalidated(node)) { + if (result.isEmpty() || random.nextInt(3) == 0) { + // Add at least one node, if we can. + result.add(Pair.of(node, defaultInvalidationType())); + } + } + } + return result; + } + + @Test + public void interruptThreadInReceiver() throws Exception { + Random random = new Random(TestUtils.getRandomSeed()); + int graphSize = 1000; + int tries = 5; + graph = new InMemoryGraph(); + SkyKey[] values = constructLargeGraph(graphSize); + eval(/*keepGoing=*/false, values); + final Thread mainThread = Thread.currentThread(); + for (int run = 0; run < tries; run++) { + Set<Pair<SkyKey, InvalidationType>> valuesToInvalidate = getValuesToInvalidate(values); + // Find how many invalidations will actually be enqueued for invalidation in the first round, + // so that we can interrupt before all of them are done. + int validValuesToDo = + Sets.difference(valuesToInvalidate, state.getInvalidationsForTesting()).size(); + for (Pair<SkyKey, InvalidationType> pair : state.getInvalidationsForTesting()) { + if (!isInvalidated(pair.first)) { + validValuesToDo++; + } + } + int countDownStart = validValuesToDo > 0 ? random.nextInt(validValuesToDo) : 0; + final CountDownLatch countDownToInterrupt = new CountDownLatch(countDownStart); + final EvaluationProgressReceiver receiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + countDownToInterrupt.countDown(); + if (countDownToInterrupt.getCount() == 0) { + mainThread.interrupt(); + try { + // Wait for the main thread to be interrupted uninterruptibly, because the main thread + // is going to interrupt us, and we don't want to get into an interrupt fight. Only + // if we get interrupted without the main thread also being interrupted will this + // throw an InterruptedException. + TrackingAwaiter.waitAndMaybeThrowInterrupt( + visitor.get().getInterruptionLatchForTestingOnly(), + "Main thread was not interrupted"); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + } + + @Override + public void enqueueing(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + throw new UnsupportedOperationException(); + } + }; + try { + invalidate(graph, receiver, + Sets.newHashSet( + Iterables.transform(valuesToInvalidate, + Pair.<SkyKey, InvalidationType>firstFunction())).toArray(new SkyKey[0])); + assertThat(state.getInvalidationsForTesting()).isEmpty(); + } catch (InterruptedException e) { + // Expected. + } + if (state.isEmpty()) { + // Ran out of values to invalidate. + break; + } + } + + eval(/*keepGoing=*/false, values); + } + + protected void setupInvalidatableGraph() throws Exception { + graph = new InMemoryGraph(); + set("a", "a"); + set("b", "b"); + tester.getOrCreate("ab").addDependency("a").addDependency("b").setComputedValue(CONCATENATE); + assertValueValue("ab", "ab"); + set("a", "c"); + } + + private static class HeavyValue implements SkyValue { + } + + /** + * Test suite for the deleting invalidator. + */ + @RunWith(JUnit4.class) + public static class DeletingInvalidatorTest extends EagerInvalidatorTest { + @Override + protected void invalidate(DirtiableGraph graph, EvaluationProgressReceiver invalidationReceiver, + SkyKey... keys) throws InterruptedException { + InvalidatingNodeVisitor invalidatingVisitor = + EagerInvalidator.createVisitor(/*delete=*/true, graph, ImmutableList.copyOf(keys), + invalidationReceiver, state, true, dirtyKeyTracker); + if (invalidatingVisitor != null) { + visitor.set(invalidatingVisitor); + invalidatingVisitor.run(); + } + } + + @Override + EvaluationProgressReceiver.InvalidationState expectedState() { + return EvaluationProgressReceiver.InvalidationState.DELETED; + } + + @Override + boolean gcExpected() { + return true; + } + + @Override + protected InvalidationState newInvalidationState() { + return new InvalidatingNodeVisitor.DeletingInvalidationState(); + } + + @Override + protected InvalidationType defaultInvalidationType() { + return InvalidationType.DELETED; + } + + @Test + public void dirtyKeyTrackerWorksWithDeletingInvalidator() throws Exception { + setupInvalidatableGraph(); + TrackingInvalidationReceiver receiver = new TrackingInvalidationReceiver(); + + // Dirty the node, and ensure that the tracker is aware of it: + InvalidatingNodeVisitor dirtyingVisitor = + EagerInvalidator.createVisitor(/*delete=*/false, graph, ImmutableList.of(skyKey("a")), + receiver, new DirtyingInvalidationState(), true, dirtyKeyTracker); + dirtyingVisitor.run(); + assertThat(dirtyKeyTracker.getDirtyKeys()).containsExactly(skyKey("a"), skyKey("ab")); + + // Delete the node, and ensure that the tracker is no longer tracking it: + InvalidatingNodeVisitor deletingVisitor = + EagerInvalidator.createVisitor(/*delete=*/true, graph, ImmutableList.of(skyKey("a")), + receiver, state, true, dirtyKeyTracker); + deletingVisitor.run(); + assertThat(dirtyKeyTracker.getDirtyKeys()).containsExactly(skyKey("ab")); + } + } + + /** + * Test suite for the dirtying invalidator. + */ + @RunWith(JUnit4.class) + public static class DirtyingInvalidatorTest extends EagerInvalidatorTest { + @Override + protected void invalidate(DirtiableGraph graph, EvaluationProgressReceiver invalidationReceiver, + SkyKey... keys) throws InterruptedException { + InvalidatingNodeVisitor invalidatingVisitor = + EagerInvalidator.createVisitor(/*delete=*/false, graph, ImmutableList.copyOf(keys), + invalidationReceiver, state, true, dirtyKeyTracker); + if (invalidatingVisitor != null) { + visitor.set(invalidatingVisitor); + invalidatingVisitor.run(); + } + } + + @Override + EvaluationProgressReceiver.InvalidationState expectedState() { + return EvaluationProgressReceiver.InvalidationState.DIRTY; + } + + @Override + boolean gcExpected() { + return false; + } + + @Override + protected InvalidationState newInvalidationState() { + return new DirtyingInvalidationState(); + } + + @Override + protected InvalidationType defaultInvalidationType() { + return InvalidationType.CHANGED; + } + + @Test + public void dirtyKeyTrackerWorksWithDirtyingInvalidator() throws Exception { + setupInvalidatableGraph(); + TrackingInvalidationReceiver receiver = new TrackingInvalidationReceiver(); + + // Dirty the node, and ensure that the tracker is aware of it: + invalidate(graph, receiver, skyKey("a")); + assertThat(dirtyKeyTracker.getDirtyKeys()).hasSize(2); + } + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/GenericFunctionException.java b/src/test/java/com/google/devtools/build/skyframe/GenericFunctionException.java new file mode 100644 index 0000000000..667f2137dc --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/GenericFunctionException.java @@ -0,0 +1,27 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +/** + * A {@link SkyFunctionException} wrapping a {@link SomeErrorException}. + */ +public final class GenericFunctionException extends SkyFunctionException { + public GenericFunctionException(SomeErrorException e, Transience transience) { + super(e, transience); + } + + public GenericFunctionException(SomeErrorException e, SkyKey childKey) { + super(e, childKey); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/GraphTester.java b/src/test/java/com/google/devtools/build/skyframe/GraphTester.java new file mode 100644 index 0000000000..6095bb8b80 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/GraphTester.java @@ -0,0 +1,340 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; + +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * A helper class to create graphs and run skyframe tests over these graphs. + * + * <p>There are two types of values, computing values, which may not be set to a constant value, + * and leaf values, which must be set to a constant value and may not have any dependencies. + * + * <p>Note that the value builder looks into the test values created here to determine how to + * behave. However, skyframe will only re-evaluate the value and call the value builder if any of + * its dependencies has changed. That means in order to change the set of dependencies of a value, + * you need to also change one of its previous dependencies to force re-evaluation. Changing a + * computing value does not mark it as modified. + */ +public class GraphTester { + + // TODO(bazel-team): Split this for computing and non-computing values? + public static final SkyFunctionName NODE_TYPE = new SkyFunctionName("Type", false); + + private final Map<SkyKey, TestFunction> values = new HashMap<>(); + private final Set<SkyKey> modifiedValues = new LinkedHashSet<>(); + + public TestFunction getOrCreate(String name) { + return getOrCreate(skyKey(name)); + } + + public TestFunction getOrCreate(SkyKey key) { + return getOrCreate(key, false); + } + + public TestFunction getOrCreate(SkyKey key, boolean markAsModified) { + TestFunction result = values.get(key); + if (result == null) { + result = new TestFunction(); + values.put(key, result); + } else if (markAsModified) { + modifiedValues.add(key); + } + return result; + } + + public TestFunction set(String key, SkyValue value) { + return set(skyKey(key), value); + } + + public TestFunction set(SkyKey key, SkyValue value) { + return getOrCreate(key, true).setConstantValue(value); + } + + public Collection<SkyKey> getModifiedValues() { + return modifiedValues; + } + + public SkyFunction getFunction() { + return new SkyFunction() { + @Override + public SkyValue compute(SkyKey key, Environment env) + throws SkyFunctionException, InterruptedException { + TestFunction builder = values.get(key); + Preconditions.checkState(builder != null, "No TestFunction for " + key); + if (builder.builder != null) { + return builder.builder.compute(key, env); + } + if (builder.warning != null) { + env.getListener().handle(Event.warn(builder.warning)); + } + if (builder.progress != null) { + env.getListener().handle(Event.progress(builder.progress)); + } + Map<SkyKey, SkyValue> deps = new LinkedHashMap<>(); + boolean oneMissing = false; + for (Pair<SkyKey, SkyValue> dep : builder.deps) { + SkyValue value; + if (dep.second == null) { + value = env.getValue(dep.first); + } else { + try { + value = env.getValueOrThrow(dep.first, SomeErrorException.class); + } catch (SomeErrorException e) { + value = dep.second; + } + } + if (value == null) { + oneMissing = true; + } else { + deps.put(dep.first, value); + } + Preconditions.checkState(oneMissing == env.valuesMissing()); + } + if (env.valuesMissing()) { + return null; + } + + if (builder.hasTransientError) { + throw new GenericFunctionException(new SomeErrorException(key.toString()), + Transience.TRANSIENT); + } + if (builder.hasError) { + throw new GenericFunctionException(new SomeErrorException(key.toString()), + Transience.PERSISTENT); + } + + if (builder.value != null) { + return builder.value; + } + + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException(key.toString()); + } + + return builder.computer.compute(deps, env); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return values.get(skyKey).tag; + } + }; + } + + public static SkyKey skyKey(String key) { + return new SkyKey(NODE_TYPE, key); + } + + /** + * A value in the testing graph that is constructed in the tester. + */ + public class TestFunction { + // TODO(bazel-team): We could use a multiset here to simulate multi-pass dependency discovery. + private final Set<Pair<SkyKey, SkyValue>> deps = new LinkedHashSet<>(); + private SkyValue value; + private ValueComputer computer; + private SkyFunction builder = null; + + private boolean hasTransientError; + private boolean hasError; + + private String warning; + private String progress; + + private String tag; + + public TestFunction addDependency(String name) { + return addDependency(skyKey(name)); + } + + public TestFunction addDependency(SkyKey key) { + deps.add(Pair.<SkyKey, SkyValue>of(key, null)); + return this; + } + + public TestFunction removeDependency(String name) { + return removeDependency(skyKey(name)); + } + + public TestFunction removeDependency(SkyKey key) { + deps.remove(Pair.<SkyKey, SkyValue>of(key, null)); + return this; + } + + public TestFunction addErrorDependency(String name, SkyValue altValue) { + return addErrorDependency(skyKey(name), altValue); + } + + public TestFunction addErrorDependency(SkyKey key, SkyValue altValue) { + deps.add(Pair.of(key, altValue)); + return this; + } + + public TestFunction setConstantValue(SkyValue value) { + Preconditions.checkState(this.computer == null); + this.value = value; + return this; + } + + public TestFunction setComputedValue(ValueComputer computer) { + Preconditions.checkState(this.value == null); + this.computer = computer; + return this; + } + + public TestFunction setBuilder(SkyFunction builder) { + Preconditions.checkState(this.value == null); + Preconditions.checkState(this.computer == null); + Preconditions.checkState(deps.isEmpty()); + Preconditions.checkState(!hasTransientError); + Preconditions.checkState(!hasError); + Preconditions.checkState(warning == null); + Preconditions.checkState(progress == null); + this.builder = builder; + return this; + } + + public TestFunction setHasTransientError(boolean hasError) { + this.hasTransientError = hasError; + return this; + } + + public TestFunction setHasError(boolean hasError) { + // TODO(bazel-team): switch to an enum for hasError. + this.hasError = hasError; + return this; + } + + public TestFunction setWarning(String warning) { + this.warning = warning; + return this; + } + + public TestFunction setProgress(String info) { + this.progress = info; + return this; + } + + public TestFunction setTag(String tag) { + this.tag = tag; + return this; + } + + } + + public static SkyKey[] toSkyKeys(String... names) { + SkyKey[] result = new SkyKey[names.length]; + for (int i = 0; i < names.length; i++) { + result[i] = new SkyKey(GraphTester.NODE_TYPE, names[i]); + } + return result; + } + + public static SkyKey toSkyKey(String name) { + return toSkyKeys(name)[0]; + } + + private class DelegatingFunction implements SkyFunction { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + return getFunction().compute(skyKey, env); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return getFunction().extractTag(skyKey); + } + } + + public DelegatingFunction createDelegatingFunction() { + return new DelegatingFunction(); + } + + /** + * Simple value class that stores strings. + */ + public static class StringValue implements SkyValue { + private final String value; + + public StringValue(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof StringValue)) { + return false; + } + return value.equals(((StringValue) o).value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return "StringValue: " + getValue(); + } + } + + /** + * A callback interface to provide the value computation. + */ + public interface ValueComputer { + /** This is called when all the declared dependencies exist. It may request new dependencies. */ + SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) + throws InterruptedException; + } + + public static final ValueComputer COPY = new ValueComputer() { + @Override + public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) { + return Iterables.getOnlyElement(deps.values()); + } + }; + + public static final ValueComputer CONCATENATE = new ValueComputer() { + @Override + public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) { + StringBuilder result = new StringBuilder(); + for (SkyValue value : deps.values()) { + result.append(((StringValue) value).value); + } + return new StringValue(result.toString()); + } + }; +} diff --git a/src/test/java/com/google/devtools/build/skyframe/MemoizingEvaluatorTest.java b/src/test/java/com/google/devtools/build/skyframe/MemoizingEvaluatorTest.java new file mode 100644 index 0000000000..00df824492 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/MemoizingEvaluatorTest.java @@ -0,0 +1,2914 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.devtools.build.skyframe.GraphTester.CONCATENATE; +import static com.google.devtools.build.skyframe.GraphTester.COPY; +import static com.google.devtools.build.skyframe.GraphTester.NODE_TYPE; +import static com.google.devtools.build.skyframe.GraphTester.skyKey; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.testing.GcFinalization; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.events.DelegatingEventHandler; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventCollector; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.testutil.JunitTestUtils; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.skyframe.GraphTester.StringValue; +import com.google.devtools.build.skyframe.GraphTester.TestFunction; +import com.google.devtools.build.skyframe.GraphTester.ValueComputer; +import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.EventType; +import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Listener; +import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Order; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.annotation.Nullable; + +/** + * Tests for {@link MemoizingEvaluator}. + */ +@RunWith(JUnit4.class) +public class MemoizingEvaluatorTest { + + private MemoizingEvaluatorTester tester; + private EventCollector eventCollector; + private EventHandler reporter; + private MemoizingEvaluator.EmittedEventState emittedEventState; + + // Knobs that control the size / duration of larger tests. + private static final int TEST_NODE_COUNT = 100; + private static final int TESTED_NODES = 10; + private static final int RUNS = 10; + + @Before + public void initializeTester() { + initializeTester(null); + } + + public void initializeTester(@Nullable TrackingInvalidationReceiver customInvalidationReceiver) { + emittedEventState = new MemoizingEvaluator.EmittedEventState(); + tester = new MemoizingEvaluatorTester(); + if (customInvalidationReceiver != null) { + tester.setInvalidationReceiver(customInvalidationReceiver); + } + tester.initialize(); + } + + @Before + public void initializeReporter() { + eventCollector = new EventCollector(EventKind.ALL_EVENTS); + reporter = new Reporter(eventCollector); + tester.resetPlayedEvents(); + } + + protected static SkyKey toSkyKey(String name) { + return new SkyKey(NODE_TYPE, name); + } + + @Test + public void smoke() throws Exception { + tester.set("x", new StringValue("y")); + StringValue value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + } + + @Test + public void invalidationWithNothingChanged() throws Exception { + tester.set("x", new StringValue("y")).setWarning("fizzlepop"); + StringValue value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop"); + JunitTestUtils.assertEventCount(1, eventCollector); + + initializeReporter(); + tester.invalidate(); + value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + private abstract static class NoExtractorFunction implements SkyFunction { + @Override + public final String extractTag(SkyKey skyKey) { + return null; + } + } + + @Test + // Regression test for bug: "[skyframe-m1]: registerIfDone() crash". + public void bubbleRace() throws Exception { + // The top-level value declares dependencies on a "badValue" in error, and a "sleepyValue" + // which is very slow. After "badValue" fails, the builder interrupts the "sleepyValue" and + // attempts to re-run "top" for error bubbling. Make sure this doesn't cause a precondition + // failure because "top" still has an outstanding dep ("sleepyValue"). + tester.getOrCreate("top").setBuilder(new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException { + env.getValue(toSkyKey("sleepyValue")); + try { + env.getValueOrThrow(toSkyKey("badValue"), SomeErrorException.class); + } catch (SomeErrorException e) { + // In order to trigger this bug, we need to request a dep on an already computed value. + env.getValue(toSkyKey("otherValue1")); + } + if (!env.valuesMissing()) { + throw new AssertionError("SleepyValue should always be unavailable"); + } + return null; + } + }); + tester.getOrCreate("sleepyValue").setBuilder(new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException { + Thread.sleep(99999); + throw new AssertionError("I should have been interrupted"); + } + }); + tester.getOrCreate("badValue").addDependency("otherValue1").setHasError(true); + tester.getOrCreate("otherValue1").setConstantValue(new StringValue("otherVal1")); + + EvaluationResult<SkyValue> result = tester.eval(false, "top"); + assertTrue(result.hasError()); + assertEquals(toSkyKey("badValue"), Iterables.getOnlyElement(result.getError().getRootCauses())); + assertThat(result.keyNames()).isEmpty(); + } + + @Test + public void deleteValues() throws Exception { + tester.getOrCreate("top").setComputedValue(CONCATENATE) + .addDependency("d1").addDependency("d2").addDependency("d3"); + tester.set("d1", new StringValue("1")); + StringValue d2 = new StringValue("2"); + tester.set("d2", d2); + StringValue d3 = new StringValue("3"); + tester.set("d3", d3); + tester.eval(true, "top"); + + tester.delete("d1"); + tester.eval(true, "d3"); + + assertThat(tester.getDirtyValues()).isEmpty(); + assertEquals( + ImmutableSet.of(new StringValue("1"), new StringValue("123")), tester.getDeletedValues()); + assertEquals(null, tester.getExistingValue("top")); + assertEquals(null, tester.getExistingValue("d1")); + assertEquals(d2, tester.getExistingValue("d2")); + assertEquals(d3, tester.getExistingValue("d3")); + } + + @Test + public void deleteOldNodesTest() throws Exception { + tester.getOrCreate("top").setComputedValue(CONCATENATE).addDependency("d1").addDependency("d2"); + tester.set("d1", new StringValue("one")); + tester.set("d2", new StringValue("two")); + tester.eval(true, "top"); + + tester.set("d2", new StringValue("three")); + tester.invalidate(); + tester.eval(true, "d2"); + + // The graph now contains the three above nodes (and ERROR_TRANSIENCE). + assertThat(tester.graph.getValues().keySet()).containsExactly( + skyKey("top"), skyKey("d1"), skyKey("d2"), ErrorTransienceValue.key()); + + String[] noKeys = {}; + tester.graph.deleteDirty(2); + tester.eval(true, noKeys); + + // The top node's value is dirty, but less than two generations old, so it wasn't deleted. + assertThat(tester.graph.getValues().keySet()).containsExactly( + skyKey("top"), skyKey("d1"), skyKey("d2"), ErrorTransienceValue.key()); + + tester.graph.deleteDirty(2); + tester.eval(true, noKeys); + + // The top node's value was dirty, and was two generations old, so it was deleted. + assertThat(tester.graph.getValues().keySet()).containsExactly( + skyKey("d1"), skyKey("d2"), ErrorTransienceValue.key()); + } + + @Test + public void deleteNonexistentValues() throws Exception { + tester.getOrCreate("d1").setConstantValue(new StringValue("1")); + tester.delete("d1"); + tester.delete("d2"); + tester.eval(true, "d1"); + } + + @Test + public void signalValueEnqueued() throws Exception { + tester.getOrCreate("top1").setComputedValue(CONCATENATE) + .addDependency("d1").addDependency("d2"); + tester.getOrCreate("top2").setComputedValue(CONCATENATE).addDependency("d3"); + tester.getOrCreate("top3"); + assertThat(tester.getEnqueuedValues()).isEmpty(); + + tester.set("d1", new StringValue("1")); + tester.set("d2", new StringValue("2")); + tester.set("d3", new StringValue("3")); + tester.eval(true, "top1"); + assertThat(tester.getEnqueuedValues()).containsExactlyElementsIn( + Arrays.asList(MemoizingEvaluatorTester.toSkyKeys("top1", "d1", "d2"))); + + tester.eval(true, "top2"); + assertThat(tester.getEnqueuedValues()).containsExactlyElementsIn( + Arrays.asList(MemoizingEvaluatorTester.toSkyKeys("top1", "d1", "d2", "top2", "d3"))); + } + + // NOTE: Some of these tests exercising errors/warnings run through a size-2 for loop in order + // to ensure that we are properly recording and replyaing these messages on null builds. + @Test + public void warningViaMultiplePaths() throws Exception { + tester.set("d1", new StringValue("d1")).setWarning("warn-d1"); + tester.set("d2", new StringValue("d2")).setWarning("warn-d2"); + tester.getOrCreate("top").setComputedValue(CONCATENATE).addDependency("d1").addDependency("d2"); + for (int i = 0; i < 2; i++) { + initializeReporter(); + tester.evalAndGet("top"); + JunitTestUtils.assertContainsEvent(eventCollector, "warn-d1"); + JunitTestUtils.assertContainsEvent(eventCollector, "warn-d2"); + JunitTestUtils.assertEventCount(2, eventCollector); + } + } + + @Test + public void warningBeforeErrorOnFailFastBuild() throws Exception { + tester.set("dep", new StringValue("dep")).setWarning("warn-dep"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).setHasError(true).addDependency("dep"); + for (int i = 0; i < 2; i++) { + initializeReporter(); + EvaluationResult<StringValue> result = tester.eval(false, "top"); + assertTrue(result.hasError()); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey); + assertEquals(topKey.toString(), result.getError(topKey).getException().getMessage()); + assertTrue(result.getError(topKey).getException() instanceof SomeErrorException); + JunitTestUtils.assertContainsEvent(eventCollector, "warn-dep"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + } + + @Test + public void warningAndErrorOnFailFastBuild() throws Exception { + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.set(topKey, new StringValue("top")).setWarning("warning msg").setHasError(true); + for (int i = 0; i < 2; i++) { + initializeReporter(); + EvaluationResult<StringValue> result = tester.eval(false, "top"); + assertTrue(result.hasError()); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey); + assertEquals(topKey.toString(), result.getError(topKey).getException().getMessage()); + assertTrue(result.getError(topKey).getException() instanceof SomeErrorException); + JunitTestUtils.assertContainsEvent(eventCollector, "warning msg"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + } + + @Test + public void warningAndErrorOnFailFastBuildAfterKeepGoingBuild() throws Exception { + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.set(topKey, new StringValue("top")).setWarning("warning msg").setHasError(true); + for (int i = 0; i < 2; i++) { + initializeReporter(); + EvaluationResult<StringValue> result = tester.eval(i == 0, "top"); + assertTrue(result.hasError()); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey); + assertEquals(topKey.toString(), result.getError(topKey).getException().getMessage()); + assertTrue(result.getError(topKey).getException() instanceof SomeErrorException); + JunitTestUtils.assertContainsEvent(eventCollector, "warning msg"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + } + + @Test + public void twoTLTsOnOneWarningValue() throws Exception { + tester.set("t1", new StringValue("t1")).addDependency("dep"); + tester.set("t2", new StringValue("t2")).addDependency("dep"); + tester.set("dep", new StringValue("dep")).setWarning("look both ways before crossing"); + for (int i = 0; i < 2; i++) { + // Make sure we see the warning exactly once. + initializeReporter(); + tester.eval(/*keepGoing=*/false, "t1", "t2"); + JunitTestUtils.assertContainsEvent(eventCollector, "look both ways before crossing"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + } + + @Test + public void errorValueDepOnWarningValue() throws Exception { + tester.getOrCreate("error-value").setHasError(true).addDependency("warning-value"); + tester.set("warning-value", new StringValue("warning-value")) + .setWarning("don't chew with your mouth open"); + + for (int i = 0; i < 2; i++) { + initializeReporter(); + tester.evalAndGetError("error-value"); + JunitTestUtils.assertContainsEvent(eventCollector, "don't chew with your mouth open"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + initializeReporter(); + tester.evalAndGet("warning-value"); + JunitTestUtils.assertContainsEvent(eventCollector, "don't chew with your mouth open"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + @Test + public void progressMessageOnlyPrintedTheFirstTime() throws Exception { + // The framework keeps track of warning and error messages, but not progress messages. + // So here we see both the progress and warning on the first build, but only the warning + // on the subsequent null build. + tester.set("x", new StringValue("y")).setWarning("fizzlepop") + .setProgress("just letting you know"); + + StringValue value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop"); + JunitTestUtils.assertContainsEvent(eventCollector, "just letting you know"); + JunitTestUtils.assertEventCount(2, eventCollector); + + // On the rebuild, we only replay warning messages. + initializeReporter(); + value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + @Test + public void invalidationWithChangeAndThenNothingChanged() throws Exception { + tester.getOrCreate("a") + .addDependency("b") + .setComputedValue(COPY); + tester.set("b", new StringValue("y")); + StringValue original = (StringValue) tester.evalAndGet("a"); + assertEquals("y", original.getValue()); + tester.set("b", new StringValue("z")); + tester.invalidate(); + StringValue old = (StringValue) tester.evalAndGet("a"); + assertEquals("z", old.getValue()); + tester.invalidate(); + StringValue current = (StringValue) tester.evalAndGet("a"); + assertSame(old, current); + } + + @Test + public void transientErrorValueInvalidation() throws Exception { + // Verify that invalidating errors causes all transient error values to be rerun. + tester.getOrCreate("error-value").setHasTransientError(true).setProgress( + "just letting you know"); + + tester.evalAndGetError("error-value"); + JunitTestUtils.assertContainsEvent(eventCollector, "just letting you know"); + JunitTestUtils.assertEventCount(1, eventCollector); + + // Change the progress message. + tester.getOrCreate("error-value").setHasTransientError(true).setProgress( + "letting you know more"); + + // Without invalidating errors, we shouldn't show the new progress message. + for (int i = 0; i < 2; i++) { + initializeReporter(); + tester.evalAndGetError("error-value"); + JunitTestUtils.assertNoEvents(eventCollector); + } + + // When invalidating errors, we should show the new progress message. + initializeReporter(); + tester.invalidateTransientErrors(); + tester.evalAndGetError("error-value"); + JunitTestUtils.assertContainsEvent(eventCollector, "letting you know more"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + @Test + public void simpleDependency() throws Exception { + tester.getOrCreate("ab") + .addDependency("a") + .setComputedValue(COPY); + tester.set("a", new StringValue("me")); + StringValue value = (StringValue) tester.evalAndGet("ab"); + assertEquals("me", value.getValue()); + } + + @Test + public void incrementalSimpleDependency() throws Exception { + tester.getOrCreate("ab") + .addDependency("a") + .setComputedValue(COPY); + tester.set("a", new StringValue("me")); + tester.evalAndGet("ab"); + + tester.set("a", new StringValue("other")); + tester.invalidate(); + StringValue value = (StringValue) tester.evalAndGet("ab"); + assertEquals("other", value.getValue()); + } + + @Test + public void diamondDependency() throws Exception { + setupDiamondDependency(); + tester.set("d", new StringValue("me")); + StringValue value = (StringValue) tester.evalAndGet("a"); + assertEquals("meme", value.getValue()); + } + + @Test + public void incrementalDiamondDependency() throws Exception { + setupDiamondDependency(); + tester.set("d", new StringValue("me")); + tester.evalAndGet("a"); + + tester.set("d", new StringValue("other")); + tester.invalidate(); + StringValue value = (StringValue) tester.evalAndGet("a"); + assertEquals("otherother", value.getValue()); + } + + private void setupDiamondDependency() { + tester.getOrCreate("a") + .addDependency("b") + .addDependency("c") + .setComputedValue(CONCATENATE); + tester.getOrCreate("b") + .addDependency("d") + .setComputedValue(COPY); + tester.getOrCreate("c") + .addDependency("d") + .setComputedValue(COPY); + } + + // Regression test: ParallelEvaluator notifies ValueProgressReceiver of already-built top-level + // values in error: we built "top" and "mid" as top-level targets; "mid" contains an error. We + // make sure "mid" is built as a dependency of "top" before enqueuing mid as a top-level target + // (by using a latch), so that the top-level enqueuing finds that mid has already been built. The + // progress receiver should not be notified of any value having been evaluated. + @Test + public void alreadyAnalyzedBadTarget() throws Exception { + final SkyKey mid = GraphTester.toSkyKey("mid"); + final CountDownLatch valueSet = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiter = new TrackingAwaiter(); + setGraphForTesting(new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (!key.equals(mid)) { + return; + } + switch (type) { + case ADD_REVERSE_DEP: + if (context == null) { + // Context is null when we are enqueuing this value as a top-level job. + trackingAwaiter.awaitLatchAndTrackExceptions(valueSet, "value not set"); + } + break; + case SET_VALUE: + valueSet.countDown(); + break; + default: + break; + } + } + })); + SkyKey top = GraphTester.skyKey("top"); + tester.getOrCreate(top).addDependency(mid).setComputedValue(CONCATENATE); + tester.getOrCreate(mid).setHasError(true); + tester.eval(/*keepGoing=*/false, top, mid); + assertEquals(0L, valueSet.getCount()); + trackingAwaiter.assertNoErrors(); + assertThat(tester.invalidationReceiver.evaluated).isEmpty(); + } + + @Test + public void receiverNotToldOfVerifiedValueDependingOnCycle() throws Exception { + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey cycle = GraphTester.toSkyKey("cycle"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.set(leaf, new StringValue("leaf")); + tester.getOrCreate(cycle).addDependency(cycle); + tester.getOrCreate(top).addDependency(leaf).addDependency(cycle); + tester.eval(/*keepGoing=*/true, top); + assertThat(tester.invalidationReceiver.evaluated).containsExactly(leaf).inOrder(); + tester.invalidationReceiver.clear(); + tester.getOrCreate(leaf, /*markAsModified=*/true); + tester.invalidate(); + tester.eval(/*keepGoing=*/true, top); + assertThat(tester.invalidationReceiver.evaluated).containsExactly(leaf).inOrder(); + } + + @Test + public void incrementalAddedDependency() throws Exception { + tester.getOrCreate("a") + .addDependency("b") + .setComputedValue(CONCATENATE); + tester.set("b", new StringValue("first")); + tester.set("c", new StringValue("second")); + tester.evalAndGet("a"); + + tester.getOrCreate("a").addDependency("c"); + tester.set("b", new StringValue("now")); + tester.invalidate(); + StringValue value = (StringValue) tester.evalAndGet("a"); + assertEquals("nowsecond", value.getValue()); + } + + @Test + public void manyValuesDependOnSingleValue() throws Exception { + initializeTester(); + String[] values = new String[TEST_NODE_COUNT]; + for (int i = 0; i < values.length; i++) { + values[i] = Integer.toString(i); + tester.getOrCreate(values[i]) + .addDependency("leaf") + .setComputedValue(COPY); + } + tester.set("leaf", new StringValue("leaf")); + + EvaluationResult<StringValue> result = tester.eval(/*keep_going=*/false, values); + for (int i = 0; i < values.length; i++) { + SkyValue actual = result.get(new SkyKey(GraphTester.NODE_TYPE, values[i])); + assertEquals(new StringValue("leaf"), actual); + } + + for (int j = 0; j < TESTED_NODES; j++) { + tester.set("leaf", new StringValue("other" + j)); + tester.invalidate(); + result = tester.eval(/*keep_going=*/false, values); + for (int i = 0; i < values.length; i++) { + SkyValue actual = result.get(new SkyKey(GraphTester.NODE_TYPE, values[i])); + assertEquals("Run " + j + ", value " + i, new StringValue("other" + j), actual); + } + } + } + + @Test + public void singleValueDependsOnManyValues() throws Exception { + initializeTester(); + String[] values = new String[TEST_NODE_COUNT]; + StringBuilder expected = new StringBuilder(); + for (int i = 0; i < values.length; i++) { + values[i] = Integer.toString(i); + tester.set(values[i], new StringValue(values[i])); + expected.append(values[i]); + } + SkyKey rootKey = new SkyKey(GraphTester.NODE_TYPE, "root"); + TestFunction value = tester.getOrCreate(rootKey) + .setComputedValue(CONCATENATE); + for (int i = 0; i < values.length; i++) { + value.addDependency(values[i]); + } + + EvaluationResult<StringValue> result = tester.eval(/*keep_going=*/false, rootKey); + assertEquals(new StringValue(expected.toString()), result.get(rootKey)); + + for (int j = 0; j < 10; j++) { + expected.setLength(0); + for (int i = 0; i < values.length; i++) { + String s = "other" + i + " " + j; + tester.set(values[i], new StringValue(s)); + expected.append(s); + } + tester.invalidate(); + + result = tester.eval(/*keep_going=*/false, rootKey); + assertEquals(new StringValue(expected.toString()), result.get(rootKey)); + } + } + + @Test + public void twoRailLeftRightDependencies() throws Exception { + initializeTester(); + String[] leftValues = new String[TEST_NODE_COUNT]; + String[] rightValues = new String[TEST_NODE_COUNT]; + for (int i = 0; i < leftValues.length; i++) { + leftValues[i] = "left-" + i; + rightValues[i] = "right-" + i; + if (i == 0) { + tester.getOrCreate(leftValues[i]) + .addDependency("leaf") + .setComputedValue(COPY); + tester.getOrCreate(rightValues[i]) + .addDependency("leaf") + .setComputedValue(COPY); + } else { + tester.getOrCreate(leftValues[i]) + .addDependency(leftValues[i - 1]) + .addDependency(rightValues[i - 1]) + .setComputedValue(new PassThroughSelected(toSkyKey(leftValues[i - 1]))); + tester.getOrCreate(rightValues[i]) + .addDependency(leftValues[i - 1]) + .addDependency(rightValues[i - 1]) + .setComputedValue(new PassThroughSelected(toSkyKey(rightValues[i - 1]))); + } + } + tester.set("leaf", new StringValue("leaf")); + + String lastLeft = "left-" + (TEST_NODE_COUNT - 1); + String lastRight = "right-" + (TEST_NODE_COUNT - 1); + + EvaluationResult<StringValue> result = tester.eval(/*keep_going=*/false, lastLeft, lastRight); + assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastLeft))); + assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastRight))); + + for (int j = 0; j < TESTED_NODES; j++) { + String value = "other" + j; + tester.set("leaf", new StringValue(value)); + tester.invalidate(); + result = tester.eval(/*keep_going=*/false, lastLeft, lastRight); + assertEquals(new StringValue(value), result.get(toSkyKey(lastLeft))); + assertEquals(new StringValue(value), result.get(toSkyKey(lastRight))); + } + } + + @Test + public void noKeepGoingAfterKeepGoingCycle() throws Exception { + initializeTester(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey goodKey = GraphTester.toSkyKey("good"); + StringValue goodValue = new StringValue("good"); + tester.set(goodKey, goodValue); + tester.getOrCreate(topKey).addDependency(midKey); + tester.getOrCreate(midKey).addDependency(aKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, topKey, goodKey); + assertEquals(goodValue, result.get(goodKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder(); + + tester.invalidate(); + result = tester.eval(/*keepGoing=*/false, topKey, goodKey); + assertEquals(null, result.get(topKey)); + errorInfo = result.getError(topKey); + cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder(); + } + + @Test + public void changeCycle() throws Exception { + initializeTester(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(COPY); + tester.getOrCreate(midKey).addDependency(aKey).setComputedValue(COPY); + tester.getOrCreate(aKey).addDependency(bKey).setComputedValue(COPY); + tester.getOrCreate(bKey).addDependency(aKey); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder(); + + tester.getOrCreate(bKey).removeDependency(aKey); + tester.set(bKey, new StringValue("bValue")); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/false, topKey); + assertEquals(new StringValue("bValue"), result.get(topKey)); + assertEquals(null, result.getError(topKey)); + } + + /** Regression test: "crash in cycle checker with dirty values". */ + @Test + public void cycleAndSelfEdgeWithDirtyValue() throws Exception { + initializeTester(); + SkyKey cycleKey1 = GraphTester.toSkyKey("cycleKey1"); + SkyKey cycleKey2 = GraphTester.toSkyKey("cycleKey2"); + tester.getOrCreate(cycleKey1).addDependency(cycleKey2).addDependency(cycleKey1) + .setComputedValue(CONCATENATE); + tester.getOrCreate(cycleKey2).addDependency(cycleKey1).setComputedValue(COPY); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, cycleKey1); + assertEquals(null, result.get(cycleKey1)); + ErrorInfo errorInfo = result.getError(cycleKey1); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1).inOrder(); + assertThat(cycleInfo.getPathToCycle()).isEmpty(); + tester.getOrCreate(cycleKey1, /*markAsModified=*/true); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/true, cycleKey1, cycleKey2); + assertEquals(null, result.get(cycleKey1)); + errorInfo = result.getError(cycleKey1); + cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1).inOrder(); + assertThat(cycleInfo.getPathToCycle()).isEmpty(); + cycleInfo = + Iterables.getOnlyElement(tester.graph.getExistingErrorForTesting(cycleKey2).getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(cycleKey2).inOrder(); + } + + /** Regression test: "crash in cycle checker with dirty values". */ + @Test + public void cycleWithDirtyValue() throws Exception { + initializeTester(); + SkyKey cycleKey1 = GraphTester.toSkyKey("cycleKey1"); + SkyKey cycleKey2 = GraphTester.toSkyKey("cycleKey2"); + tester.getOrCreate(cycleKey1).addDependency(cycleKey2).setComputedValue(COPY); + tester.getOrCreate(cycleKey2).addDependency(cycleKey1).setComputedValue(COPY); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, cycleKey1); + assertEquals(null, result.get(cycleKey1)); + ErrorInfo errorInfo = result.getError(cycleKey1); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1, cycleKey2).inOrder(); + assertThat(cycleInfo.getPathToCycle()).isEmpty(); + tester.getOrCreate(cycleKey1, /*markAsModified=*/true); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/true, cycleKey1); + assertEquals(null, result.get(cycleKey1)); + errorInfo = result.getError(cycleKey1); + cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(cycleKey1, cycleKey2).inOrder(); + assertThat(cycleInfo.getPathToCycle()).isEmpty(); + } + + /** + * Regression test: IllegalStateException in BuildingState.isReady(). The ParallelEvaluator used + * to assume during cycle-checking that all values had been built as fully as possible -- that + * evaluation had not been interrupted. However, we also do cycle-checking in nokeep-going mode + * when a value throws an error (possibly prematurely shutting down evaluation) but that error + * then bubbles up into a cycle. + * + * <p>We want to achieve the following state: we are checking for a cycle; the value we examine + * has not yet finished checking its children to see if they are dirty; but all children checked + * so far have been unchanged. This value is "otherTop". We first build otherTop, then mark its + * first child changed (without actually changing it), and then do a second build. On the second + * build, we also build "top", which requests a cycle that depends on an error. We wait to signal + * otherTop that its first child is done until the error throws and shuts down evaluation. The + * error then bubbles up to the cycle, and so the bubbling is aborted. Finally, cycle checking + * happens, and otherTop is examined, as desired. + */ + @Test + public void cycleAndErrorAndReady() throws Exception { + // This value will not have finished building on the second build when the error is thrown. + final SkyKey otherTop = GraphTester.toSkyKey("otherTop"); + final SkyKey errorKey = GraphTester.toSkyKey("error"); + // Is the graph state all set up and ready for the error to be thrown? + final CountDownLatch valuesReady = new CountDownLatch(3); + // Is evaluation being shut down? This is counted down by the exceptionMarker's builder, after + // it has waited for the threadpool's exception latch to be released. + final CountDownLatch errorThrown = new CountDownLatch(1); + // We don't do anything on the first build. + final AtomicBoolean secondBuild = new AtomicBoolean(false); + final TrackingAwaiter trackingAwaiter = new TrackingAwaiter(); + setGraphForTesting(new DeterministicInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (!secondBuild.get()) { + return; + } + if (key.equals(errorKey) && type == EventType.SET_VALUE) { + // If the error is about to be thrown, make sure all listeners are ready. + trackingAwaiter.awaitLatchAndTrackExceptions(valuesReady, "waiting values not ready"); + return; + } + if (key.equals(otherTop) && type == EventType.SIGNAL) { + // otherTop is being signaled that dep1 is done. Tell the error value that it is ready, + // then wait until the error is thrown, so that otherTop's builder is not re-entered. + valuesReady.countDown(); + trackingAwaiter.awaitLatchAndTrackExceptions(errorThrown, "error not thrown"); + return; + } + } + })); + final SkyKey dep1 = GraphTester.toSkyKey("dep1"); + tester.set(dep1, new StringValue("dep1")); + final SkyKey dep2 = GraphTester.toSkyKey("dep2"); + tester.set(dep2, new StringValue("dep2")); + // otherTop should request the deps one at a time, so that it can be in the CHECK_DEPENDENCIES + // state even after one dep is re-evaluated. + tester.getOrCreate(otherTop).setBuilder(new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + env.getValue(dep1); + if (env.valuesMissing()) { + return null; + } + env.getValue(dep2); + return env.valuesMissing() ? null : new StringValue("otherTop"); + } + }); + // Prime the graph with otherTop, so we can dirty it next build. + assertEquals(new StringValue("otherTop"), tester.evalAndGet(/*keepGoing=*/false, otherTop)); + // Mark dep1 changed, so otherTop will be dirty and request re-evaluation of dep1. + tester.getOrCreate(dep1, /*markAsModified=*/true); + SkyKey topKey = GraphTester.toSkyKey("top"); + // Note that since DeterministicInMemoryGraph alphabetizes reverse deps, it is important that + // "cycle2" comes before "top". + final SkyKey cycle1Key = GraphTester.toSkyKey("cycle1"); + final SkyKey cycle2Key = GraphTester.toSkyKey("cycle2"); + tester.getOrCreate(topKey).addDependency(cycle1Key).setComputedValue(CONCATENATE); + tester.getOrCreate(cycle1Key).addDependency(errorKey).addDependency(cycle2Key) + .setComputedValue(CONCATENATE); + tester.getOrCreate(errorKey).setHasError(true); + // Make sure cycle2Key has declared its dependence on cycle1Key before error throws. + tester.getOrCreate(cycle2Key).setBuilder(new ChainedFunction(/*notifyStart=*/valuesReady, + null, null, false, new StringValue("never returned"), ImmutableList.<SkyKey>of(cycle1Key))); + // Value that waits until an exception is thrown to finish building. We use it just to be + // informed when the threadpool is shutting down. + final SkyKey exceptionMarker = GraphTester.toSkyKey("exceptionMarker"); + tester.getOrCreate(exceptionMarker).setBuilder(new ChainedFunction( + /*notifyStart=*/valuesReady, /*waitToFinish=*/new CountDownLatch(0), + /*notifyFinish=*/errorThrown, + /*waitForException=*/true, new StringValue("exception marker"), + ImmutableList.<SkyKey>of())); + tester.invalidate(); + secondBuild.set(true); + // otherTop must be first, since we check top-level values for cycles in the order in which + // they appear here. + EvaluationResult<StringValue> result = + tester.eval(/*keepGoing=*/false, otherTop, topKey, exceptionMarker); + trackingAwaiter.assertNoErrors(); + assertThat(result.errorMap().keySet()).containsExactly(topKey); + Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo(); + assertWithMessage(result.toString()).that(cycleInfos).isNotEmpty(); + CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey); + assertThat(cycleInfo.getCycle()).containsExactly(cycle1Key, cycle2Key); + } + + @Test + public void limitEvaluatorThreads() throws Exception { + initializeTester(); + + int numKeys = 10; + final Object lock = new Object(); + final AtomicInteger inProgressCount = new AtomicInteger(); + final int[] maxValue = {0}; + + SkyKey topLevel = GraphTester.toSkyKey("toplevel"); + TestFunction topLevelBuilder = tester.getOrCreate(topLevel); + for (int i = 0; i < numKeys; i++) { + topLevelBuilder.addDependency("subKey" + i); + tester.getOrCreate("subKey" + i).setComputedValue(new ValueComputer() { + @Override + public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) { + int val = inProgressCount.incrementAndGet(); + synchronized (lock) { + if (val > maxValue[0]) { + maxValue[0] = val; + } + } + Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); + + inProgressCount.decrementAndGet(); + return new StringValue("abc"); + } + }); + } + topLevelBuilder.setConstantValue(new StringValue("xyz")); + + EvaluationResult<StringValue> result = tester.eval( + /*keepGoing=*/true, /*numThreads=*/5, topLevel); + assertFalse(result.hasError()); + assertEquals(5, maxValue[0]); + } + + /** + * Regression test: error on clearMaybeDirtyValue. We do an evaluation of topKey, which registers + * dependencies on midKey and errorKey. midKey enqueues slowKey, and waits. errorKey throws an + * error, which bubbles up to topKey. If topKey does not unregister its dependence on midKey, it + * will have a dangling reference to midKey after unfinished values are cleaned from the graph. + * Note that slowKey will wait until errorKey has thrown and the threadpool has caught the + * exception before returning, so the Evaluator will already have stopped enqueuing new jobs, so + * midKey is not evaluated. + */ + @Test + public void incompleteDirectDepsAreClearedBeforeInvalidation() throws Exception { + initializeTester(); + CountDownLatch slowStart = new CountDownLatch(1); + CountDownLatch errorFinish = new CountDownLatch(1); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart, + /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null, + /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey slowKey = GraphTester.toSkyKey("slow"); + tester.getOrCreate(slowKey).setBuilder( + new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish, + /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("slow"), + /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(midKey).addDependency(errorKey) + .setComputedValue(CONCATENATE); + // slowKey starts -> errorKey finishes, written to graph -> slowKey finishes & (Visitor aborts) + // -> topKey builds. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertThat(result.getError().getRootCauses()).containsExactly(errorKey); + // Make sure midKey didn't finish building. + assertEquals(null, tester.graph.getExistingValueForTesting(midKey)); + // Give slowKey a nice ordinary builder. + tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null) + .setConstantValue(new StringValue("slow")); + // Put midKey into the graph. It won't have a reverse dependence on topKey. + tester.evalAndGet(/*keepGoing=*/false, midKey); + tester.differencer.invalidate(ImmutableList.of(errorKey)); + // topKey should not access midKey as if it were already registered as a dependency. + tester.eval(/*keepGoing=*/false, topKey); + } + + /** + * Regression test: error on clearMaybeDirtyValue. Same as the previous test, but the second + * evaluation is keepGoing, which should cause an access of the children of topKey. + */ + @Test + public void incompleteDirectDepsAreClearedBeforeKeepGoing() throws Exception { + initializeTester(); + CountDownLatch slowStart = new CountDownLatch(1); + CountDownLatch errorFinish = new CountDownLatch(1); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart, + /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null, + /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey slowKey = GraphTester.toSkyKey("slow"); + tester.getOrCreate(slowKey).setBuilder( + new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish, + /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("slow"), + /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(midKey).addDependency(errorKey) + .setComputedValue(CONCATENATE); + // slowKey starts -> errorKey finishes, written to graph -> slowKey finishes & (Visitor aborts) + // -> topKey builds. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertThat(result.getError().getRootCauses()).containsExactly(errorKey); + // Make sure midKey didn't finish building. + assertEquals(null, tester.graph.getExistingValueForTesting(midKey)); + // Give slowKey a nice ordinary builder. + tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null) + .setConstantValue(new StringValue("slow")); + // Put midKey into the graph. It won't have a reverse dependence on topKey. + tester.evalAndGet(/*keepGoing=*/false, midKey); + // topKey should not access midKey as if it were already registered as a dependency. + // We don't invalidate errors, but because topKey wasn't actually written to the graph last + // build, it should be rebuilt here. + tester.eval(/*keepGoing=*/true, topKey); + } + + /** + * Regression test: tests that pass before other build actions fail yield crash in non -k builds. + */ + @Test + public void passThenFailToBuild() throws Exception { + CountDownLatch blocker = new CountDownLatch(1); + SkyKey successKey = GraphTester.toSkyKey("success"); + tester.getOrCreate(successKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null, + /*notifyFinish=*/blocker, /*waitForException=*/false, new StringValue("yippee"), + /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey slowFailKey = GraphTester.toSkyKey("slow_then_fail"); + tester.getOrCreate(slowFailKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/blocker, + /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null, + /*deps=*/ImmutableList.<SkyKey>of())); + + EvaluationResult<StringValue> result = tester.eval( + /*keepGoing=*/false, successKey, slowFailKey); + assertThat(result.getError().getRootCauses()).containsExactly(slowFailKey); + assertThat(result.values()).containsExactly(new StringValue("yippee")); + } + + @Test + public void passThenFailToBuildAlternateOrder() throws Exception { + CountDownLatch blocker = new CountDownLatch(1); + SkyKey successKey = GraphTester.toSkyKey("success"); + tester.getOrCreate(successKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null, + /*notifyFinish=*/blocker, /*waitForException=*/false, new StringValue("yippee"), + /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey slowFailKey = GraphTester.toSkyKey("slow_then_fail"); + tester.getOrCreate(slowFailKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/blocker, + /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null, + /*deps=*/ImmutableList.<SkyKey>of())); + + EvaluationResult<StringValue> result = tester.eval( + /*keepGoing=*/false, slowFailKey, successKey); + assertThat(result.getError().getRootCauses()).containsExactly(slowFailKey); + assertThat(result.values()).containsExactly(new StringValue("yippee")); + } + + @Test + public void incompleteDirectDepsForDirtyValue() throws Exception { + initializeTester(); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.set(topKey, new StringValue("initial")); + // Put topKey into graph so it will be dirtied on next run. + assertEquals(new StringValue("initial"), tester.evalAndGet(/*keepGoing=*/false, topKey)); + CountDownLatch slowStart = new CountDownLatch(1); + CountDownLatch errorFinish = new CountDownLatch(1); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart, + /*notifyFinish=*/errorFinish, + /*waitForException=*/false, /*value=*/null, /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey slowKey = GraphTester.toSkyKey("slow"); + tester.getOrCreate(slowKey).setBuilder( + new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish, + /*notifyFinish=*/null, /*waitForException=*/true, + new StringValue("slow"), /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY); + tester.set(topKey, null); + tester.getOrCreate(topKey).addDependency(midKey).addDependency(errorKey) + .setComputedValue(CONCATENATE); + tester.invalidate(); + // slowKey starts -> errorKey finishes, written to graph -> slowKey finishes & (Visitor aborts) + // -> topKey builds. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertThat(result.getError().getRootCauses()).containsExactly(errorKey); + // Make sure midKey didn't finish building. + assertEquals(null, tester.graph.getExistingValueForTesting(midKey)); + // Give slowKey a nice ordinary builder. + tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null) + .setConstantValue(new StringValue("slow")); + // Put midKey into the graph. It won't have a reverse dependence on topKey. + tester.evalAndGet(/*keepGoing=*/false, midKey); + // topKey should not access midKey as if it were already registered as a dependency. + // We don't invalidate errors, but since topKey wasn't actually written to the graph before, it + // will be rebuilt. + tester.eval(/*keepGoing=*/true, topKey); + } + + @Test + public void continueWithErrorDep() throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + tester.set("after", new StringValue("after")); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE).addDependency("after"); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, parentKey); + assertThat(result.errorMap()).isEmpty(); + assertEquals("recoveredafter", result.get(parentKey).getValue()); + tester.set("after", new StringValue("before")); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/true, parentKey); + assertThat(result.errorMap()).isEmpty(); + assertEquals("recoveredbefore", result.get(parentKey).getValue()); + } + + @Test + public void continueWithErrorDepTurnedGood() throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + tester.set("after", new StringValue("after")); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE).addDependency("after"); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/true, parentKey); + assertThat(result.errorMap()).isEmpty(); + assertEquals("recoveredafter", result.get(parentKey).getValue()); + tester.set(errorKey, new StringValue("reformed")).setHasError(false); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/true, parentKey); + assertThat(result.errorMap()).isEmpty(); + assertEquals("reformedafter", result.get(parentKey).getValue()); + } + + @Test + public void errorDepAlreadyThereThenTurnedGood() throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setHasError(true); + // Prime the graph by putting the error value in it beforehand. + assertThat(tester.evalAndGetError(errorKey).getRootCauses()).containsExactly(errorKey); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, parentKey); + // Request the parent. + assertThat(result.getError(parentKey).getRootCauses()).containsExactly(parentKey).inOrder(); + // Change the error value to no longer throw. + tester.set(errorKey, new StringValue("reformed")).setHasError(false); + tester.getOrCreate(parentKey, /*markAsModified=*/false).setHasError(false) + .setComputedValue(COPY); + tester.differencer.invalidate(ImmutableList.of(errorKey)); + tester.invalidate(); + // Request the parent again. This time it should succeed. + result = tester.eval(/*keepGoing=*/false, parentKey); + assertThat(result.errorMap()).isEmpty(); + assertEquals("reformed", result.get(parentKey).getValue()); + // Confirm that the parent no longer depends on the error transience value -- make it + // unbuildable again, but without invalidating it, and invalidate transient errors. The parent + // should not be rebuilt. + tester.getOrCreate(parentKey, /*markAsModified=*/false).setHasError(true); + tester.invalidateTransientErrors(); + result = tester.eval(/*keepGoing=*/false, parentKey); + assertThat(result.errorMap()).isEmpty(); + assertEquals("reformed", result.get(parentKey).getValue()); + } + + /** + * Regression test for 2014 bug: error transience value is registered before newly requested deps. + * A value requests a child, gets it back immediately, and then throws, causing the error + * transience value to be registered as a dep. The following build, the error is invalidated via + * that child. + */ + @Test + public void doubleDepOnErrorTransienceValue() throws Exception { + initializeTester(); + SkyKey leafKey = GraphTester.toSkyKey("leaf"); + tester.set(leafKey, new StringValue("leaf")); + // Prime the graph by putting leaf in beforehand. + assertEquals(new StringValue("leaf"), tester.evalAndGet(/*keepGoing=*/false, leafKey)); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(leafKey).setHasError(true); + // Build top -- it has an error. + assertThat(tester.evalAndGetError(topKey).getRootCauses()).containsExactly(topKey).inOrder(); + // Invalidate top via leaf, and rebuild. + tester.set(leafKey, new StringValue("leaf2")); + tester.invalidate(); + assertThat(tester.evalAndGetError(topKey).getRootCauses()).containsExactly(topKey).inOrder(); + } + + /** Regression test for crash bug. */ + @Test + public void errorTransienceDepCleared() throws Exception { + initializeTester(); + final SkyKey top = GraphTester.toSkyKey("top"); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + tester.set(leaf, new StringValue("leaf")); + tester.getOrCreate(top).addDependency(leaf).setHasTransientError(true); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top); + assertTrue(result.toString(), result.hasError()); + tester.getOrCreate(leaf, /*markAsModified=*/true); + tester.invalidate(); + SkyKey irrelevant = GraphTester.toSkyKey("irrelevant"); + tester.set(irrelevant, new StringValue("irrelevant")); + tester.eval(/*keepGoing=*/true, irrelevant); + tester.invalidateTransientErrors(); + result = tester.eval(/*keepGoing=*/true, top); + assertTrue(result.toString(), result.hasError()); + } + + @Test + public void incompleteValueAlreadyThereNotUsed() throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(COPY); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(midKey, new StringValue("don't use this")) + .setComputedValue(COPY); + // Prime the graph by evaluating the mid-level value. It shouldn't be stored in the graph + // because + // it was only called during the bubbling-up phase. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, midKey); + assertEquals(null, result.get(midKey)); + assertThat(result.getError().getRootCauses()).containsExactly(errorKey); + // In a keepGoing build, midKey should be re-evaluated. + assertEquals("recovered", + ((StringValue) tester.evalAndGet(/*keepGoing=*/true, parentKey)).getValue()); + } + + /** + * "top" requests a dependency group in which the first value, called "error", throws an + * exception, so "mid" and "mid2", which depend on "slow", never get built. + */ + @Test + public void errorInDependencyGroup() throws Exception { + initializeTester(); + SkyKey topKey = GraphTester.toSkyKey("top"); + CountDownLatch slowStart = new CountDownLatch(1); + CountDownLatch errorFinish = new CountDownLatch(1); + final SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/slowStart, + /*notifyFinish=*/errorFinish, /*waitForException=*/false, + // ChainedFunction throws when value is null. + /*value=*/null, /*deps=*/ImmutableList.<SkyKey>of())); + SkyKey slowKey = GraphTester.toSkyKey("slow"); + tester.getOrCreate(slowKey).setBuilder( + new ChainedFunction(/*notifyStart=*/slowStart, /*waitToFinish=*/errorFinish, + /*notifyFinish=*/null, /*waitForException=*/true, + new StringValue("slow"), /*deps=*/ImmutableList.<SkyKey>of())); + final SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY); + final SkyKey mid2Key = GraphTester.toSkyKey("mid2"); + tester.getOrCreate(mid2Key).addDependency(slowKey).setComputedValue(COPY); + tester.set(topKey, null); + tester.getOrCreate(topKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + env.getValues(ImmutableList.of(errorKey, midKey, mid2Key)); + if (env.valuesMissing()) { + return null; + } + return new StringValue("top"); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + + // Assert that build fails and "error" really is in error. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertTrue(result.hasError()); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey); + + // Ensure that evaluation succeeds if errorKey does not throw an error. + tester.getOrCreate(errorKey).setBuilder(null); + tester.set(errorKey, new StringValue("ok")); + tester.invalidate(); + assertEquals(new StringValue("top"), tester.evalAndGet("top")); + } + + /** + * Regression test -- if value top requests {depA, depB}, depC, with depA and depC there and depB + * absent, and then throws an exception, the stored deps should be depA, depC (in different + * groups), not {depA, depC} (same group). + */ + @Test + public void valueInErrorWithGroups() throws Exception { + initializeTester(); + SkyKey topKey = GraphTester.toSkyKey("top"); + final SkyKey groupDepA = GraphTester.toSkyKey("groupDepA"); + final SkyKey groupDepB = GraphTester.toSkyKey("groupDepB"); + SkyKey depC = GraphTester.toSkyKey("depC"); + tester.set(groupDepA, new StringValue("depC")); + tester.set(groupDepB, new StringValue("")); + tester.getOrCreate(depC).setHasError(true); + tester.getOrCreate(topKey).setBuilder(new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + StringValue val = ((StringValue) env.getValues( + ImmutableList.of(groupDepA, groupDepB)).get(groupDepA)); + if (env.valuesMissing()) { + return null; + } + String nextDep = val.getValue(); + try { + env.getValueOrThrow(GraphTester.toSkyKey(nextDep), SomeErrorException.class); + } catch (SomeErrorException e) { + throw new GenericFunctionException(e, Transience.PERSISTENT); + } + return env.valuesMissing() ? null : new StringValue("top"); + } + }); + + EvaluationResult<StringValue> evaluationResult = tester.eval( + /*keepGoing=*/true, groupDepA, depC); + assertTrue(evaluationResult.hasError()); + assertEquals("depC", evaluationResult.get(groupDepA).getValue()); + assertThat(evaluationResult.getError(depC).getRootCauses()).containsExactly(depC).inOrder(); + evaluationResult = tester.eval(/*keepGoing=*/false, topKey); + assertTrue(evaluationResult.hasError()); + assertThat(evaluationResult.getError(topKey).getRootCauses()).containsExactly(topKey).inOrder(); + + tester.set(groupDepA, new StringValue("groupDepB")); + tester.getOrCreate(depC, /*markAsModified=*/true); + tester.invalidate(); + evaluationResult = tester.eval(/*keepGoing=*/false, topKey); + assertFalse(evaluationResult.toString(), evaluationResult.hasError()); + assertEquals("top", evaluationResult.get(topKey).getValue()); + } + + @Test + public void errorOnlyEmittedOnce() throws Exception { + initializeTester(); + tester.set("x", new StringValue("y")).setWarning("fizzlepop"); + StringValue value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "fizzlepop"); + JunitTestUtils.assertEventCount(1, eventCollector); + + tester.invalidate(); + value = (StringValue) tester.evalAndGet("x"); + assertEquals("y", value.getValue()); + // No new events emitted. + JunitTestUtils.assertEventCount(1, eventCollector); + } + + /** + * We are checking here that we are resilient to a race condition in which a value that is + * checking its children for dirtiness is signaled by all of its children, putting it in a ready + * state, before the thread has terminated. Optionally, one of its children may throw an error, + * shutting down the threadpool. This is similar to + * {@link ParallelEvaluatorTest#slowChildCleanup}: a child about to throw signals its parent and + * the parent's builder restarts itself before the exception is thrown. Here, the signaling + * happens while dirty dependencies are being checked, as opposed to during actual evaluation, but + * the principle is the same. We control the timing by blocking "top"'s registering itself on its + * deps. + */ + private void dirtyChildEnqueuesParentDuringCheckDependencies(final boolean throwError) + throws Exception { + // Value to be built. It will be signaled to rebuild before it has finished checking its deps. + final SkyKey top = GraphTester.toSkyKey("top"); + // Dep that blocks before it acknowledges being added as a dep by top, so the firstKey value has + // time to signal top. + final SkyKey slowAddingDep = GraphTester.toSkyKey("dep"); + // Don't perform any blocking on the first build. + final AtomicBoolean delayTopSignaling = new AtomicBoolean(false); + final CountDownLatch topSignaled = new CountDownLatch(1); + final CountDownLatch topRestartedBuild = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiter = new TrackingAwaiter(); + setGraphForTesting(new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (!delayTopSignaling.get()) { + return; + } + if (key.equals(top) && type == EventType.SIGNAL && order == Order.AFTER) { + // top is signaled by firstKey (since slowAddingDep is blocking), so slowAddingDep is now + // free to acknowledge top as a parent. + topSignaled.countDown(); + return; + } + if (key.equals(slowAddingDep) && type == EventType.ADD_REVERSE_DEP + && context.equals(top) && order == Order.BEFORE) { + // If top is trying to declare a dep on slowAddingDep, wait until firstKey has signaled + // top. Then this add dep will return DONE and top will be signaled, making it ready, so + // it will be enqueued. + trackingAwaiter.awaitLatchAndTrackExceptions(topSignaled, + "first key didn't signal top in time"); + } + } + })); + // Value that is modified on the second build. Its thread won't finish until it signals top, + // which will wait for the signal before it enqueues its next dep. We prevent the thread from + // finishing by having the listener to which it reports its warning block until top's builder + // starts. + final SkyKey firstKey = GraphTester.skyKey("first"); + tester.set(firstKey, new StringValue("biding")); + tester.set(slowAddingDep, new StringValue("dep")); + final AtomicInteger numTopInvocations = new AtomicInteger(0); + tester.getOrCreate(top).setBuilder(new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey key, SkyFunction.Environment env) { + numTopInvocations.incrementAndGet(); + if (delayTopSignaling.get()) { + // The reporter will be given firstKey's warning to emit when it is requested as a dep + // below, if firstKey is already built, so we release the reporter's latch beforehand. + topRestartedBuild.countDown(); + } + // top's builder just requests both deps in a group. + env.getValuesOrThrow(ImmutableList.of(firstKey, slowAddingDep), SomeErrorException.class); + return env.valuesMissing() ? null : new StringValue("top"); + } + }); + reporter = new DelegatingEventHandler(reporter) { + @Override + public void handle(Event e) { + super.handle(e); + if (e.getKind() == EventKind.WARNING) { + if (!throwError) { + trackingAwaiter.awaitLatchAndTrackExceptions(topRestartedBuild, + "top's builder did not start in time"); + } + } + } + }; + // First build : just prime the graph. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top); + assertFalse(result.hasError()); + assertEquals(new StringValue("top"), result.get(top)); + assertEquals(2, numTopInvocations.get()); + // Now dirty the graph, and maybe have firstKey throw an error. + String warningText = "warning text"; + tester.getOrCreate(firstKey, /*markAsModified=*/true).setHasError(throwError) + .setWarning(warningText); + tester.invalidate(); + delayTopSignaling.set(true); + result = tester.eval(/*keepGoing=*/false, top); + trackingAwaiter.assertNoErrors(); + if (throwError) { + assertTrue(result.hasError()); + assertThat(result.keyNames()).isEmpty(); // No successfully evaluated values. + ErrorInfo errorInfo = result.getError(top); + assertThat(errorInfo.getRootCauses()).containsExactly(firstKey); + assertEquals("on the incremental build, top's builder should have only been used in error " + + "bubbling", 3, numTopInvocations.get()); + } else { + assertEquals(new StringValue("top"), result.get(top)); + assertFalse(result.hasError()); + assertEquals("on the incremental build, top's builder should have only been executed once in " + + "normal evaluation", 3, numTopInvocations.get()); + } + JunitTestUtils.assertContainsEvent(eventCollector, warningText); + assertEquals(0, topSignaled.getCount()); + assertEquals(0, topRestartedBuild.getCount()); + } + + @Test + public void dirtyChildEnqueuesParentDuringCheckDependencies_ThrowDoesntEnqueue() + throws Exception { + dirtyChildEnqueuesParentDuringCheckDependencies(/*throwError=*/true); + } + + @Test + public void dirtyChildEnqueuesParentDuringCheckDependencies_NoThrow() throws Exception { + dirtyChildEnqueuesParentDuringCheckDependencies(/*throwError=*/false); + } + + /** + * The same dep is requested in two groups, but its value determines what the other dep in the + * second group is. When it changes, the other dep in the second group should not be requested. + */ + @Test + public void sameDepInTwoGroups() throws Exception { + initializeTester(); + + // leaf4 should not built in the second build. + final SkyKey leaf4 = GraphTester.toSkyKey("leaf4"); + final AtomicBoolean shouldNotBuildLeaf4 = new AtomicBoolean(false); + setGraphForTesting(new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (shouldNotBuildLeaf4.get() && key.equals(leaf4)) { + throw new IllegalStateException("leaf4 should not have been considered this build: " + + type + ", " + order + ", " + context); + } + } + })); + tester.set(leaf4, new StringValue("leaf4")); + + // Create leaf0, leaf1 and leaf2 values with values "leaf2", "leaf3", "leaf4" respectively. + // These will be requested as one dependency group. In the second build, leaf2 will have the + // value "leaf5". + final List<SkyKey> leaves = new ArrayList<>(); + for (int i = 0; i <= 2; i++) { + SkyKey leaf = GraphTester.toSkyKey("leaf" + i); + leaves.add(leaf); + tester.set(leaf, new StringValue("leaf" + (i + 2))); + } + + // Create "top" value. It depends on all leaf values in two overlapping dependency groups. + SkyKey topKey = GraphTester.toSkyKey("top"); + final SkyValue topValue = new StringValue("top"); + tester.getOrCreate(topKey).setBuilder(new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + // Request the first group, [leaf0, leaf1, leaf2]. + // In the first build, it has values ["leaf2", "leaf3", "leaf4"]. + // In the second build it has values ["leaf2", "leaf3", "leaf5"] + Map<SkyKey, SkyValue> values = env.getValues(leaves); + if (env.valuesMissing()) { + return null; + } + + // Request the second group. In the first build it's [leaf2, leaf4]. + // In the second build it's [leaf2, leaf5] + env.getValues(ImmutableList.of(leaves.get(2), + GraphTester.toSkyKey(((StringValue) values.get(leaves.get(2))).getValue()))); + if (env.valuesMissing()) { + return null; + } + + return topValue; + } + }); + + // First build: assert we can evaluate "top". + assertEquals(topValue, tester.evalAndGet(/*keepGoing=*/false, topKey)); + + // Second build: replace "leaf4" by "leaf5" in leaf2's value. Assert leaf4 is not requested. + final SkyKey leaf5 = GraphTester.toSkyKey("leaf5"); + tester.set(leaf5, new StringValue("leaf5")); + tester.set(leaves.get(2), new StringValue("leaf5")); + tester.invalidate(); + shouldNotBuildLeaf4.set(true); + assertEquals(topValue, tester.evalAndGet(/*keepGoing=*/false, topKey)); + } + + @Test + public void dirtyAndChanged() throws Exception { + initializeTester(); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey mid = GraphTester.toSkyKey("mid"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(mid).setComputedValue(COPY); + tester.getOrCreate(mid).addDependency(leaf).setComputedValue(COPY); + tester.set(leaf, new StringValue("leafy")); + // For invalidation. + tester.set("dummy", new StringValue("dummy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("leafy", topValue.getValue()); + tester.set(leaf, new StringValue("crunchy")); + tester.invalidate(); + // For invalidation. + tester.evalAndGet("dummy"); + tester.getOrCreate(mid, /*markAsModified=*/true); + tester.invalidate(); + topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("crunchy", topValue.getValue()); + } + + /** + * Test whether a value that was already marked changed will be incorrectly marked dirty, not + * changed, if another thread tries to mark it just dirty. To exercise this, we need to have a + * race condition where both threads see that the value is not dirty yet, then the "changed" + * thread marks the value changed before the "dirty" thread marks the value dirty. To accomplish + * this, we use a countdown latch to make the "dirty" thread wait until the "changed" thread is + * done, and another countdown latch to make both of them wait until they have both checked if the + * value is currently clean. + */ + @Test + public void dirtyAndChangedValueIsChanged() throws Exception { + final SkyKey parent = GraphTester.toSkyKey("parent"); + final AtomicBoolean blockingEnabled = new AtomicBoolean(false); + final CountDownLatch waitForChanged = new CountDownLatch(1); + // changed thread checks value entry once (to see if it is changed). dirty thread checks twice, + // to see if it is changed, and if it is dirty. + final CountDownLatch threadsStarted = new CountDownLatch(3); + final TrackingAwaiter trackingAwaiter = new TrackingAwaiter(); + setGraphForTesting(new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (!blockingEnabled.get()) { + return; + } + if (!key.equals(parent)) { + return; + } + if (type == EventType.IS_CHANGED && order == Order.BEFORE) { + threadsStarted.countDown(); + } + // Dirtiness only checked by dirty thread. + if (type == EventType.IS_DIRTY && order == Order.BEFORE) { + threadsStarted.countDown(); + } + if (type == EventType.MARK_DIRTY) { + trackingAwaiter.awaitLatchAndTrackExceptions(threadsStarted, + "Both threads did not query if value isChanged in time"); + boolean isChanged = (Boolean) context; + if (order == Order.BEFORE && !isChanged) { + trackingAwaiter.awaitLatchAndTrackExceptions(waitForChanged, + "'changed' thread did not mark value changed in time"); + return; + } + if (order == Order.AFTER && isChanged) { + waitForChanged.countDown(); + } + } + } + })); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + tester.set(leaf, new StringValue("leaf")); + tester.getOrCreate(parent).addDependency(leaf).setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result; + result = tester.eval(/*keepGoing=*/false, parent); + assertEquals("leaf", result.get(parent).getValue()); + // Invalidate leaf, but don't actually change it. It will transitively dirty parent + // concurrently with parent directly dirtying itself. + tester.getOrCreate(leaf, /*markAsModified=*/true); + SkyKey other2 = GraphTester.toSkyKey("other2"); + tester.set(other2, new StringValue("other2")); + // Invalidate parent, actually changing it. + tester.getOrCreate(parent, /*markAsModified=*/true).addDependency(other2); + tester.invalidate(); + blockingEnabled.set(true); + result = tester.eval(/*keepGoing=*/false, parent); + assertEquals("leafother2", result.get(parent).getValue()); + trackingAwaiter.assertNoErrors(); + assertEquals(0, waitForChanged.getCount()); + assertEquals(0, threadsStarted.getCount()); + } + + @Test + public void singleValueDependsOnManyDirtyValues() throws Exception { + initializeTester(); + SkyKey[] values = new SkyKey[TEST_NODE_COUNT]; + StringBuilder expected = new StringBuilder(); + for (int i = 0; i < values.length; i++) { + String valueName = Integer.toString(i); + values[i] = GraphTester.toSkyKey(valueName); + tester.set(values[i], new StringValue(valueName)); + expected.append(valueName); + } + SkyKey topKey = new SkyKey(GraphTester.NODE_TYPE, "top"); + TestFunction value = tester.getOrCreate(topKey) + .setComputedValue(CONCATENATE); + for (int i = 0; i < values.length; i++) { + value.addDependency(values[i]); + } + + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertEquals(new StringValue(expected.toString()), result.get(topKey)); + + for (int j = 0; j < RUNS; j++) { + for (int i = 0; i < values.length; i++) { + tester.getOrCreate(values[i], /*markAsModified=*/true); + } + // This value has an error, but we should never discover it because it is not marked changed + // and all of its dependencies re-evaluate to the same thing. + tester.getOrCreate(topKey, /*markAsModified=*/false).setHasError(true); + tester.invalidate(); + + result = tester.eval(/*keep_going=*/false, topKey); + assertEquals(new StringValue(expected.toString()), result.get(topKey)); + } + } + + /** + * Tests scenario where we have dirty values in the graph, and then one of them is deleted since + * its evaluation did not complete before an error was thrown. Can either test the graph via an + * evaluation of that deleted value, or an invalidation of a child, and can either remove the + * thrown error or throw it again on that evaluation. + */ + private void dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort( + boolean reevaluateMissingValue, boolean removeError) throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.set(errorKey, new StringValue("biding time")); + SkyKey slowKey = GraphTester.toSkyKey("slow"); + tester.set(slowKey, new StringValue("slow")); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY); + SkyKey lastKey = GraphTester.toSkyKey("last"); + tester.set(lastKey, new StringValue("last")); + SkyKey motherKey = GraphTester.toSkyKey("mother"); + tester.getOrCreate(motherKey).addDependency(errorKey) + .addDependency(midKey).addDependency(lastKey).setComputedValue(CONCATENATE); + SkyKey fatherKey = GraphTester.toSkyKey("father"); + tester.getOrCreate(fatherKey).addDependency(errorKey) + .addDependency(midKey).addDependency(lastKey).setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, motherKey, fatherKey); + assertEquals("biding timeslowlast", result.get(motherKey).getValue()); + assertEquals("biding timeslowlast", result.get(fatherKey).getValue()); + tester.set(slowKey, null); + // Each parent depends on errorKey, midKey, lastKey. We keep slowKey waiting until errorKey is + // finished. So there is no way lastKey can be enqueued by either parent. Thus, the parent that + // is cleaned has not interacted with lastKey this build. Still, lastKey's reverse dep on that + // parent should be removed. + CountDownLatch errorFinish = new CountDownLatch(1); + tester.set(errorKey, null); + tester.getOrCreate(errorKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null, + /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null, + /*deps=*/ImmutableList.<SkyKey>of())); + tester.getOrCreate(slowKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/errorFinish, + /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("leaf2"), + /*deps=*/ImmutableList.<SkyKey>of())); + tester.invalidate(); + // errorKey finishes, written to graph -> leafKey maybe starts+finishes & (Visitor aborts) + // -> one of mother or father builds. The other one should be cleaned, and no references to it + // left in the graph. + result = tester.eval(/*keepGoing=*/false, motherKey, fatherKey); + assertTrue(result.hasError()); + // Only one of mother or father should be in the graph. + assertTrue(result.getError(motherKey) + ", " + result.getError(fatherKey), + (result.getError(motherKey) == null) != (result.getError(fatherKey) == null)); + SkyKey parentKey = (reevaluateMissingValue == (result.getError(motherKey) == null)) + ? motherKey : fatherKey; + // Give slowKey a nice ordinary builder. + tester.getOrCreate(slowKey, /*markAsModified=*/false).setBuilder(null) + .setConstantValue(new StringValue("leaf2")); + if (removeError) { + tester.getOrCreate(errorKey, /*markAsModified=*/true).setBuilder(null) + .setConstantValue(new StringValue("reformed")); + } + String lastString = "last"; + if (!reevaluateMissingValue) { + // Mark the last key modified if we're not trying the absent value again. This invalidation + // will test if lastKey still has a reference to the absent value. + lastString = "last2"; + tester.set(lastKey, new StringValue(lastString)); + } + tester.invalidate(); + result = tester.eval(/*keepGoing=*/false, parentKey); + if (removeError) { + assertEquals("reformedleaf2" + lastString, result.get(parentKey).getValue()); + } else { + assertNotNull(result.getError(parentKey)); + } + } + + /** + * The following four tests (dirtyChildrenProperlyRemovedWith*) test the consistency of the graph + * after a failed build in which a dirty value should have been deleted from the graph. The + * consistency is tested via either evaluating the missing value, or the re-evaluating the present + * value, and either clearing the error or keeping it. To evaluate the present value, we + * invalidate the error value to force re-evaluation. Related to bug "skyframe m1: graph may not + * be properly cleaned on interrupt or failure". + */ + @Test + public void dirtyChildrenProperlyRemovedWithInvalidateRemoveError() throws Exception { + dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/false, + /*removeError=*/true); + } + + @Test + public void dirtyChildrenProperlyRemovedWithInvalidateKeepError() throws Exception { + dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/false, + /*removeError=*/false); + } + + @Test + public void dirtyChildrenProperlyRemovedWithReevaluateRemoveError() throws Exception { + dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/true, + /*removeError=*/true); + } + + @Test + public void dirtyChildrenProperlyRemovedWithReevaluateKeepError() throws Exception { + dirtyValueChildrenProperlyRemovedOnEarlyBuildAbort(/*reevaluateMissingValue=*/true, + /*removeError=*/false); + } + + /** + * Regression test: enqueue so many values that some of them won't have started processing, and + * then either interrupt processing or have a child throw an error. In the latter case, this also + * tests that a value that hasn't started processing can still have a child error bubble up to it. + * In both cases, it tests that the graph is properly cleaned of the dirty values and references + * to them. + */ + private void manyDirtyValuesClearChildrenOnFail(boolean interrupt) throws Exception { + SkyKey leafKey = GraphTester.toSkyKey("leaf"); + tester.set(leafKey, new StringValue("leafy")); + SkyKey lastKey = GraphTester.toSkyKey("last"); + tester.set(lastKey, new StringValue("last")); + final List<SkyKey> tops = new ArrayList<>(); + // Request far more top-level values than there are threads, so some of them will block until + // the + // leaf child is enqueued for processing. + for (int i = 0; i < 10000; i++) { + SkyKey topKey = GraphTester.toSkyKey("top" + i); + tester.getOrCreate(topKey).addDependency(leafKey).addDependency(lastKey) + .setComputedValue(CONCATENATE); + tops.add(topKey); + } + tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0])); + final CountDownLatch notifyStart = new CountDownLatch(1); + tester.set(leafKey, null); + if (interrupt) { + // leaf will wait for an interrupt if desired. We cannot use the usual ChainedFunction + // because we need to actually throw the interrupt. + final AtomicBoolean shouldSleep = new AtomicBoolean(true); + tester.getOrCreate(leafKey, /*markAsModified=*/true).setBuilder( + new NoExtractorFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException { + notifyStart.countDown(); + if (shouldSleep.get()) { + // Should be interrupted within 5 seconds. + Thread.sleep(5000); + throw new AssertionError("leaf was not interrupted"); + } + return new StringValue("crunchy"); + } + }); + tester.invalidate(); + TestThread evalThread = new TestThread() { + @Override + public void runTest() { + try { + tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0])); + Assert.fail(); + } catch (InterruptedException e) { + // Expected. + } + } + }; + evalThread.start(); + assertTrue(notifyStart.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + evalThread.interrupt(); + evalThread.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS); + // Free leafKey to compute next time. + shouldSleep.set(false); + } else { + // Non-interrupt case. Just throw an error in the child. + tester.getOrCreate(leafKey, /*markAsModified=*/true).setHasError(true); + tester.invalidate(); + // The error thrown may non-deterministically bubble up to a parent that has not yet started + // processing, but has been enqueued for processing. + tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0])); + tester.getOrCreate(leafKey, /*markAsModified=*/true).setHasError(false); + tester.set(leafKey, new StringValue("crunchy")); + } + // lastKey was not touched during the previous build, but its reverse deps on its parents should + // still be accurate. + tester.set(lastKey, new StringValue("new last")); + tester.invalidate(); + EvaluationResult<StringValue> result = + tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0])); + for (SkyKey topKey : tops) { + assertEquals(topKey.toString(), "crunchynew last", result.get(topKey).getValue()); + } + } + + /** + * Regression test: make sure that if an evaluation fails before a dirty value starts evaluation + * (in particular, before it is reset), the graph remains consistent. + */ + @Test + public void manyDirtyValuesClearChildrenOnError() throws Exception { + manyDirtyValuesClearChildrenOnFail(/*interrupt=*/false); + } + + /** + * Regression test: Make sure that if an evaluation is interrupted before a dirty value starts + * evaluation (in particular, before it is reset), the graph remains consistent. + */ + @Test + public void manyDirtyValuesClearChildrenOnInterrupt() throws Exception { + manyDirtyValuesClearChildrenOnFail(/*interrupt=*/true); + } + + /** + * Regression test for case where the user requests that we delete nodes that are already in the + * queue to be dirtied. We should handle that gracefully and not complain. + */ + @Test + public void deletingDirtyNodes() throws Exception { + final Thread thread = Thread.currentThread(); + final AtomicBoolean interruptInvalidation = new AtomicBoolean(false); + initializeTester(new TrackingInvalidationReceiver() { + private final AtomicBoolean firstInvalidation = new AtomicBoolean(true); + @Override + public void invalidated(SkyValue value, InvalidationState state) { + if (interruptInvalidation.get() && !firstInvalidation.getAndSet(false)) { + thread.interrupt(); + } + super.invalidated(value, state); + } + }); + SkyKey key = null; + // Create a long chain of nodes. Most of them will not actually be dirtied, but the last one to + // be dirtied will enqueue its parent for dirtying, so it will be in the queue for the next run. + for (int i = 0; i < TEST_NODE_COUNT; i++) { + key = GraphTester.toSkyKey("node" + i); + if (i > 0) { + tester.getOrCreate(key).addDependency("node" + (i - 1)).setComputedValue(COPY); + } else { + tester.set(key, new StringValue("node0")); + } + } + // Seed the graph. + assertEquals("node0", ((StringValue) tester.evalAndGet(/*keepGoing=*/false, key)).getValue()); + // Start the dirtying process. + tester.set("node0", new StringValue("new")); + tester.invalidate(); + interruptInvalidation.set(true); + try { + tester.eval(/*keepGoing=*/false, key); + fail(); + } catch (InterruptedException e) { + // Expected. + } + interruptInvalidation.set(false); + // Now delete all the nodes. The node that was going to be dirtied is also deleted, which we + // should handle. + tester.graph.delete(Predicates.<SkyKey>alwaysTrue()); + assertEquals("new", ((StringValue) tester.evalAndGet(/*keepGoing=*/false, key)).getValue()); + } + + @Test + public void changePruning() throws Exception { + initializeTester(); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey mid = GraphTester.toSkyKey("mid"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(mid).setComputedValue(COPY); + tester.getOrCreate(mid).addDependency(leaf).setComputedValue(COPY); + tester.set(leaf, new StringValue("leafy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("leafy", topValue.getValue()); + // Mark leaf changed, but don't actually change it. + tester.getOrCreate(leaf, /*markAsModified=*/true); + // mid will give an error if re-evaluated, but it shouldn't be because it is not marked changed, + // and its dirty child will evaluate to the same element. + tester.getOrCreate(mid, /*markAsModified=*/false).setHasError(true); + tester.invalidate(); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top); + assertFalse(result.hasError()); + topValue = result.get(top); + assertEquals("leafy", topValue.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + } + + @Test + public void changePruningWithDoneValue() throws Exception { + initializeTester(); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey mid = GraphTester.toSkyKey("mid"); + SkyKey top = GraphTester.toSkyKey("top"); + SkyKey suffix = GraphTester.toSkyKey("suffix"); + StringValue suffixValue = new StringValue("suffix"); + tester.set(suffix, suffixValue); + tester.getOrCreate(top).addDependency(mid).addDependency(suffix).setComputedValue(CONCATENATE); + tester.getOrCreate(mid).addDependency(leaf).addDependency(suffix).setComputedValue(CONCATENATE); + SkyValue leafyValue = new StringValue("leafy"); + tester.set(leaf, leafyValue); + StringValue value = (StringValue) tester.evalAndGet("top"); + assertEquals("leafysuffixsuffix", value.getValue()); + // Mark leaf changed, but don't actually change it. + tester.getOrCreate(leaf, /*markAsModified=*/true); + // mid will give an error if re-evaluated, but it shouldn't be because it is not marked changed, + // and its dirty child will evaluate to the same element. + tester.getOrCreate(mid, /*markAsModified=*/false).setHasError(true); + tester.invalidate(); + value = (StringValue) tester.evalAndGet("leaf"); + assertEquals("leafy", value.getValue()); + assertThat(tester.getDirtyValues()).containsExactly(new StringValue("leafysuffix"), + new StringValue("leafysuffixsuffix")); + assertThat(tester.getDeletedValues()).isEmpty(); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top); + assertFalse(result.hasError()); + value = result.get(top); + assertEquals("leafysuffixsuffix", value.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + } + + @Test + public void changedChildChangesDepOfParent() throws Exception { + initializeTester(); + final SkyKey buildFile = GraphTester.toSkyKey("buildFile"); + ValueComputer authorDrink = new ValueComputer() { + @Override + public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) { + String author = ((StringValue) deps.get(buildFile)).getValue(); + StringValue beverage; + switch (author) { + case "hemingway": + beverage = (StringValue) env.getValue(GraphTester.toSkyKey("absinthe")); + break; + case "joyce": + beverage = (StringValue) env.getValue(GraphTester.toSkyKey("whiskey")); + break; + default: + throw new IllegalStateException(author); + } + if (beverage == null) { + return null; + } + return new StringValue(author + " drank " + beverage.getValue()); + } + }; + + tester.set(buildFile, new StringValue("hemingway")); + SkyKey absinthe = GraphTester.toSkyKey("absinthe"); + tester.set(absinthe, new StringValue("absinthe")); + SkyKey whiskey = GraphTester.toSkyKey("whiskey"); + tester.set(whiskey, new StringValue("whiskey")); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(buildFile).setComputedValue(authorDrink); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("hemingway drank absinthe", topValue.getValue()); + tester.set(buildFile, new StringValue("joyce")); + // Don't evaluate absinthe successfully anymore. + tester.getOrCreate(absinthe).setHasError(true); + tester.invalidate(); + topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("joyce drank whiskey", topValue.getValue()); + assertThat(tester.getDirtyValues()).containsExactly(new StringValue("hemingway"), + new StringValue("hemingway drank absinthe")); + assertThat(tester.getDeletedValues()).isEmpty(); + } + + @Test + public void dirtyDepIgnoresChildren() throws Exception { + initializeTester(); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey mid = GraphTester.toSkyKey("mid"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.set(mid, new StringValue("ignore")); + tester.getOrCreate(top).addDependency(mid).setComputedValue(COPY); + tester.getOrCreate(mid).addDependency(leaf); + tester.set(leaf, new StringValue("leafy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("ignore", topValue.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + // Change leaf. + tester.set(leaf, new StringValue("crunchy")); + tester.invalidate(); + topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("ignore", topValue.getValue()); + assertThat(tester.getDirtyValues()).containsExactly(new StringValue("leafy")); + assertThat(tester.getDeletedValues()).isEmpty(); + tester.set(leaf, new StringValue("smushy")); + tester.invalidate(); + topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("ignore", topValue.getValue()); + assertThat(tester.getDirtyValues()).containsExactly(new StringValue("crunchy")); + assertThat(tester.getDeletedValues()).isEmpty(); + } + + private static final SkyFunction INTERRUPT_BUILDER = new SkyFunction() { + + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + throw new InterruptedException(); + } + + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + }; + + /** + * Utility function to induce a graph clean of whatever value is requested, by trying to build + * this value and interrupting the build as soon as this value's function evaluation starts. + */ + private void failBuildAndRemoveValue(final SkyKey value) { + tester.set(value, null); + // Evaluator will think leaf was interrupted because it threw, so it will be cleaned from graph. + tester.getOrCreate(value, /*markAsModified=*/true).setBuilder(INTERRUPT_BUILDER); + tester.invalidate(); + try { + tester.eval(/*keepGoing=*/false, value); + Assert.fail(); + } catch (InterruptedException e) { + // Expected. + } + tester.getOrCreate(value, /*markAsModified=*/false).setBuilder(null); + } + + /** + * Make sure that when a dirty value is building, the fact that a child may no longer exist in the + * graph doesn't cause problems. + */ + @Test + public void dirtyBuildAfterFailedBuild() throws Exception { + initializeTester(); + final SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(leaf).setComputedValue(COPY); + tester.set(leaf, new StringValue("leafy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("leafy", topValue.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + failBuildAndRemoveValue(leaf); + // Leaf should no longer exist in the graph. Check that this doesn't cause problems. + tester.set(leaf, null); + tester.set(leaf, new StringValue("crunchy")); + tester.invalidate(); + topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("crunchy", topValue.getValue()); + } + + /** + * Regression test: error when clearing reverse deps on dirty value about to be rebuilt, because + * child values were deleted and recreated in interim, forgetting they had reverse dep on dirty + * value in the first place. + */ + @Test + public void changedBuildAfterFailedThenSuccessfulBuild() throws Exception { + initializeTester(); + final SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(leaf).setComputedValue(COPY); + tester.set(leaf, new StringValue("leafy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("leafy", topValue.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + failBuildAndRemoveValue(leaf); + tester.set(leaf, new StringValue("crunchy")); + tester.invalidate(); + tester.eval(/*keepGoing=*/false, leaf); + // Leaf no longer has reverse dep on top. Check that this doesn't cause problems, even if the + // top value is evaluated unconditionally. + tester.getOrCreate(top, /*markAsModified=*/true); + tester.invalidate(); + topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("crunchy", topValue.getValue()); + } + + /** + * Regression test: child value that has been deleted since it and its parent were marked dirty no + * longer knows it has a reverse dep on its parent. + * + * <p>Start with: + * <pre> + * top0 ... top1000 + * \ | / + * leaf + * </pre> + * Then fail to build leaf. Now the entry for leaf should have no "memory" that it was ever + * depended on by tops. Now build tops, but fail again. + */ + @Test + public void manyDirtyValuesClearChildrenOnSecondFail() throws Exception { + final SkyKey leafKey = GraphTester.toSkyKey("leaf"); + tester.set(leafKey, new StringValue("leafy")); + SkyKey lastKey = GraphTester.toSkyKey("last"); + tester.set(lastKey, new StringValue("last")); + final List<SkyKey> tops = new ArrayList<>(); + // Request far more top-level values than there are threads, so some of them will block until + // the leaf child is enqueued for processing. + for (int i = 0; i < 10000; i++) { + SkyKey topKey = GraphTester.toSkyKey("top" + i); + tester.getOrCreate(topKey).addDependency(leafKey).addDependency(lastKey) + .setComputedValue(CONCATENATE); + tops.add(topKey); + } + tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0])); + failBuildAndRemoveValue(leafKey); + // Request the tops. Since leaf was deleted from the graph last build, it no longer knows that + // its parents depend on it. When leaf throws, at least one of its parents (hopefully) will not + // have re-informed leaf that the parent depends on it, exposing the bug, since the parent + // should then not try to clean the reverse dep from leaf. + tester.set(leafKey, null); + // Evaluator will think leaf was interrupted because it threw, so it will be cleaned from graph. + tester.getOrCreate(leafKey, /*markAsModified=*/true).setBuilder(INTERRUPT_BUILDER); + tester.invalidate(); + try { + tester.eval(/*keepGoing=*/false, tops.toArray(new SkyKey[0])); + Assert.fail(); + } catch (InterruptedException e) { + // Expected. + } + } + + @Test + public void failedDirtyBuild() throws Exception { + initializeTester(); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addErrorDependency(leaf, new StringValue("recover")) + .setComputedValue(COPY); + tester.set(leaf, new StringValue("leafy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("leafy", topValue.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + // Change leaf. + tester.getOrCreate(leaf, /*markAsModified=*/true).setHasError(true); + tester.getOrCreate(top, /*markAsModified=*/false).setHasError(true); + tester.invalidate(); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top); + assertNull("value should not have completed evaluation", result.get(top)); + assertWithMessage( + "The error thrown by leaf should have been swallowed by the error thrown by top") + .that(result.getError().getRootCauses()).containsExactly(top); + } + + @Test + public void failedDirtyBuildInBuilder() throws Exception { + initializeTester(); + SkyKey leaf = GraphTester.toSkyKey("leaf"); + SkyKey secondError = GraphTester.toSkyKey("secondError"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(leaf) + .addErrorDependency(secondError, new StringValue("recover")).setComputedValue(CONCATENATE); + tester.set(secondError, new StringValue("secondError")).addDependency(leaf); + tester.set(leaf, new StringValue("leafy")); + StringValue topValue = (StringValue) tester.evalAndGet("top"); + assertEquals("leafysecondError", topValue.getValue()); + assertThat(tester.getDirtyValues()).isEmpty(); + assertThat(tester.getDeletedValues()).isEmpty(); + // Invalidate leaf. + tester.getOrCreate(leaf, /*markAsModified=*/true); + tester.set(leaf, new StringValue("crunchy")); + tester.getOrCreate(secondError, /*markAsModified=*/true).setHasError(true); + tester.getOrCreate(top, /*markAsModified=*/false).setHasError(true); + tester.invalidate(); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, top); + assertNull("value should not have completed evaluation", result.get(top)); + assertWithMessage( + "The error thrown by leaf should have been swallowed by the error thrown by top") + .that(result.getError().getRootCauses()).containsExactly(top); + } + + @Test + public void dirtyErrorTransienceValue() throws Exception { + initializeTester(); + SkyKey error = GraphTester.toSkyKey("error"); + tester.getOrCreate(error).setHasError(true); + assertNotNull(tester.evalAndGetError(error)); + tester.invalidateTransientErrors(); + SkyKey secondError = GraphTester.toSkyKey("secondError"); + tester.getOrCreate(secondError).setHasError(true); + // secondError declares a new dependence on ErrorTransienceValue, but not until it has already + // thrown an error. + assertNotNull(tester.evalAndGetError(secondError)); + } + + @Test + public void dirtyDependsOnErrorTurningGood() throws Exception { + initializeTester(); + SkyKey error = GraphTester.toSkyKey("error"); + tester.getOrCreate(error).setHasError(true); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(error).setComputedValue(COPY); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(error); + tester.getOrCreate(error).setHasError(false); + StringValue val = new StringValue("reformed"); + tester.set(error, val); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/false, topKey); + assertEquals(val, result.get(topKey)); + assertFalse(result.hasError()); + } + + /** Regression test for crash bug. */ + @Test + public void dirtyWithOwnErrorDependsOnTransientErrorTurningGood() throws Exception { + initializeTester(); + final SkyKey error = GraphTester.toSkyKey("error"); + tester.getOrCreate(error).setHasTransientError(true); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyFunction errorFunction = new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException, + InterruptedException { + try { + return env.getValueOrThrow(error, SomeErrorException.class); + } catch (SomeErrorException e) { + throw new GenericFunctionException(e, Transience.PERSISTENT); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + }; + tester.getOrCreate(topKey).setBuilder(errorFunction); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + tester.invalidateTransientErrors(); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey); + tester.getOrCreate(error).setHasTransientError(false); + StringValue reformed = new StringValue("reformed"); + tester.set(error, reformed); + tester.getOrCreate(topKey).setBuilder(null).addDependency(error).setComputedValue(COPY); + tester.invalidate(); + tester.invalidateTransientErrors(); + result = tester.eval(/*keepGoing=*/false, topKey); + assertEquals(reformed, result.get(topKey)); + assertFalse(result.hasError()); + } + + /** + * Make sure that when an error is thrown, it is given for handling only to parents that have + * already registered a dependence on the value that threw the error. + * + * <pre> + * topBubbleKey topErrorFirstKey + * | \ / + * midKey errorKey + * | + * slowKey + * </pre> + * + * On the second build, errorKey throws, and the threadpool aborts before midKey finishes. + * topBubbleKey therefore has not yet requested errorKey this build. If errorKey bubbles up to it, + * topBubbleKey must be able to handle that. (The evaluator can deal with this either by not + * allowing errorKey to bubble up to topBubbleKey, or by dealing with that case.) + */ + @Test + public void errorOnlyBubblesToRequestingParents() throws Exception { + // We need control over the order of reverse deps, so use a deterministic graph. + setGraphForTesting(new DeterministicInMemoryGraph()); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.set(errorKey, new StringValue("biding time")); + SkyKey slowKey = GraphTester.toSkyKey("slow"); + tester.set(slowKey, new StringValue("slow")); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(midKey).addDependency(slowKey).setComputedValue(COPY); + SkyKey topErrorFirstKey = GraphTester.toSkyKey("2nd top alphabetically"); + tester.getOrCreate(topErrorFirstKey).addDependency(errorKey).setComputedValue(CONCATENATE); + SkyKey topBubbleKey = GraphTester.toSkyKey("1st top alphabetically"); + tester.getOrCreate(topBubbleKey).addDependency(midKey).addDependency(errorKey) + .setComputedValue(CONCATENATE); + // First error-free evaluation, to put all values in graph. + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, + topErrorFirstKey, topBubbleKey); + assertEquals("biding time", result.get(topErrorFirstKey).getValue()); + assertEquals("slowbiding time", result.get(topBubbleKey).getValue()); + // Set up timing of child values: slowKey waits to finish until errorKey has thrown an + // exception that has been caught by the threadpool. + tester.set(slowKey, null); + CountDownLatch errorFinish = new CountDownLatch(1); + tester.set(errorKey, null); + tester.getOrCreate(errorKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/null, + /*notifyFinish=*/errorFinish, /*waitForException=*/false, /*value=*/null, + /*deps=*/ImmutableList.<SkyKey>of())); + tester.getOrCreate(slowKey).setBuilder( + new ChainedFunction(/*notifyStart=*/null, /*waitToFinish=*/errorFinish, + /*notifyFinish=*/null, /*waitForException=*/true, new StringValue("leaf2"), + /*deps=*/ImmutableList.<SkyKey>of())); + tester.invalidate(); + // errorKey finishes, written to graph -> slowKey maybe starts+finishes & (Visitor aborts) + // -> some top key builds. + result = tester.eval(/*keepGoing=*/false, topErrorFirstKey, topBubbleKey); + assertTrue(result.hasError()); + assertNotNull(result.getError(topErrorFirstKey)); + } + + @Test + public void dirtyWithRecoveryErrorDependsOnErrorTurningGood() throws Exception { + initializeTester(); + final SkyKey error = GraphTester.toSkyKey("error"); + tester.getOrCreate(error).setHasError(true); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyFunction recoveryErrorFunction = new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + try { + env.getValueOrThrow(error, SomeErrorException.class); + } catch (SomeErrorException e) { + throw new GenericFunctionException(e, Transience.PERSISTENT); + } + return null; + } + + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + }; + tester.getOrCreate(topKey).setBuilder(recoveryErrorFunction); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, topKey); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey); + tester.getOrCreate(error).setHasError(false); + StringValue reformed = new StringValue("reformed"); + tester.set(error, reformed); + tester.getOrCreate(topKey).setBuilder(null).addDependency(error).setComputedValue(COPY); + tester.invalidate(); + result = tester.eval(/*keepGoing=*/false, topKey); + assertEquals(reformed, result.get(topKey)); + assertFalse(result.hasError()); + } + + + @Test + public void absentParent() throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.set(errorKey, new StringValue("biding time")); + SkyKey absentParentKey = GraphTester.toSkyKey("absentParent"); + tester.getOrCreate(absentParentKey).addDependency(errorKey).setComputedValue(CONCATENATE); + assertEquals(new StringValue("biding time"), + tester.evalAndGet(/*keepGoing=*/false, absentParentKey)); + tester.getOrCreate(errorKey, /*markAsModified=*/true).setHasError(true); + SkyKey newParent = GraphTester.toSkyKey("newParent"); + tester.getOrCreate(newParent).addDependency(errorKey).setComputedValue(CONCATENATE); + tester.invalidate(); + EvaluationResult<StringValue> result = tester.eval(/*keepGoing=*/false, newParent); + ErrorInfo error = result.getError(newParent); + assertThat(error.getRootCauses()).containsExactly(errorKey); + } + + // Tests that we have a sane implementation of error transience. + @Test + public void errorTransienceBug() throws Exception { + tester.getOrCreate("key").setHasTransientError(true); + assertNotNull(tester.evalAndGetError("key").getException()); + StringValue value = new StringValue("hi"); + tester.getOrCreate("key").setHasTransientError(false).setConstantValue(value); + tester.invalidateTransientErrors(); + assertEquals(value, tester.evalAndGet("key")); + // This works because the version of the ValueEntry for the ErrorTransience value is always + // increased on each InMemoryMemoizingEvaluator#evaluate call. But that's not the only way to + // implement error transience; another valid implementation would be to unconditionally mark + // values depending on the ErrorTransience value as being changed (rather than merely dirtied) + // during invalidation. + } + + @Test + public void transientErrorTurningGoodHasNoError() throws Exception { + initializeTester(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasTransientError(true); + ErrorInfo errorInfo = tester.evalAndGetError(errorKey); + assertNotNull(errorInfo); + assertThat(errorInfo.getRootCauses()).containsExactly(errorKey); + // Re-evaluates to same thing when errors are invalidated + tester.invalidateTransientErrors(); + errorInfo = tester.evalAndGetError(errorKey); + assertNotNull(errorInfo); + StringValue value = new StringValue("reformed"); + assertThat(errorInfo.getRootCauses()).containsExactly(errorKey); + tester.getOrCreate(errorKey, /*markAsModified=*/false).setHasTransientError(false) + .setConstantValue(value); + tester.invalidateTransientErrors(); + StringValue stringValue = (StringValue) tester.evalAndGet(/*keepGoing=*/true, errorKey); + assertSame(stringValue, value); + // Value builder will now throw, but we should never get to it because it isn't dirty. + tester.getOrCreate(errorKey, /*markAsModified=*/false).setHasTransientError(true); + tester.invalidateTransientErrors(); + stringValue = (StringValue) tester.evalAndGet(/*keepGoing=*/true, errorKey); + assertSame(stringValue, value); + } + + @Test + public void deleteInvalidatedValue() throws Exception { + initializeTester(); + SkyKey top = GraphTester.toSkyKey("top"); + SkyKey toDelete = GraphTester.toSkyKey("toDelete"); + // Must be a concatenation -- COPY doesn't actually copy. + tester.getOrCreate(top).addDependency(toDelete).setComputedValue(CONCATENATE); + tester.set(toDelete, new StringValue("toDelete")); + SkyValue value = tester.evalAndGet("top"); + SkyKey forceInvalidation = GraphTester.toSkyKey("forceInvalidation"); + tester.set(forceInvalidation, new StringValue("forceInvalidation")); + tester.getOrCreate(toDelete, /*markAsModified=*/true); + tester.invalidate(); + tester.eval(/*keepGoing=*/false, forceInvalidation); + tester.delete("toDelete"); + WeakReference<SkyValue> ref = new WeakReference<>(value); + value = null; + tester.eval(/*keepGoing=*/false, forceInvalidation); + tester.invalidate(); // So that invalidation receiver doesn't hang on to reference. + GcFinalization.awaitClear(ref); + } + + /** + * General stress/fuzz test of the evaluator with failure. Construct a large graph, and then throw + * exceptions during building at various points. + */ + @Test + public void twoRailLeftRightDependenciesWithFailure() throws Exception { + initializeTester(); + SkyKey[] leftValues = new SkyKey[TEST_NODE_COUNT]; + SkyKey[] rightValues = new SkyKey[TEST_NODE_COUNT]; + for (int i = 0; i < TEST_NODE_COUNT; i++) { + leftValues[i] = GraphTester.toSkyKey("left-" + i); + rightValues[i] = GraphTester.toSkyKey("right-" + i); + if (i == 0) { + tester.getOrCreate(leftValues[i]) + .addDependency("leaf") + .setComputedValue(COPY); + tester.getOrCreate(rightValues[i]) + .addDependency("leaf") + .setComputedValue(COPY); + } else { + tester.getOrCreate(leftValues[i]) + .addDependency(leftValues[i - 1]) + .addDependency(rightValues[i - 1]) + .setComputedValue(new PassThroughSelected(leftValues[i - 1])); + tester.getOrCreate(rightValues[i]) + .addDependency(leftValues[i - 1]) + .addDependency(rightValues[i - 1]) + .setComputedValue(new PassThroughSelected(rightValues[i - 1])); + } + } + tester.set("leaf", new StringValue("leaf")); + + String lastLeft = "left-" + (TEST_NODE_COUNT - 1); + String lastRight = "right-" + (TEST_NODE_COUNT - 1); + + for (int i = 0; i < TESTED_NODES; i++) { + try { + tester.getOrCreate(leftValues[i], /*markAsModified=*/true).setHasError(true); + tester.invalidate(); + EvaluationResult<StringValue> result = tester.eval( + /*keep_going=*/false, lastLeft, lastRight); + assertTrue(result.hasError()); + tester.differencer.invalidate(ImmutableList.of(leftValues[i])); + tester.invalidate(); + result = tester.eval(/*keep_going=*/false, lastLeft, lastRight); + assertTrue(result.hasError()); + tester.getOrCreate(leftValues[i], /*markAsModified=*/true).setHasError(false); + tester.invalidate(); + result = tester.eval(/*keep_going=*/false, lastLeft, lastRight); + assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastLeft))); + assertEquals(new StringValue("leaf"), result.get(toSkyKey(lastRight))); + } catch (Exception e) { + System.err.println("twoRailLeftRightDependenciesWithFailure exception on run " + i); + throw e; + } + } + } + + @Test + public void valueInjection() throws Exception { + SkyKey key = GraphTester.toSkyKey("new_value"); + SkyValue val = new StringValue("val"); + + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("new_value")); + } + + @Test + public void valueInjectionOverExistingEntry() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.getOrCreate(key).setConstantValue(new StringValue("old_val")); + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + } + + @Test + public void valueInjectionOverExistingDirtyEntry() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.getOrCreate(key).setConstantValue(new StringValue("old_val")); + tester.differencer.inject(ImmutableMap.of(key, val)); + tester.eval(/*keepGoing=*/false, new SkyKey[0]); // Create the value. + + tester.differencer.invalidate(ImmutableList.of(key)); + tester.eval(/*keepGoing=*/false, new SkyKey[0]); // Mark value as dirty. + + tester.differencer.inject(ImmutableMap.of(key, val)); + tester.eval(/*keepGoing=*/false, new SkyKey[0]); // Inject again. + assertEquals(val, tester.evalAndGet("value")); + } + + @Test + public void valueInjectionOverExistingEntryMarkedForInvalidation() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.getOrCreate(key).setConstantValue(new StringValue("old_val")); + tester.differencer.invalidate(ImmutableList.of(key)); + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + } + + @Test + public void valueInjectionOverExistingEntryMarkedForDeletion() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.getOrCreate(key).setConstantValue(new StringValue("old_val")); + tester.graph.delete(Predicates.<SkyKey>alwaysTrue()); + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + } + + @Test + public void valueInjectionOverExistingEqualEntryMarkedForInvalidation() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + + tester.differencer.invalidate(ImmutableList.of(key)); + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + } + + @Test + public void valueInjectionOverExistingEqualEntryMarkedForDeletion() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + + tester.graph.delete(Predicates.<SkyKey>alwaysTrue()); + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet("value")); + } + + @Test + public void valueInjectionOverValueWithDeps() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + StringValue prevVal = new StringValue("foo"); + + tester.getOrCreate("other").setConstantValue(prevVal); + tester.getOrCreate(key).addDependency("other").setComputedValue(COPY); + assertEquals(prevVal, tester.evalAndGet("value")); + tester.differencer.inject(ImmutableMap.of(key, val)); + try { + tester.evalAndGet("value"); + Assert.fail("injection over value with deps should have failed"); + } catch (IllegalStateException e) { + assertEquals("existing entry for Type:value has deps: [Type:other]", e.getMessage()); + } + } + + @Test + public void valueInjectionOverEqualValueWithDeps() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.getOrCreate("other").setConstantValue(val); + tester.getOrCreate(key).addDependency("other").setComputedValue(COPY); + assertEquals(val, tester.evalAndGet("value")); + tester.differencer.inject(ImmutableMap.of(key, val)); + try { + tester.evalAndGet("value"); + Assert.fail("injection over value with deps should have failed"); + } catch (IllegalStateException e) { + assertEquals("existing entry for Type:value has deps: [Type:other]", e.getMessage()); + } + } + + @Test + public void valueInjectionOverValueWithErrors() throws Exception { + SkyKey key = GraphTester.toSkyKey("value"); + SkyValue val = new StringValue("val"); + + tester.getOrCreate(key).setHasError(true); + tester.evalAndGetError(key); + + tester.differencer.inject(ImmutableMap.of(key, val)); + assertEquals(val, tester.evalAndGet(false, key)); + } + + @Test + public void valueInjectionInvalidatesReverseDeps() throws Exception { + SkyKey childKey = GraphTester.toSkyKey("child"); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + StringValue oldVal = new StringValue("old_val"); + + tester.getOrCreate(childKey).setConstantValue(oldVal); + tester.getOrCreate(parentKey).addDependency("child").setComputedValue(COPY); + + EvaluationResult<SkyValue> result = tester.eval(false, parentKey); + assertFalse(result.hasError()); + assertEquals(oldVal, result.get(parentKey)); + + SkyValue val = new StringValue("val"); + tester.differencer.inject(ImmutableMap.of(childKey, val)); + assertEquals(val, tester.evalAndGet("child")); + // Injecting a new child should have invalidated the parent. + Assert.assertNull(tester.getExistingValue("parent")); + + tester.eval(false, childKey); + assertEquals(val, tester.getExistingValue("child")); + Assert.assertNull(tester.getExistingValue("parent")); + assertEquals(val, tester.evalAndGet("parent")); + } + + @Test + public void valueInjectionOverExistingEqualEntryDoesNotInvalidate() throws Exception { + SkyKey childKey = GraphTester.toSkyKey("child"); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + SkyValue val = new StringValue("same_val"); + + tester.getOrCreate(parentKey).addDependency("child").setComputedValue(COPY); + tester.getOrCreate(childKey).setConstantValue(new StringValue("same_val")); + assertEquals(val, tester.evalAndGet("parent")); + + tester.differencer.inject(ImmutableMap.of(childKey, val)); + assertEquals(val, tester.getExistingValue("child")); + // Since we are injecting an equal value, the parent should not have been invalidated. + assertEquals(val, tester.getExistingValue("parent")); + } + + @Test + public void valueInjectionInterrupt() throws Exception { + SkyKey key = GraphTester.toSkyKey("key"); + SkyValue val = new StringValue("val"); + + tester.differencer.inject(ImmutableMap.of(key, val)); + Thread.currentThread().interrupt(); + try { + tester.evalAndGet("key"); + fail(); + } catch (InterruptedException expected) { + // Expected. + } + SkyValue newVal = tester.evalAndGet("key"); + assertEquals(val, newVal); + } + + @Test + public void persistentErrorsNotRerun() throws Exception { + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey transientErrorKey = GraphTester.toSkyKey("transientError"); + SkyKey persistentErrorKey1 = GraphTester.toSkyKey("persistentError1"); + SkyKey persistentErrorKey2 = GraphTester.toSkyKey("persistentError2"); + + tester.getOrCreate(topKey) + .addErrorDependency(transientErrorKey, new StringValue("doesn't matter")) + .addErrorDependency(persistentErrorKey1, new StringValue("doesn't matter")) + .setHasError(true); + tester.getOrCreate(persistentErrorKey1).setHasError(true); + tester.getOrCreate(transientErrorKey) + .addErrorDependency(persistentErrorKey2, new StringValue("doesn't matter")) + .setHasTransientError(true); + tester.getOrCreate(persistentErrorKey2).setHasError(true); + + tester.evalAndGetError(topKey); + assertThat(tester.getEnqueuedValues()).containsExactly( + topKey, transientErrorKey, persistentErrorKey1, persistentErrorKey2); + + tester.invalidate(); + tester.invalidateTransientErrors(); + tester.evalAndGetError(topKey); + // TODO(bazel-team): We can do better here once we implement change pruning for errors. + assertThat(tester.getEnqueuedValues()).containsExactly(topKey, transientErrorKey); + } + + @Test + public void cachedChildErrorDepWithSiblingDepOnNoKeepGoingEval() throws Exception { + SkyKey parent1Key = GraphTester.toSkyKey("parent1"); + SkyKey parent2Key = GraphTester.toSkyKey("parent2"); + final SkyKey errorKey = GraphTester.toSkyKey("error"); + final SkyKey otherKey = GraphTester.toSkyKey("other"); + SkyFunction parentBuilder = new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + env.getValue(errorKey); + env.getValue(otherKey); + if (env.valuesMissing()) { + return null; + } + return new StringValue("parent"); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }; + tester.getOrCreate(parent1Key).setBuilder(parentBuilder); + tester.getOrCreate(parent2Key).setBuilder(parentBuilder); + tester.getOrCreate(errorKey).setConstantValue(new StringValue("no error yet")); + tester.getOrCreate(otherKey).setConstantValue(new StringValue("other")); + tester.eval(/*keepGoing=*/true, parent1Key); + tester.eval(/*keepGoing=*/false, parent2Key); + tester.getOrCreate(errorKey, /*markAsModified=*/true).setHasError(true); + tester.invalidate(); + tester.eval(/*keepGoing=*/true, parent1Key); + tester.eval(/*keepGoing=*/false, parent2Key); + } + + private void setGraphForTesting(NotifyingInMemoryGraph notifyingInMemoryGraph) { + InMemoryMemoizingEvaluator memoizingEvaluator = (InMemoryMemoizingEvaluator) tester.graph; + memoizingEvaluator.setGraphForTesting(notifyingInMemoryGraph); + } + + private static final class PassThroughSelected implements ValueComputer { + private final SkyKey key; + + public PassThroughSelected(SkyKey key) { + this.key = key; + } + + @Override + public SkyValue compute(Map<SkyKey, SkyValue> deps, SkyFunction.Environment env) { + return Preconditions.checkNotNull(deps.get(key)); + } + } + + /** + * A graph tester that is specific to the memoizing evaluator, with some convenience methods. + */ + private class MemoizingEvaluatorTester extends GraphTester { + private RecordingDifferencer differencer; + private MemoizingEvaluator graph; + private SequentialBuildDriver driver; + private TrackingInvalidationReceiver invalidationReceiver = new TrackingInvalidationReceiver(); + + public void initialize() { + this.differencer = new RecordingDifferencer(); + this.graph = new InMemoryMemoizingEvaluator( + ImmutableMap.of(NODE_TYPE, createDelegatingFunction()), differencer, + invalidationReceiver, emittedEventState, true); + this.driver = new SequentialBuildDriver(graph); + } + + public void setInvalidationReceiver(TrackingInvalidationReceiver customInvalidationReceiver) { + Preconditions.checkState(graph == null, "graph already initialized"); + invalidationReceiver = customInvalidationReceiver; + } + + public void invalidate() { + differencer.invalidate(getModifiedValues()); + getModifiedValues().clear(); + invalidationReceiver.clear(); + } + + public void invalidateTransientErrors() { + differencer.invalidateTransientErrors(); + } + + public void delete(String key) { + graph.delete(Predicates.equalTo(GraphTester.skyKey(key))); + } + + public void resetPlayedEvents() { + emittedEventState.clear(); + } + + public Set<SkyValue> getDirtyValues() { + return invalidationReceiver.dirty; + } + + public Set<SkyValue> getDeletedValues() { + return invalidationReceiver.deleted; + } + + public Set<SkyKey> getEnqueuedValues() { + return invalidationReceiver.enqueued; + } + + public <T extends SkyValue> EvaluationResult<T> eval( + boolean keepGoing, int numThreads, SkyKey... keys) throws InterruptedException { + assertThat(getModifiedValues()).isEmpty(); + return driver.evaluate(ImmutableList.copyOf(keys), keepGoing, numThreads, reporter); + } + + public <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, SkyKey... keys) + throws InterruptedException { + return eval(keepGoing, 100, keys); + } + + public <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, String... keys) + throws InterruptedException { + return eval(keepGoing, toSkyKeys(keys)); + } + + public SkyValue evalAndGet(boolean keepGoing, String key) + throws InterruptedException { + return evalAndGet(keepGoing, new SkyKey(NODE_TYPE, key)); + } + + public SkyValue evalAndGet(String key) throws InterruptedException { + return evalAndGet(/*keepGoing=*/false, key); + } + + public SkyValue evalAndGet(boolean keepGoing, SkyKey key) + throws InterruptedException { + EvaluationResult<StringValue> evaluationResult = eval(keepGoing, key); + SkyValue result = evaluationResult.get(key); + assertNotNull(evaluationResult.toString(), result); + return result; + } + + public ErrorInfo evalAndGetError(SkyKey key) throws InterruptedException { + EvaluationResult<StringValue> evaluationResult = eval(/*keepGoing=*/true, key); + ErrorInfo result = evaluationResult.getError(key); + assertNotNull(evaluationResult.toString(), result); + return result; + } + + public ErrorInfo evalAndGetError(String key) throws InterruptedException { + return evalAndGetError(new SkyKey(NODE_TYPE, key)); + } + + @Nullable + public SkyValue getExistingValue(String key) { + return graph.getExistingValueForTesting(new SkyKey(NODE_TYPE, key)); + } + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/NodeEntryTest.java b/src/test/java/com/google/devtools/build/skyframe/NodeEntryTest.java new file mode 100644 index 0000000000..378b09712d --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/NodeEntryTest.java @@ -0,0 +1,668 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.util.GroupedList.GroupedListHelper; +import com.google.devtools.build.skyframe.NodeEntry.DependencyState; +import com.google.devtools.build.skyframe.SkyFunctionException.ReifiedSkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; + +/** + * Tests for {@link NodeEntry}. + */ +@RunWith(JUnit4.class) +public class NodeEntryTest { + + private static final SkyFunctionName NODE_TYPE = new SkyFunctionName("Type", false); + private static final NestedSet<TaggedEvents> NO_EVENTS = + NestedSetBuilder.<TaggedEvents>emptySet(Order.STABLE_ORDER); + + private static SkyKey key(String name) { + return new SkyKey(NODE_TYPE, name); + } + + @Test + public void createEntry() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + assertFalse(entry.isDirty()); + assertFalse(entry.isChanged()); + assertThat(entry.getTemporaryDirectDeps()).isEmpty(); + } + + @Test + public void signalEntry() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep1 = key("dep1"); + addTemporaryDirectDep(entry, dep1); + assertFalse(entry.isReady()); + assertTrue(entry.signalDep()); + assertTrue(entry.isReady()); + assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep1); + SkyKey dep2 = key("dep2"); + SkyKey dep3 = key("dep3"); + addTemporaryDirectDep(entry, dep2); + addTemporaryDirectDep(entry, dep3); + assertFalse(entry.isReady()); + assertFalse(entry.signalDep()); + assertFalse(entry.isReady()); + assertTrue(entry.signalDep()); + assertTrue(entry.isReady()); + assertThat(setValue(entry, new SkyValue() {}, + /*errorInfo=*/null, /*graphVersion=*/0L)).isEmpty(); + assertTrue(entry.isDone()); + assertEquals(new IntVersion(0L), entry.getVersion()); + assertThat(entry.getDirectDeps()).containsExactly(dep1, dep2, dep3); + } + + @Test + public void reverseDeps() { + NodeEntry entry = new NodeEntry(); + SkyKey mother = key("mother"); + SkyKey father = key("father"); + assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(mother)); + assertEquals(DependencyState.ADDED_DEP, entry.addReverseDepAndCheckIfDone(null)); + assertEquals(DependencyState.ADDED_DEP, entry.addReverseDepAndCheckIfDone(father)); + assertThat(setValue(entry, new SkyValue() {}, + /*errorInfo=*/null, /*graphVersion=*/0L)).containsExactly(mother, father); + assertThat(entry.getReverseDeps()).containsExactly(mother, father); + assertTrue(entry.isDone()); + entry.removeReverseDep(mother); + assertFalse(Iterables.contains(entry.getReverseDeps(), mother)); + } + + @Test + public void errorValue() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException( + new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT), + key("cause")); + ErrorInfo errorInfo = new ErrorInfo(exception); + assertThat(setValue(entry, /*value=*/null, errorInfo, /*graphVersion=*/0L)).isEmpty(); + assertTrue(entry.isDone()); + assertNull(entry.getValue()); + assertEquals(errorInfo, entry.getErrorInfo()); + } + + @Test + public void errorAndValue() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException( + new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT), + key("cause")); + ErrorInfo errorInfo = new ErrorInfo(exception); + setValue(entry, new SkyValue() {}, errorInfo, /*graphVersion=*/0L); + assertTrue(entry.isDone()); + assertEquals(errorInfo, entry.getErrorInfo()); + } + + @Test + public void crashOnNullErrorAndValue() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + try { + setValue(entry, /*value=*/null, /*errorInfo=*/null, /*graphVersion=*/0L); + fail(); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnTooManySignals() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + try { + entry.signalDep(); + fail(); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnDifferentValue() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + try { + // Value() {} and Value() {} are not .equals(). + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/1L); + fail(); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void dirtyLifecycle() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/false); + assertTrue(entry.isDirty()); + assertFalse(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + assertThat(entry.getTemporaryDirectDeps()).isEmpty(); + SkyKey parent = key("parent"); + entry.addReverseDepAndCheckIfDone(parent); + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep); + assertTrue(entry.isReady()); + assertThat(setValue(entry, new SkyValue() {}, /*errorInfo=*/null, + /*graphVersion=*/1L)).containsExactly(parent); + } + + @Test + public void changedLifecycle() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/true); + assertTrue(entry.isDirty()); + assertTrue(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + SkyKey parent = key("parent"); + entry.addReverseDepAndCheckIfDone(parent); + assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState()); + assertTrue(entry.isReady()); + assertThat(entry.getTemporaryDirectDeps()).isEmpty(); + assertThat(setValue(entry, new SkyValue() {}, /*errorInfo=*/null, + /*graphVersion=*/1L)).containsExactly(parent); + assertEquals(new IntVersion(1L), entry.getVersion()); + } + + @Test + public void markDirtyThenChanged() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + addTemporaryDirectDep(entry, key("dep")); + entry.signalDep(); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/false); + assertTrue(entry.isDirty()); + assertFalse(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + entry.markDirty(/*isChanged=*/true); + assertTrue(entry.isDirty()); + assertTrue(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + } + + + @Test + public void markChangedThenDirty() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + addTemporaryDirectDep(entry, key("dep")); + entry.signalDep(); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/true); + assertTrue(entry.isDirty()); + assertTrue(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + entry.markDirty(/*isChanged=*/false); + assertTrue(entry.isDirty()); + assertTrue(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + } + + @Test + public void crashOnTwiceMarkedChanged() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/true); + try { + entry.markDirty(/*isChanged=*/true); + fail("Cannot mark entry changed twice"); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnTwiceMarkedDirty() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + addTemporaryDirectDep(entry, key("dep")); + entry.signalDep(); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + entry.markDirty(/*isChanged=*/false); + try { + entry.markDirty(/*isChanged=*/false); + fail("Cannot mark entry dirty twice"); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnAddReverseDepTwice() { + NodeEntry entry = new NodeEntry(); + SkyKey parent = key("parent"); + assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(parent)); + try { + entry.addReverseDepAndCheckIfDone(parent); + entry.getReverseDeps(); + fail("Cannot add same dep twice"); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnAddReverseDepTwiceAfterDone() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + SkyKey parent = key("parent"); + assertEquals(DependencyState.DONE, entry.addReverseDepAndCheckIfDone(parent)); + try { + entry.addReverseDepAndCheckIfDone(parent); + // We only check for duplicates when we request all the reverse deps. + entry.getReverseDeps(); + fail("Cannot add same dep twice"); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnAddReverseDepBeforeAfterDone() { + NodeEntry entry = new NodeEntry(); + SkyKey parent = key("parent"); + assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(parent)); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + try { + entry.addReverseDepAndCheckIfDone(parent); + // We only check for duplicates when we request all the reverse deps. + entry.getReverseDeps(); + fail("Cannot add same dep twice"); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void crashOnAddDirtyReverseDep() { + NodeEntry entry = new NodeEntry(); + SkyKey parent = key("parent"); + assertEquals(DependencyState.NEEDS_SCHEDULING, entry.addReverseDepAndCheckIfDone(parent)); + try { + entry.addReverseDepAndCheckIfDone(parent); + // We only check for duplicates when we request all the reverse deps. + entry.getReverseDeps(); + fail("Cannot add same dep twice in one build, even if dirty"); + } catch (IllegalStateException e) { + // Expected. + } + } + + @Test + public void pruneBeforeBuild() { + NodeEntry entry = new NodeEntry(); + SkyKey dep = key("dep"); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/false); + assertTrue(entry.isDirty()); + assertFalse(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + SkyKey parent = key("parent"); + entry.addReverseDepAndCheckIfDone(parent); + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + entry.signalDep(new IntVersion(0L)); + assertEquals(BuildingState.DirtyState.VERIFIED_CLEAN, entry.getDirtyState()); + assertThat(entry.markClean()).containsExactly(parent); + assertTrue(entry.isDone()); + assertEquals(new IntVersion(0L), entry.getVersion()); + } + + private static class IntegerValue implements SkyValue { + private final int value; + + IntegerValue(int value) { + this.value = value; + } + + @Override + public boolean equals(Object that) { + return (that instanceof IntegerValue) && (((IntegerValue) that).value == value); + } + + @Override + public int hashCode() { + return value; + } + } + + @Test + public void pruneAfterBuild() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L); + entry.markDirty(/*isChanged=*/false); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + entry.signalDep(new IntVersion(1L)); + assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState()); + assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep); + setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/1L); + assertTrue(entry.isDone()); + assertEquals(new IntVersion(0L), entry.getVersion()); + } + + + @Test + public void noPruneWhenDetailsChange() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L); + assertFalse(entry.isDirty()); + assertTrue(entry.isDone()); + entry.markDirty(/*isChanged=*/false); + assertTrue(entry.isDirty()); + assertFalse(entry.isChanged()); + assertFalse(entry.isDone()); + assertTrue(entry.isReady()); + SkyKey parent = key("parent"); + entry.addReverseDepAndCheckIfDone(parent); + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + entry.signalDep(new IntVersion(1L)); + assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState()); + assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep); + ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException( + new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT), + key("cause")); + setValue(entry, new IntegerValue(5), new ErrorInfo(exception), + /*graphVersion=*/1L); + assertTrue(entry.isDone()); + assertEquals("Version increments when setValue changes", new IntVersion(1), entry.getVersion()); + } + + @Test + public void pruneErrorValue() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException( + new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT), + key("cause")); + ErrorInfo errorInfo = new ErrorInfo(exception); + setValue(entry, /*value=*/null, errorInfo, /*graphVersion=*/0L); + entry.markDirty(/*isChanged=*/false); + entry.addReverseDepAndCheckIfDone(null); // Restart evaluation. + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + entry.signalDep(new IntVersion(1L)); + assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState()); + assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep); + setValue(entry, /*value=*/null, errorInfo, /*graphVersion=*/1L); + assertTrue(entry.isDone()); + assertEquals(new IntVersion(0L), entry.getVersion()); + } + + @Test + public void getDependencyGroup() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + SkyKey dep2 = key("dep2"); + SkyKey dep3 = key("dep3"); + addTemporaryDirectDeps(entry, dep, dep2); + addTemporaryDirectDep(entry, dep3); + entry.signalDep(); + entry.signalDep(); + entry.signalDep(); + setValue(entry, /*value=*/new IntegerValue(5), null, 0L); + entry.markDirty(/*isChanged=*/false); + entry.addReverseDepAndCheckIfDone(null); // Restart evaluation. + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep, dep2).inOrder(); + addTemporaryDirectDeps(entry, dep, dep2); + entry.signalDep(new IntVersion(0L)); + entry.signalDep(new IntVersion(0L)); + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep3).inOrder(); + } + + @Test + public void maintainDependencyGroupAfterRemoval() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + SkyKey dep2 = key("dep2"); + SkyKey dep3 = key("dep3"); + SkyKey dep4 = key("dep4"); + SkyKey dep5 = key("dep5"); + addTemporaryDirectDeps(entry, dep, dep2, dep3); + addTemporaryDirectDep(entry, dep4); + addTemporaryDirectDep(entry, dep5); + entry.signalDep(); + entry.signalDep(); + // Oops! Evaluation terminated with an error, but we're going to set this entry's value anyway. + entry.removeUnfinishedDeps(ImmutableSet.of(dep2, dep3, dep5)); + ReifiedSkyFunctionException exception = new ReifiedSkyFunctionException( + new GenericFunctionException(new SomeErrorException("oops"), Transience.PERSISTENT), + key("key")); + setValue(entry, null, new ErrorInfo(exception), 0L); + entry.markDirty(/*isChanged=*/false); + entry.addReverseDepAndCheckIfDone(null); // Restart evaluation. + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + entry.signalDep(new IntVersion(0L)); + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep4).inOrder(); + } + + @Test + public void noPruneWhenDepsChange() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + SkyKey dep = key("dep"); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L); + entry.markDirty(/*isChanged=*/false); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(dep).inOrder(); + addTemporaryDirectDep(entry, dep); + assertTrue(entry.signalDep(new IntVersion(1L))); + assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState()); + assertThat(entry.getTemporaryDirectDeps()).containsExactly(dep); + addTemporaryDirectDep(entry, key("dep2")); + assertTrue(entry.signalDep(new IntVersion(1L))); + setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/1L); + assertTrue(entry.isDone()); + assertEquals("Version increments when deps change", new IntVersion(1L), entry.getVersion()); + } + + @Test + public void checkDepsOneByOne() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(null); // Start evaluation. + List<SkyKey> deps = new ArrayList<>(); + for (int ii = 0; ii < 10; ii++) { + SkyKey dep = key(Integer.toString(ii)); + deps.add(dep); + addTemporaryDirectDep(entry, dep); + entry.signalDep(); + } + setValue(entry, new IntegerValue(5), /*errorInfo=*/null, /*graphVersion=*/0L); + entry.markDirty(/*isChanged=*/false); + entry.addReverseDepAndCheckIfDone(null); // Start new evaluation. + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + for (int ii = 0; ii < 10; ii++) { + assertThat(entry.getNextDirtyDirectDeps()).containsExactly(deps.get(ii)).inOrder(); + addTemporaryDirectDep(entry, deps.get(ii)); + assertTrue(entry.signalDep(new IntVersion(0L))); + if (ii < 9) { + assertEquals(BuildingState.DirtyState.CHECK_DEPENDENCIES, entry.getDirtyState()); + } else { + assertEquals(BuildingState.DirtyState.VERIFIED_CLEAN, entry.getDirtyState()); + } + } + } + + @Test + public void signalOnlyNewParents() { + NodeEntry entry = new NodeEntry(); + entry.addReverseDepAndCheckIfDone(key("parent")); + setValue(entry, new SkyValue() {}, /*errorInfo=*/null, /*graphVersion=*/0L); + entry.markDirty(/*isChanged=*/true); + SkyKey newParent = key("new parent"); + entry.addReverseDepAndCheckIfDone(newParent); + assertEquals(BuildingState.DirtyState.REBUILDING, entry.getDirtyState()); + assertThat(setValue(entry, new SkyValue() {}, /*errorInfo=*/null, + /*graphVersion=*/1L)).containsExactly(newParent); + } + + @Test + public void testClone() { + NodeEntry entry = new NodeEntry(); + IntVersion version = new IntVersion(0); + IntegerValue originalValue = new IntegerValue(42); + SkyKey originalChild = key("child"); + addTemporaryDirectDep(entry, originalChild); + entry.signalDep(); + entry.setValue(originalValue, version); + entry.addReverseDepAndCheckIfDone(key("parent1")); + NodeEntry clone1 = entry.cloneNodeEntry(); + entry.addReverseDepAndCheckIfDone(key("parent2")); + NodeEntry clone2 = entry.cloneNodeEntry(); + entry.removeReverseDep(key("parent1")); + entry.removeReverseDep(key("parent2")); + IntegerValue updatedValue = new IntegerValue(52); + clone2.markDirty(true); + clone2.addReverseDepAndCheckIfDone(null); + SkyKey newChild = key("newchild"); + addTemporaryDirectDep(clone2, newChild); + clone2.signalDep(); + clone2.setValue(updatedValue, version.next()); + + assertThat(entry.getVersion()).isEqualTo(version); + assertThat(clone1.getVersion()).isEqualTo(version); + assertThat(clone2.getVersion()).isEqualTo(version.next()); + + assertThat(entry.getValue()).isEqualTo(originalValue); + assertThat(clone1.getValue()).isEqualTo(originalValue); + assertThat(clone2.getValue()).isEqualTo(updatedValue); + + assertThat(entry.getDirectDeps()).containsExactly(originalChild); + assertThat(clone1.getDirectDeps()).containsExactly(originalChild); + assertThat(clone2.getDirectDeps()).containsExactly(newChild); + + assertThat(entry.getReverseDeps()).hasSize(0); + assertThat(clone1.getReverseDeps()).containsExactly(key("parent1")); + assertThat(clone2.getReverseDeps()).containsExactly(key("parent1"), key("parent2")); + } + + private static Set<SkyKey> setValue(NodeEntry entry, SkyValue value, + @Nullable ErrorInfo errorInfo, long graphVersion) { + return entry.setValue(ValueWithMetadata.normal(value, errorInfo, NO_EVENTS), + new IntVersion(graphVersion)); + } + + private static void addTemporaryDirectDep(NodeEntry entry, SkyKey key) { + GroupedListHelper<SkyKey> helper = new GroupedListHelper<>(); + helper.add(key); + entry.addTemporaryDirectDeps(helper); + } + + private static void addTemporaryDirectDeps(NodeEntry entry, SkyKey... keys) { + GroupedListHelper<SkyKey> helper = new GroupedListHelper<>(); + helper.startGroup(); + for (SkyKey key : keys) { + helper.add(key); + } + helper.endGroup(); + entry.addTemporaryDirectDeps(helper); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/NotifyingInMemoryGraph.java b/src/test/java/com/google/devtools/build/skyframe/NotifyingInMemoryGraph.java new file mode 100644 index 0000000000..aa88278e78 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/NotifyingInMemoryGraph.java @@ -0,0 +1,128 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.util.Pair; + +import java.util.Set; + +/** + * Class that allows clients to be notified on each access of the graph. Clients can simply track + * accesses, or they can block to achieve desired synchronization. + */ +public class NotifyingInMemoryGraph extends InMemoryGraph { + private final Listener graphListener; + + public NotifyingInMemoryGraph(Listener graphListener) { + this.graphListener = graphListener; + } + + @Override + public NodeEntry createIfAbsent(SkyKey key) { + graphListener.accept(key, EventType.CREATE_IF_ABSENT, Order.BEFORE, null); + NodeEntry newval = getEntry(key); + NodeEntry oldval = getNodeMap().putIfAbsent(key, newval); + return oldval == null ? newval : oldval; + } + + // Subclasses should override if they wish to subclass NotifyingNodeEntry. + protected NotifyingNodeEntry getEntry(SkyKey key) { + return new NotifyingNodeEntry(key); + } + + /** Receiver to be informed when an event for a given key occurs. */ + public interface Listener { + @ThreadSafe + void accept(SkyKey key, EventType type, Order order, Object context); + + public static Listener NULL_LISTENER = new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) {} + }; + } + + /** + * Graph/value entry events that the receiver can be informed of. When writing tests, feel free to + * add additional events here if needed. + */ + public enum EventType { + CREATE_IF_ABSENT, + ADD_REVERSE_DEP, + SIGNAL, + SET_VALUE, + MARK_DIRTY, + IS_CHANGED, + IS_DIRTY + } + + public enum Order { + BEFORE, + AFTER + } + + protected class NotifyingNodeEntry extends NodeEntry { + private final SkyKey myKey; + + protected NotifyingNodeEntry(SkyKey key) { + myKey = key; + } + + // Note that these methods are not synchronized. Necessary synchronization happens when calling + // the super() methods. + @Override + DependencyState addReverseDepAndCheckIfDone(SkyKey reverseDep) { + graphListener.accept(myKey, EventType.ADD_REVERSE_DEP, Order.BEFORE, reverseDep); + DependencyState result = super.addReverseDepAndCheckIfDone(reverseDep); + graphListener.accept(myKey, EventType.ADD_REVERSE_DEP, Order.AFTER, reverseDep); + return result; + } + + @Override + boolean signalDep(Version childVersion) { + graphListener.accept(myKey, EventType.SIGNAL, Order.BEFORE, childVersion); + boolean result = super.signalDep(childVersion); + graphListener.accept(myKey, EventType.SIGNAL, Order.AFTER, childVersion); + return result; + } + + @Override + public Set<SkyKey> setValue(SkyValue value, Version version) { + graphListener.accept(myKey, EventType.SET_VALUE, Order.BEFORE, value); + Set<SkyKey> result = super.setValue(value, version); + graphListener.accept(myKey, EventType.SET_VALUE, Order.AFTER, value); + return result; + } + + @Override + Pair<? extends Iterable<SkyKey>, ? extends SkyValue> markDirty(boolean isChanged) { + graphListener.accept(myKey, EventType.MARK_DIRTY, Order.BEFORE, isChanged); + Pair<? extends Iterable<SkyKey>, ? extends SkyValue> result = super.markDirty(isChanged); + graphListener.accept(myKey, EventType.MARK_DIRTY, Order.AFTER, isChanged); + return result; + } + + @Override + boolean isChanged() { + graphListener.accept(myKey, EventType.IS_CHANGED, Order.BEFORE, this); + return super.isChanged(); + } + + @Override + public boolean isDirty() { + graphListener.accept(myKey, EventType.IS_DIRTY, Order.BEFORE, this); + return super.isDirty(); + } + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/ParallelEvaluatorTest.java b/src/test/java/com/google/devtools/build/skyframe/ParallelEvaluatorTest.java new file mode 100644 index 0000000000..5394437c62 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/ParallelEvaluatorTest.java @@ -0,0 +1,2260 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.devtools.build.skyframe.GraphTester.CONCATENATE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventCollector; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.OutputFilter.RegexOutputFilter; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.testutil.JunitTestUtils; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.testutil.TestThread; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.skyframe.GraphTester.StringValue; +import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.EventType; +import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Listener; +import com.google.devtools.build.skyframe.NotifyingInMemoryGraph.Order; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; + +/** + * Tests for {@link ParallelEvaluator}. + */ +@RunWith(JUnit4.class) +public class ParallelEvaluatorTest { + protected ProcessableGraph graph; + protected IntVersion graphVersion = new IntVersion(0); + protected GraphTester tester = new GraphTester(); + + private EventCollector eventCollector; + private EventHandler reporter; + + private EvaluationProgressReceiver revalidationReceiver; + + @Before + public void initializeReporter() { + eventCollector = new EventCollector(EventKind.ALL_EVENTS); + reporter = new Reporter(eventCollector); + } + + private ParallelEvaluator makeEvaluator(ProcessableGraph graph, + ImmutableMap<SkyFunctionName, ? extends SkyFunction> builders, boolean keepGoing) { + Version oldGraphVersion = graphVersion; + graphVersion = graphVersion.next(); + return new ParallelEvaluator(graph, oldGraphVersion, + builders, reporter, new MemoizingEvaluator.EmittedEventState(), keepGoing, + 150, revalidationReceiver, new DirtyKeyTrackerImpl()); + } + + /** Convenience method for eval-ing a single value. */ + protected SkyValue eval(boolean keepGoing, SkyKey key) throws InterruptedException { + return eval(keepGoing, ImmutableList.of(key)).get(key); + } + + protected ErrorInfo evalValueInError(SkyKey key) throws InterruptedException { + return eval(true, ImmutableList.of(key)).getError(key); + } + + protected <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, SkyKey... keys) + throws InterruptedException { + return eval(keepGoing, ImmutableList.copyOf(keys)); + } + + protected <T extends SkyValue> EvaluationResult<T> eval(boolean keepGoing, Iterable<SkyKey> keys) + throws InterruptedException { + ParallelEvaluator evaluator = makeEvaluator(graph, + ImmutableMap.of(GraphTester.NODE_TYPE, tester.createDelegatingFunction()), + keepGoing); + return evaluator.eval(keys); + } + + protected GraphTester.TestFunction set(String name, String value) { + return tester.set(name, new StringValue(value)); + } + + @Test + public void smoke() throws Exception { + graph = new InMemoryGraph(); + set("a", "a"); + set("b", "b"); + tester.getOrCreate("ab").addDependency("a").addDependency("b").setComputedValue(CONCATENATE); + StringValue value = (StringValue) eval(false, GraphTester.toSkyKey("ab")); + assertEquals("ab", value.getValue()); + JunitTestUtils.assertNoEvents(eventCollector); + } + + /** + * Test interruption handling when a long-running SkyFunction gets interrupted. + */ + @Test + public void interruptedFunction() throws Exception { + runInterruptionTest(new SkyFunctionFactory() { + @Override + public SkyFunction create(final Semaphore threadStarted, final String[] errorMessage) { + return new SkyFunction() { + @Override + public SkyValue compute(SkyKey key, Environment env) throws InterruptedException { + // Signal the waiting test thread that the evaluator thread has really started. + threadStarted.release(); + + // Simulate a SkyFunction that runs for 10 seconds (this number was chosen arbitrarily). + // The main thread should interrupt it shortly after it got started. + Thread.sleep(10 * 1000); + + // Set an error message to indicate that the expected interruption didn't happen. + // We can't use Assert.fail(String) on an async thread. + errorMessage[0] = "SkyFunction should have been interrupted"; + return null; + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }; + } + }); + } + + /** + * Test interruption handling when the Evaluator is in-between running SkyFunctions. + * + * <p>This is the point in time after a SkyFunction requested a dependency which is not yet built + * so the builder returned null to the Evaluator, and the latter is about to schedule evaluation + * of the missing dependency but gets interrupted before the dependency's SkyFunction could start. + */ + @Test + public void interruptedEvaluatorThread() throws Exception { + runInterruptionTest(new SkyFunctionFactory() { + @Override + public SkyFunction create(final Semaphore threadStarted, final String[] errorMessage) { + return new SkyFunction() { + // No need to synchronize access to this field; we always request just one more + // dependency, so it's only one SkyFunction running at any time. + private int valueIdCounter = 0; + + @Override + public SkyValue compute(SkyKey key, Environment env) { + // Signal the waiting test thread that the Evaluator thread has really started. + threadStarted.release(); + + // Keep the evaluator busy until the test's thread gets scheduled and can + // interrupt the Evaluator's thread. + env.getValue(GraphTester.toSkyKey("a" + valueIdCounter++)); + + // This method never throws InterruptedException, therefore it's the responsibility + // of the Evaluator to detect the interrupt and avoid calling subsequent SkyFunctions. + return null; + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }; + } + }); + } + + private void runPartialResultOnInterruption(boolean buildFastFirst) throws Exception { + graph = new InMemoryGraph(); + // Two runs for fastKey's builder and one for the start of waitKey's builder. + final CountDownLatch allValuesReady = new CountDownLatch(3); + final SkyKey waitKey = GraphTester.toSkyKey("wait"); + final SkyKey fastKey = GraphTester.toSkyKey("fast"); + SkyKey leafKey = GraphTester.toSkyKey("leaf"); + tester.getOrCreate(waitKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException { + allValuesReady.countDown(); + Thread.sleep(10000); + throw new AssertionError("Should have been interrupted"); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + tester.getOrCreate(fastKey).setBuilder(new ChainedFunction(null, null, allValuesReady, false, + new StringValue("fast"), ImmutableList.of(leafKey))); + tester.set(leafKey, new StringValue("leaf")); + if (buildFastFirst) { + eval(/*keepGoing=*/false, fastKey); + } + final Set<SkyKey> receivedValues = Sets.newConcurrentHashSet(); + revalidationReceiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) {} + + @Override + public void enqueueing(SkyKey key) {} + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + receivedValues.add(skyKey); + } + }; + TestThread evalThread = new TestThread() { + @Override + public void runTest() throws Exception { + try { + eval(/*keepGoing=*/true, waitKey, fastKey); + fail(); + } catch (InterruptedException e) { + // Expected. + } + } + }; + evalThread.start(); + assertTrue(allValuesReady.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + evalThread.interrupt(); + evalThread.join(TestUtils.WAIT_TIMEOUT_MILLISECONDS); + assertFalse(evalThread.isAlive()); + if (buildFastFirst) { + // If leafKey was already built, it is not reported to the receiver. + assertThat(receivedValues).containsExactly(fastKey); + } else { + // On first time being built, leafKey is registered too. + assertThat(receivedValues).containsExactly(fastKey, leafKey); + } + } + + @Test + public void partialResultOnInterruption() throws Exception { + runPartialResultOnInterruption(/*buildFastFirst=*/false); + } + + @Test + public void partialCachedResultOnInterruption() throws Exception { + runPartialResultOnInterruption(/*buildFastFirst=*/true); + } + + /** + * Factory for SkyFunctions for interruption testing (see {@link #runInterruptionTest}). + */ + private interface SkyFunctionFactory { + /** + * Creates a SkyFunction suitable for a specific test scenario. + * + * @param threadStarted a latch which the returned SkyFunction must + * {@link Semaphore#release() release} once it started (otherwise the test won't work) + * @param errorMessage a single-element array; the SkyFunction can put a error message in it + * to indicate that an assertion failed (calling {@code fail} from async thread doesn't + * work) + */ + SkyFunction create(final Semaphore threadStarted, final String[] errorMessage); + } + + /** + * Test that we can handle the Evaluator getting interrupted at various points. + * + * <p>This method creates an Evaluator with the specified SkyFunction for GraphTested.NODE_TYPE, + * then starts a thread, requests evaluation and asserts that evaluation started. It then + * interrupts the Evaluator thread and asserts that it acknowledged the interruption. + * + * @param valueBuilderFactory creates a SkyFunction which may or may not handle interruptions + * (depending on the test) + */ + private void runInterruptionTest(SkyFunctionFactory valueBuilderFactory) throws Exception { + final Semaphore threadStarted = new Semaphore(0); + final Semaphore threadInterrupted = new Semaphore(0); + final String[] wasError = new String[] { null }; + final ParallelEvaluator evaluator = makeEvaluator(new InMemoryGraph(), + ImmutableMap.of(GraphTester.NODE_TYPE, valueBuilderFactory.create(threadStarted, wasError)), + false); + + Thread t = new Thread(new Runnable() { + @Override + public void run() { + try { + evaluator.eval(ImmutableList.of(GraphTester.toSkyKey("a"))); + + // There's no real need to set an error here. If the thread is not interrupted then + // threadInterrupted is not released and the test thread will fail to acquire it. + wasError[0] = "evaluation should have been interrupted"; + } catch (InterruptedException e) { + // This is the interrupt we are waiting for. It should come straight from the + // evaluator (more precisely, the AbstractQueueVisitor). + // Signal the waiting test thread that the interrupt was acknowledged. + threadInterrupted.release(); + } + } + }); + + // Start the thread and wait for a semaphore. This ensures that the thread was really started. + t.start(); + assertTrue(threadStarted.tryAcquire(TestUtils.WAIT_TIMEOUT_MILLISECONDS, + TimeUnit.MILLISECONDS)); + + // Interrupt the thread and wait for a semaphore. This ensures that the thread was really + // interrupted and this fact was acknowledged. + t.interrupt(); + assertTrue(threadInterrupted.tryAcquire(TestUtils.WAIT_TIMEOUT_MILLISECONDS, + TimeUnit.MILLISECONDS)); + + // The SkyFunction may have reported an error. + if (wasError[0] != null) { + fail(wasError[0]); + } + + // Wait for the thread to finish. + t.join(TestUtils.WAIT_TIMEOUT_MILLISECONDS); + } + + @Test + public void unrecoverableError() throws Exception { + class CustomRuntimeException extends RuntimeException {} + final CustomRuntimeException expected = new CustomRuntimeException(); + + final SkyFunction builder = new SkyFunction() { + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + throw expected; + } + + @Override + @Nullable + public String extractTag(SkyKey skyKey) { + return null; + } + }; + + final ParallelEvaluator evaluator = makeEvaluator(new InMemoryGraph(), + ImmutableMap.of(GraphTester.NODE_TYPE, builder), + false); + + SkyKey valueToEval = GraphTester.toSkyKey("a"); + try { + evaluator.eval(ImmutableList.of(valueToEval)); + } catch (RuntimeException re) { + assertTrue(re.getMessage() + .contains("Unrecoverable error while evaluating node '" + valueToEval.toString() + "'")); + assertTrue(re.getCause() instanceof CustomRuntimeException); + } + } + + @Test + public void simpleWarning() throws Exception { + graph = new InMemoryGraph(); + set("a", "a").setWarning("warning on 'a'"); + StringValue value = (StringValue) eval(false, GraphTester.toSkyKey("a")); + assertEquals("a", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "warning on 'a'"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + @Test + public void warningMatchesRegex() throws Exception { + graph = new InMemoryGraph(); + ((Reporter) reporter).setOutputFilter(RegexOutputFilter.forRegex("a")); + set("example", "a value").setWarning("warning message"); + SkyKey a = GraphTester.toSkyKey("example"); + tester.getOrCreate(a).setTag("a"); + StringValue value = (StringValue) eval(false, a); + assertEquals("a value", value.getValue()); + JunitTestUtils.assertContainsEvent(eventCollector, "warning message"); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + @Test + public void warningMatchesRegexOnlyTag() throws Exception { + graph = new InMemoryGraph(); + ((Reporter) reporter).setOutputFilter(RegexOutputFilter.forRegex("a")); + set("a", "a value").setWarning("warning on 'a'"); + SkyKey a = GraphTester.toSkyKey("a"); + tester.getOrCreate(a).setTag("b"); + StringValue value = (StringValue) eval(false, a); + assertEquals("a value", value.getValue()); + JunitTestUtils.assertEventCount(0, eventCollector); } + + @Test + public void warningDoesNotMatchRegex() throws Exception { + graph = new InMemoryGraph(); + ((Reporter) reporter).setOutputFilter(RegexOutputFilter.forRegex("b")); + set("a", "a").setWarning("warning on 'a'"); + SkyKey a = GraphTester.toSkyKey("a"); + tester.getOrCreate(a).setTag("a"); + StringValue value = (StringValue) eval(false, a); + assertEquals("a", value.getValue()); + JunitTestUtils.assertEventCount(0, eventCollector); + } + + /** Regression test: events from already-done value not replayed. */ + @Test + public void eventFromDoneChildRecorded() throws Exception { + graph = new InMemoryGraph(); + set("a", "a").setWarning("warning on 'a'"); + SkyKey a = GraphTester.toSkyKey("a"); + SkyKey top = GraphTester.toSkyKey("top"); + tester.getOrCreate(top).addDependency(a).setComputedValue(CONCATENATE); + // Build a so that it is already in the graph. + eval(false, a); + JunitTestUtils.assertEventCount(1, eventCollector); + eventCollector.clear(); + // Build top. The warning from a should be reprinted. + eval(false, top); + JunitTestUtils.assertEventCount(1, eventCollector); + eventCollector.clear(); + // Build top again. The warning should have been stored in the value. + eval(false, top); + JunitTestUtils.assertEventCount(1, eventCollector); + } + + @Test + public void shouldCreateErrorValueWithRootCause() throws Exception { + graph = new InMemoryGraph(); + set("a", "a"); + SkyKey parentErrorKey = GraphTester.toSkyKey("parent"); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(parentErrorKey).addDependency("a").addDependency(errorKey) + .setComputedValue(CONCATENATE); + tester.getOrCreate(errorKey).setHasError(true); + ErrorInfo error = evalValueInError(parentErrorKey); + assertThat(error.getRootCauses()).containsExactly(errorKey); + } + + @Test + public void shouldBuildOneTarget() throws Exception { + graph = new InMemoryGraph(); + set("a", "a"); + set("b", "b"); + SkyKey parentErrorKey = GraphTester.toSkyKey("parent"); + SkyKey errorFreeKey = GraphTester.toSkyKey("ab"); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(parentErrorKey).addDependency(errorKey).addDependency("a") + .setComputedValue(CONCATENATE); + tester.getOrCreate(errorKey).setHasError(true); + tester.getOrCreate(errorFreeKey).addDependency("a").addDependency("b") + .setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(true, parentErrorKey, errorFreeKey); + ErrorInfo error = result.getError(parentErrorKey); + assertThat(error.getRootCauses()).containsExactly(errorKey); + StringValue abValue = result.get(errorFreeKey); + assertEquals("ab", abValue.getValue()); + } + + @Test + public void catastropheHaltsBuild_KeepGoing_KeepEdges() throws Exception { + catastrophicBuild(true, true); + } + + @Test + public void catastropheHaltsBuild_KeepGoing_NoKeepEdges() throws Exception { + catastrophicBuild(true, false); + } + + @Test + public void catastropheInBuild_NoKeepGoing_KeepEdges() throws Exception { + catastrophicBuild(false, true); + } + + private void catastrophicBuild(boolean keepGoing, boolean keepEdges) throws Exception { + graph = new InMemoryGraph(keepEdges); + + SkyKey catastropheKey = GraphTester.toSkyKey("catastrophe"); + SkyKey otherKey = GraphTester.toSkyKey("someKey"); + + tester.getOrCreate(catastropheKey).setBuilder(new SkyFunction() { + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + throw new SkyFunctionException(new SomeErrorException("bad"), + Transience.PERSISTENT) { + @Override + public boolean isCatastrophic() { + return true; + } + }; + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + + tester.getOrCreate(otherKey).setBuilder(new SkyFunction() { + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException { + new CountDownLatch(1).await(); + throw new RuntimeException("can't get here"); + } + + @Nullable + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(catastropheKey).setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(keepGoing, topKey, otherKey); + if (!keepGoing) { + ErrorInfo error = result.getError(topKey); + assertThat(error.getRootCauses()).containsExactly(catastropheKey); + } else { + assertTrue(result.hasError()); + assertThat(result.errorMap()).isEmpty(); + } + } + + @Test + public void parentFailureDoesntAffectChild() throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).setHasError(true); + SkyKey childKey = GraphTester.toSkyKey("child"); + set("child", "onions"); + tester.getOrCreate(parentKey).addDependency(childKey).setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, parentKey, childKey); + // Child is guaranteed to complete successfully before parent can run (and fail), + // since parent depends on it. + StringValue childValue = result.get(childKey); + Assert.assertNotNull(childValue); + assertEquals("onions", childValue.getValue()); + ErrorInfo error = result.getError(parentKey); + Assert.assertNotNull(error); + assertThat(error.getRootCauses()).containsExactly(parentKey); + } + + @Test + public void newParentOfErrorShouldHaveError() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setHasError(true); + ErrorInfo error = evalValueInError(errorKey); + assertThat(error.getRootCauses()).containsExactly(errorKey); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addDependency("error").setComputedValue(CONCATENATE); + error = evalValueInError(parentKey); + assertThat(error.getRootCauses()).containsExactly(errorKey); + } + + @Test + public void errorTwoLevelsDeep() throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setHasError(true); + tester.getOrCreate("mid").addDependency(errorKey).setComputedValue(CONCATENATE); + tester.getOrCreate(parentKey).addDependency("mid").setComputedValue(CONCATENATE); + ErrorInfo error = evalValueInError(parentKey); + assertThat(error.getRootCauses()).containsExactly(errorKey); + } + + /** + * A recreation of BuildViewTest#testHasErrorRaceCondition. Also similar to errorTwoLevelsDeep, + * except here we request multiple toplevel values. + */ + @Test + public void errorPropagationToTopLevelValues() throws Exception { + graph = new InMemoryGraph(); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey badKey = GraphTester.toSkyKey("bad"); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE); + tester.getOrCreate(midKey).addDependency(badKey).setComputedValue(CONCATENATE); + tester.getOrCreate(badKey).setHasError(true); + EvaluationResult<SkyValue> result = eval(/*keepGoing=*/false, topKey, midKey); + assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey); + // Do it again with keepGoing. We should also see an error for the top key this time. + result = eval(/*keepGoing=*/true, topKey, midKey); + assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(badKey); + } + + @Test + public void valueNotUsedInFailFastErrorRecovery() throws Exception { + graph = new InMemoryGraph(); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey recoveryKey = GraphTester.toSkyKey("midRecovery"); + SkyKey badKey = GraphTester.toSkyKey("bad"); + + tester.getOrCreate(topKey).addDependency(recoveryKey).setComputedValue(CONCATENATE); + tester.getOrCreate(recoveryKey).addErrorDependency(badKey, new StringValue("i recovered")) + .setComputedValue(CONCATENATE); + tester.getOrCreate(badKey).setHasError(true); + + EvaluationResult<SkyValue> result = eval(/*keepGoing=*/true, ImmutableList.of(recoveryKey)); + assertThat(result.errorMap()).isEmpty(); + assertTrue(result.hasError()); + assertEquals(new StringValue("i recovered"), result.get(recoveryKey)); + + result = eval(/*keepGoing=*/false, ImmutableList.of(topKey)); + assertTrue(result.hasError()); + assertThat(result.keyNames()).isEmpty(); + assertEquals(1, result.errorMap().size()); + assertNotNull(result.getError(topKey).getException()); + } + + /** + * Regression test: "clearing incomplete values on --keep_going build is racy". + * Tests that if a value is requested on the first (non-keep-going) build and its child throws + * an error, when the second (keep-going) build runs, there is not a race that keeps it as a + * reverse dep of its children. + */ + @Test + public void raceClearingIncompleteValues() throws Exception { + SkyKey topKey = GraphTester.toSkyKey("top"); + final SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey badKey = GraphTester.toSkyKey("bad"); + final AtomicBoolean waitForSecondCall = new AtomicBoolean(false); + final TrackingAwaiter trackingAwaiter = new TrackingAwaiter(); + final CountDownLatch otherThreadWinning = new CountDownLatch(1); + final AtomicReference<Thread> firstThread = new AtomicReference<>(); + graph = new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (!waitForSecondCall.get()) { + return; + } + if (key.equals(midKey)) { + if (type == EventType.CREATE_IF_ABSENT) { + // The first thread to create midKey will not be the first thread to add a reverse dep + // to it. + firstThread.compareAndSet(null, Thread.currentThread()); + return; + } + if (type == EventType.ADD_REVERSE_DEP) { + if (order == Order.BEFORE && Thread.currentThread().equals(firstThread.get())) { + // If this thread created midKey, block until the other thread adds a dep on it. + trackingAwaiter.awaitLatchAndTrackExceptions(otherThreadWinning, + "other thread didn't pass this one"); + } else if (order == Order.AFTER && !Thread.currentThread().equals(firstThread.get())) { + // This thread has added a dep. Allow the other thread to proceed. + otherThreadWinning.countDown(); + } + } + } + } + }); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE); + tester.getOrCreate(midKey).addDependency(badKey).setComputedValue(CONCATENATE); + tester.getOrCreate(badKey).setHasError(true); + EvaluationResult<SkyValue> result = eval(/*keepGoing=*/false, topKey, midKey); + assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey); + waitForSecondCall.set(true); + result = eval(/*keepGoing=*/true, topKey, midKey); + trackingAwaiter.assertNoErrors(); + assertNotNull(firstThread.get()); + assertEquals(0, otherThreadWinning.getCount()); + assertThat(result.getError(midKey).getRootCauses()).containsExactly(badKey); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(badKey); + } + + @Test + public void multipleRootCauses() throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + SkyKey errorKey = GraphTester.toSkyKey("error"); + SkyKey errorKey2 = GraphTester.toSkyKey("error2"); + SkyKey errorKey3 = GraphTester.toSkyKey("error3"); + tester.getOrCreate(errorKey).setHasError(true); + tester.getOrCreate(errorKey2).setHasError(true); + tester.getOrCreate(errorKey3).setHasError(true); + tester.getOrCreate("mid").addDependency(errorKey).addDependency(errorKey2) + .setComputedValue(CONCATENATE); + tester.getOrCreate(parentKey) + .addDependency("mid").addDependency(errorKey2).addDependency(errorKey3) + .setComputedValue(CONCATENATE); + ErrorInfo error = evalValueInError(parentKey); + assertThat(error.getRootCauses()).containsExactly(errorKey, errorKey2, errorKey3); + } + + @Test + public void rootCauseWithNoKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setHasError(true); + tester.getOrCreate("mid").addDependency(errorKey).setComputedValue(CONCATENATE); + tester.getOrCreate(parentKey).addDependency("mid").setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(false, ImmutableList.of(parentKey)); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(parentKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(errorKey); + } + + @Test + public void errorBubblesToParentsOfTopLevelValue() throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + final SkyKey errorKey = GraphTester.toSkyKey("error"); + final CountDownLatch latch = new CountDownLatch(1); + tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, /*waitToFinish=*/latch, null, + false, /*value=*/null, ImmutableList.<SkyKey>of())); + tester.getOrCreate(parentKey).setBuilder(new ChainedFunction(/*notifyStart=*/latch, null, null, + false, new StringValue("unused"), ImmutableList.of(errorKey))); + EvaluationResult<StringValue> result = eval( /*keepGoing=*/false, + ImmutableList.of(parentKey, errorKey)); + assertEquals(result.toString(), 2, result.errorMap().size()); + } + + @Test + public void noKeepGoingAfterKeepGoingFails() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addDependency(errorKey); + ErrorInfo error = evalValueInError(parentKey); + assertThat(error.getRootCauses()).containsExactly(errorKey); + SkyKey[] list = { parentKey }; + EvaluationResult<StringValue> result = eval(false, list); + ErrorInfo errorInfo = result.getError(); + assertEquals(errorKey, Iterables.getOnlyElement(errorInfo.getRootCauses())); + assertEquals(errorKey.toString(), errorInfo.getException().getMessage()); + } + + @Test + public void twoErrors() throws Exception { + graph = new InMemoryGraph(); + SkyKey firstError = GraphTester.toSkyKey("error1"); + SkyKey secondError = GraphTester.toSkyKey("error2"); + CountDownLatch firstStart = new CountDownLatch(1); + CountDownLatch secondStart = new CountDownLatch(1); + tester.getOrCreate(firstError).setBuilder(new ChainedFunction(firstStart, secondStart, + /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null, + ImmutableList.<SkyKey>of())); + tester.getOrCreate(secondError).setBuilder(new ChainedFunction(secondStart, firstStart, + /*notifyFinish=*/null, /*waitForException=*/false, /*value=*/null, + ImmutableList.<SkyKey>of())); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, firstError, secondError); + assertTrue(result.toString(), result.hasError()); + // With keepGoing=false, the eval call will terminate with exactly one error (the first one + // thrown). But the first one thrown here is non-deterministic since we synchronize the + // builders so that they run at roughly the same time. + assertThat(ImmutableSet.of(firstError, secondError)).contains( + Iterables.getOnlyElement(result.errorMap().keySet())); + } + + @Test + public void simpleCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey); + ErrorInfo errorInfo = eval(false, ImmutableList.of(aKey)).getError(); + assertEquals(null, errorInfo.getException()); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertTrue(cycleInfo.getPathToCycle().isEmpty()); + } + + @Test + public void cycleWithHead() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(topKey).addDependency(midKey); + tester.getOrCreate(midKey).addDependency(aKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey); + ErrorInfo errorInfo = eval(false, ImmutableList.of(topKey)).getError(); + assertEquals(null, errorInfo.getException()); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder(); + } + + @Test + public void selfEdgeWithHead() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + tester.getOrCreate(topKey).addDependency(midKey); + tester.getOrCreate(midKey).addDependency(aKey); + tester.getOrCreate(aKey).addDependency(aKey); + ErrorInfo errorInfo = eval(false, ImmutableList.of(topKey)).getError(); + assertEquals(null, errorInfo.getException()); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder(); + } + + @Test + public void cycleWithKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey goodKey = GraphTester.toSkyKey("good"); + StringValue goodValue = new StringValue("good"); + tester.set(goodKey, goodValue); + tester.getOrCreate(topKey).addDependency(midKey); + tester.getOrCreate(midKey).addDependency(aKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey); + EvaluationResult<StringValue> result = eval(true, topKey, goodKey); + assertEquals(goodValue, result.get(goodKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey, midKey).inOrder(); + } + + @Test + public void twoCycles() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + SkyKey dKey = GraphTester.toSkyKey("d"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(aKey).addDependency(cKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey); + tester.getOrCreate(cKey).addDependency(dKey); + tester.getOrCreate(dKey).addDependency(cKey); + EvaluationResult<StringValue> result = eval(false, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + Iterable<CycleInfo> cycles = CycleInfo.prepareCycles(topKey, + ImmutableList.of(new CycleInfo(ImmutableList.of(aKey, bKey)), + new CycleInfo(ImmutableList.of(cKey, dKey)))); + assertThat(cycles).contains(getOnlyElement(errorInfo.getCycleInfo())); + } + + + @Test + public void twoCyclesKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + SkyKey dKey = GraphTester.toSkyKey("d"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(aKey).addDependency(cKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey); + tester.getOrCreate(cKey).addDependency(dKey); + tester.getOrCreate(dKey).addDependency(cKey); + EvaluationResult<StringValue> result = eval(true, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo aCycle = new CycleInfo(ImmutableList.of(topKey), ImmutableList.of(aKey, bKey)); + CycleInfo cCycle = new CycleInfo(ImmutableList.of(topKey), ImmutableList.of(cKey, dKey)); + assertThat(errorInfo.getCycleInfo()).containsExactly(aCycle, cCycle); + } + + @Test + public void triangleBelowHeadCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(aKey); + tester.getOrCreate(aKey).addDependency(bKey).addDependency(cKey); + tester.getOrCreate(bKey).addDependency(cKey); + tester.getOrCreate(cKey).addDependency(topKey); + EvaluationResult<StringValue> result = eval(true, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo topCycle = new CycleInfo(ImmutableList.of(topKey, aKey, cKey)); + assertThat(errorInfo.getCycleInfo()).containsExactly(topCycle); + } + + @Test + public void longCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(aKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(cKey); + tester.getOrCreate(cKey).addDependency(topKey); + EvaluationResult<StringValue> result = eval(true, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo topCycle = new CycleInfo(ImmutableList.of(topKey, aKey, bKey, cKey)); + assertThat(errorInfo.getCycleInfo()).containsExactly(topCycle); + } + + @Test + public void cycleWithTail() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(aKey).addDependency(cKey); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(aKey).addDependency(cKey); + tester.getOrCreate(cKey); + tester.set(cKey, new StringValue("cValue")); + EvaluationResult<StringValue> result = eval(false, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + ErrorInfo errorInfo = result.getError(topKey); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(aKey, bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey).inOrder(); + } + + /** Regression test: "value cannot be ready in a cycle". */ + @Test + public void selfEdgeWithExtraChildrenUnderCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(cKey).addDependency(bKey); + tester.getOrCreate(cKey).addDependency(aKey); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey)); + assertEquals(null, result.get(aKey)); + ErrorInfo errorInfo = result.getError(aKey); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(aKey).inOrder(); + } + + /** Regression test: "value cannot be ready in a cycle". */ + @Test + public void cycleWithExtraChildrenUnderCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + SkyKey dKey = GraphTester.toSkyKey("d"); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(cKey).addDependency(dKey); + tester.getOrCreate(cKey).addDependency(aKey); + tester.getOrCreate(dKey).addDependency(bKey); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey)); + assertEquals(null, result.get(aKey)); + ErrorInfo errorInfo = result.getError(aKey); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(bKey, dKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(aKey).inOrder(); + } + + /** Regression test: "value cannot be ready in a cycle". */ + @Test + public void cycleAboveIndependentCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey bKey = GraphTester.toSkyKey("b"); + SkyKey cKey = GraphTester.toSkyKey("c"); + tester.getOrCreate(aKey).addDependency(bKey); + tester.getOrCreate(bKey).addDependency(cKey); + tester.getOrCreate(cKey).addDependency(aKey).addDependency(bKey); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey)); + assertEquals(null, result.get(aKey)); + assertThat(result.getError(aKey).getCycleInfo()).containsExactly( + new CycleInfo(ImmutableList.of(aKey, bKey, cKey)), + new CycleInfo(ImmutableList.of(aKey), ImmutableList.of(bKey, cKey))); + } + + public void valueAboveCycleAndExceptionReportsException() throws Exception { + graph = new InMemoryGraph(); + SkyKey aKey = GraphTester.toSkyKey("a"); + SkyKey errorKey = GraphTester.toSkyKey("error"); + SkyKey bKey = GraphTester.toSkyKey("b"); + tester.getOrCreate(aKey).addDependency(bKey).addDependency(errorKey); + tester.getOrCreate(bKey).addDependency(bKey); + tester.getOrCreate(errorKey).setHasError(true); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(aKey)); + assertEquals(null, result.get(aKey)); + assertNotNull(result.getError(aKey).getException()); + CycleInfo cycleInfo = Iterables.getOnlyElement(result.getError(aKey).getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(bKey).inOrder(); + assertThat(cycleInfo.getPathToCycle()).containsExactly(aKey).inOrder(); + } + + @Test + public void errorValueStored() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + EvaluationResult<StringValue> result = eval(false, ImmutableList.of(errorKey)); + assertThat(result.keyNames()).isEmpty(); + assertThat(result.errorMap().keySet()).containsExactly(errorKey); + ErrorInfo errorInfo = result.getError(); + assertThat(errorInfo.getRootCauses()).containsExactly(errorKey); + // Update value. But builder won't rebuild it. + tester.getOrCreate(errorKey).setHasError(false); + tester.set(errorKey, new StringValue("no error?")); + result = eval(false, ImmutableList.of(errorKey)); + assertThat(result.keyNames()).isEmpty(); + assertThat(result.errorMap().keySet()).containsExactly(errorKey); + errorInfo = result.getError(); + assertThat(errorInfo.getRootCauses()).containsExactly(errorKey); + } + + /** + * Regression test: "OOM in Skyframe cycle detection". + * We only store the first 20 cycles found below any given root value. + */ + @Test + public void manyCycles() throws Exception { + graph = new InMemoryGraph(); + SkyKey topKey = GraphTester.toSkyKey("top"); + for (int i = 0; i < 100; i++) { + SkyKey dep = GraphTester.toSkyKey(Integer.toString(i)); + tester.getOrCreate(topKey).addDependency(dep); + tester.getOrCreate(dep).addDependency(dep); + } + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + assertManyCycles(result.getError(topKey), topKey, /*selfEdge=*/false); + } + + /** + * Regression test: "OOM in Skyframe cycle detection". + * We filter out multiple paths to a cycle that go through the same child value. + */ + @Test + public void manyPathsToCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey cycleKey = GraphTester.toSkyKey("cycle"); + tester.getOrCreate(topKey).addDependency(midKey); + tester.getOrCreate(cycleKey).addDependency(cycleKey); + for (int i = 0; i < 100; i++) { + SkyKey dep = GraphTester.toSkyKey(Integer.toString(i)); + tester.getOrCreate(midKey).addDependency(dep); + tester.getOrCreate(dep).addDependency(cycleKey); + } + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(topKey)); + assertEquals(null, result.get(topKey)); + CycleInfo cycleInfo = Iterables.getOnlyElement(result.getError(topKey).getCycleInfo()); + assertEquals(1, cycleInfo.getCycle().size()); + assertEquals(3, cycleInfo.getPathToCycle().size()); + assertThat(cycleInfo.getPathToCycle().subList(0, 2)).containsExactly(topKey, midKey).inOrder(); + } + + /** + * Checks that errorInfo has many self-edge cycles, and that one of them is a self-edge of + * topKey, if {@code selfEdge} is true. + */ + private static void assertManyCycles(ErrorInfo errorInfo, SkyKey topKey, boolean selfEdge) { + MoreAsserts.assertGreaterThan(1, Iterables.size(errorInfo.getCycleInfo())); + MoreAsserts.assertLessThan(50, Iterables.size(errorInfo.getCycleInfo())); + boolean foundSelfEdge = false; + for (CycleInfo cycle : errorInfo.getCycleInfo()) { + assertEquals(1, cycle.getCycle().size()); // Self-edge. + if (!Iterables.isEmpty(cycle.getPathToCycle())) { + assertThat(cycle.getPathToCycle()).containsExactly(topKey).inOrder(); + } else { + assertThat(cycle.getCycle()).containsExactly(topKey).inOrder(); + foundSelfEdge = true; + } + } + assertEquals(errorInfo + ", " + topKey, selfEdge, foundSelfEdge); + } + + @Test + public void manyUnprocessedValuesInCycle() throws Exception { + graph = new InMemoryGraph(); + SkyKey lastSelfKey = GraphTester.toSkyKey("lastSelf"); + SkyKey firstSelfKey = GraphTester.toSkyKey("firstSelf"); + SkyKey midSelfKey = GraphTester.toSkyKey("midSelf"); + // We add firstSelf first so that it is processed last in cycle detection (LIFO), meaning that + // none of the dep values have to be cleared from firstSelf. + tester.getOrCreate(firstSelfKey).addDependency(firstSelfKey); + for (int i = 0; i < 100; i++) { + SkyKey firstDep = GraphTester.toSkyKey("first" + i); + SkyKey midDep = GraphTester.toSkyKey("mid" + i); + SkyKey lastDep = GraphTester.toSkyKey("last" + i); + tester.getOrCreate(firstSelfKey).addDependency(firstDep); + tester.getOrCreate(midSelfKey).addDependency(midDep); + tester.getOrCreate(lastSelfKey).addDependency(lastDep); + if (i == 90) { + // Most of the deps will be cleared from midSelf. + tester.getOrCreate(midSelfKey).addDependency(midSelfKey); + } + tester.getOrCreate(firstDep).addDependency(firstDep); + tester.getOrCreate(midDep).addDependency(midDep); + tester.getOrCreate(lastDep).addDependency(lastDep); + } + // All the deps will be cleared from lastSelf. + tester.getOrCreate(lastSelfKey).addDependency(lastSelfKey); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, + ImmutableList.of(lastSelfKey, firstSelfKey, midSelfKey)); + assertWithMessage(result.toString()).that(result.keyNames()).isEmpty(); + assertThat(result.errorMap().keySet()).containsExactly(lastSelfKey, firstSelfKey, midSelfKey); + + // Check lastSelfKey. + ErrorInfo errorInfo = result.getError(lastSelfKey); + assertEquals(errorInfo.toString(), 1, Iterables.size(errorInfo.getCycleInfo())); + CycleInfo cycleInfo = Iterables.getOnlyElement(errorInfo.getCycleInfo()); + assertThat(cycleInfo.getCycle()).containsExactly(lastSelfKey); + assertThat(cycleInfo.getPathToCycle()).isEmpty(); + + // Check firstSelfKey. It should not have discovered its own self-edge, because there were too + // many other values before it in the queue. + assertManyCycles(result.getError(firstSelfKey), firstSelfKey, /*selfEdge=*/false); + + // Check midSelfKey. It should have discovered its own self-edge. + assertManyCycles(result.getError(midSelfKey), midSelfKey, /*selfEdge=*/true); + } + + @Test + public void errorValueStoredWithKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + EvaluationResult<StringValue> result = eval(true, ImmutableList.of(errorKey)); + assertThat(result.keyNames()).isEmpty(); + assertThat(result.errorMap().keySet()).containsExactly(errorKey); + ErrorInfo errorInfo = result.getError(); + assertThat(errorInfo.getRootCauses()).containsExactly(errorKey); + // Update value. But builder won't rebuild it. + tester.getOrCreate(errorKey).setHasError(false); + tester.set(errorKey, new StringValue("no error?")); + result = eval(true, ImmutableList.of(errorKey)); + assertThat(result.keyNames()).isEmpty(); + assertThat(result.errorMap().keySet()).containsExactly(errorKey); + errorInfo = result.getError(); + assertThat(errorInfo.getRootCauses()).containsExactly(errorKey); + } + + @Test + public void continueWithErrorDep() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + tester.set("after", new StringValue("after")); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE).addDependency("after"); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(parentKey)); + assertThat(result.errorMap()).isEmpty(); + assertEquals("recoveredafter", result.get(parentKey).getValue()); + result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey)); + assertThat(result.keyNames()).isEmpty(); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(parentKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(errorKey); + } + + @Test + public void breakWithErrorDep() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + tester.set("after", new StringValue("after")); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE).addDependency("after"); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey)); + assertThat(result.keyNames()).isEmpty(); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(parentKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(errorKey); + result = eval(/*keepGoing=*/true, ImmutableList.of(parentKey)); + assertThat(result.errorMap()).isEmpty(); + assertEquals("recoveredafter", result.get(parentKey).getValue()); + } + + @Test + public void breakWithInterruptibleErrorDep() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE); + // When the error value throws, the propagation will cause an interrupted exception in parent. + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey)); + assertThat(result.keyNames()).isEmpty(); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(parentKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(errorKey); + assertFalse(Thread.interrupted()); + result = eval(/*keepGoing=*/true, ImmutableList.of(parentKey)); + assertThat(result.errorMap()).isEmpty(); + assertEquals("recovered", result.get(parentKey).getValue()); + } + + @Test + public void transformErrorDep() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + SkyKey parentErrorKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setHasError(true); + EvaluationResult<StringValue> result = eval( + /*keepGoing=*/false, ImmutableList.of(parentErrorKey)); + assertThat(result.keyNames()).isEmpty(); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(parentErrorKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(parentErrorKey); + } + + @Test + public void transformErrorDepKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + SkyKey parentErrorKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setHasError(true); + EvaluationResult<StringValue> result = eval( + /*keepGoing=*/true, ImmutableList.of(parentErrorKey)); + assertThat(result.keyNames()).isEmpty(); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(parentErrorKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(parentErrorKey); + } + + @Test + public void transformErrorDepOneLevelDownKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + tester.set("after", new StringValue("after")); + SkyKey parentErrorKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered")); + tester.set(parentErrorKey, new StringValue("parent value")); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(parentErrorKey).addDependency("after") + .setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(topKey)); + assertThat(ImmutableList.<String>copyOf(result.<String>keyNames())).containsExactly("top"); + assertEquals("parent valueafter", result.get(topKey).getValue()); + assertThat(result.errorMap()).isEmpty(); + } + + @Test + public void transformErrorDepOneLevelDownNoKeepGoing() throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + tester.getOrCreate(errorKey).setHasError(true); + tester.set("after", new StringValue("after")); + SkyKey parentErrorKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentErrorKey).addErrorDependency(errorKey, new StringValue("recovered")); + tester.set(parentErrorKey, new StringValue("parent value")); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(parentErrorKey).addDependency("after") + .setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(topKey)); + assertThat(result.keyNames()).isEmpty(); + Map.Entry<SkyKey, ErrorInfo> error = Iterables.getOnlyElement(result.errorMap().entrySet()); + assertEquals(topKey, error.getKey()); + assertThat(error.getValue().getRootCauses()).containsExactly(errorKey); + } + + /** + * Make sure that multiple unfinished children can be cleared from a cycle value. + */ + @Test + public void cycleWithMultipleUnfinishedChildren() throws Exception { + graph = new InMemoryGraph(); + tester = new GraphTester(); + SkyKey cycleKey = GraphTester.toSkyKey("cycle"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey topKey = GraphTester.toSkyKey("top"); + SkyKey selfEdge1 = GraphTester.toSkyKey("selfEdge1"); + SkyKey selfEdge2 = GraphTester.toSkyKey("selfEdge2"); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE); + // selfEdge* come before cycleKey, so cycleKey's path will be checked first (LIFO), and the + // cycle with mid will be detected before the selfEdge* cycles are. + tester.getOrCreate(midKey).addDependency(selfEdge1).addDependency(selfEdge2) + .addDependency(cycleKey) + .setComputedValue(CONCATENATE); + tester.getOrCreate(cycleKey).addDependency(midKey); + tester.getOrCreate(selfEdge1).addDependency(selfEdge1); + tester.getOrCreate(selfEdge2).addDependency(selfEdge2); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableSet.of(topKey)); + assertThat(result.errorMap().keySet()).containsExactly(topKey); + Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo(); + CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey); + assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey); + } + + /** + * Regression test: "value in cycle depends on error". + * The mid value will have two parents -- top and cycle. Error bubbles up from mid to cycle, and + * we should detect cycle. + */ + private void cycleAndErrorInBubbleUp(boolean keepGoing) throws Exception { + graph = new DeterministicInMemoryGraph(); + tester = new GraphTester(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + SkyKey cycleKey = GraphTester.toSkyKey("cycle"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE); + tester.getOrCreate(midKey).addDependency(errorKey).addDependency(cycleKey) + .setComputedValue(CONCATENATE); + + // We need to ensure that cycle value has finished his work, and we have recorded dependencies + CountDownLatch cycleFinish = new CountDownLatch(1); + tester.getOrCreate(cycleKey).setBuilder(new ChainedFunction(null, + null, cycleFinish, false, new StringValue(""), ImmutableSet.<SkyKey>of(midKey))); + tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, cycleFinish, + null, /*waitForException=*/false, null, ImmutableSet.<SkyKey>of())); + + EvaluationResult<StringValue> result = eval(keepGoing, ImmutableSet.of(topKey)); + assertThat(result.errorMap().keySet()).containsExactly(topKey); + Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo(); + if (keepGoing) { + // The error thrown will only be recorded in keep_going mode. + assertThat(result.getError().getRootCauses()).containsExactly(errorKey); + } + assertThat(cycleInfos).isNotEmpty(); + CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey); + assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey); + } + + @Test + public void cycleAndErrorInBubbleUpNoKeepGoing() throws Exception { + cycleAndErrorInBubbleUp(false); + } + + @Test + public void cycleAndErrorInBubbleUpKeepGoing() throws Exception { + cycleAndErrorInBubbleUp(true); + } + + /** + * Regression test: "value in cycle depends on error". + * We add another value that won't finish building before the threadpool shuts down, to check that + * the cycle detection can handle unfinished values. + */ + @Test + public void cycleAndErrorAndOtherInBubbleUp() throws Exception { + graph = new DeterministicInMemoryGraph(); + tester = new GraphTester(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + SkyKey cycleKey = GraphTester.toSkyKey("cycle"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE); + // We should add cycleKey first and errorKey afterwards. Otherwise there is a chance that + // during error propagation cycleKey will not be processed, and we will not detect the cycle. + tester.getOrCreate(midKey).addDependency(errorKey).addDependency(cycleKey) + .setComputedValue(CONCATENATE); + SkyKey otherTop = GraphTester.toSkyKey("otherTop"); + CountDownLatch topStartAndCycleFinish = new CountDownLatch(2); + // In nokeep_going mode, otherTop will wait until the threadpool has received an exception, + // then request its own dep. This guarantees that there is a value that is not finished when + // cycle detection happens. + tester.getOrCreate(otherTop).setBuilder(new ChainedFunction(topStartAndCycleFinish, + new CountDownLatch(0), null, /*waitForException=*/true, new StringValue("never returned"), + ImmutableSet.<SkyKey>of(GraphTester.toSkyKey("dep that never builds")))); + + tester.getOrCreate(cycleKey).setBuilder(new ChainedFunction(null, null, + topStartAndCycleFinish, /*waitForException=*/false, new StringValue(""), + ImmutableSet.<SkyKey>of(midKey))); + // error waits until otherTop starts and cycle finishes, to make sure otherTop will request + // its dep before the threadpool shuts down. + tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, topStartAndCycleFinish, + null, /*waitForException=*/false, null, + ImmutableSet.<SkyKey>of())); + EvaluationResult<StringValue> result = + eval(/*keepGoing=*/false, ImmutableSet.of(topKey, otherTop)); + assertThat(result.errorMap().keySet()).containsExactly(topKey); + Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo(); + assertThat(cycleInfos).isNotEmpty(); + CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey); + assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey); + } + + /** + * Regression test: "value in cycle depends on error". + * Here, we add an additional top-level key in error, just to mix it up. + */ + private void cycleAndErrorAndError(boolean keepGoing) throws Exception { + graph = new DeterministicInMemoryGraph(); + tester = new GraphTester(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + SkyKey cycleKey = GraphTester.toSkyKey("cycle"); + SkyKey midKey = GraphTester.toSkyKey("mid"); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addDependency(midKey).setComputedValue(CONCATENATE); + tester.getOrCreate(midKey).addDependency(errorKey).addDependency(cycleKey) + .setComputedValue(CONCATENATE); + SkyKey otherTop = GraphTester.toSkyKey("otherTop"); + CountDownLatch topStartAndCycleFinish = new CountDownLatch(2); + // In nokeep_going mode, otherTop will wait until the threadpool has received an exception, + // then throw its own exception. This guarantees that its exception will not be the one + // bubbling up, but that there is a top-level value with an exception by the time the bubbling + // up starts. + tester.getOrCreate(otherTop).setBuilder(new ChainedFunction(topStartAndCycleFinish, + new CountDownLatch(0), null, /*waitForException=*/!keepGoing, null, + ImmutableSet.<SkyKey>of())); + // error waits until otherTop starts and cycle finishes, to make sure otherTop will request + // its dep before the threadpool shuts down. + tester.getOrCreate(errorKey).setBuilder(new ChainedFunction(null, topStartAndCycleFinish, + null, /*waitForException=*/false, null, + ImmutableSet.<SkyKey>of())); + tester.getOrCreate(cycleKey).setBuilder(new ChainedFunction(null, null, + topStartAndCycleFinish, /*waitForException=*/false, new StringValue(""), + ImmutableSet.<SkyKey>of(midKey))); + EvaluationResult<StringValue> result = + eval(keepGoing, ImmutableSet.of(topKey, otherTop)); + if (keepGoing) { + assertThat(result.errorMap().keySet()).containsExactly(otherTop, topKey); + assertThat(result.getError(otherTop).getRootCauses()).containsExactly(otherTop); + // The error thrown will only be recorded in keep_going mode. + assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey); + } + Iterable<CycleInfo> cycleInfos = result.getError(topKey).getCycleInfo(); + assertThat(cycleInfos).isNotEmpty(); + CycleInfo cycleInfo = Iterables.getOnlyElement(cycleInfos); + assertThat(cycleInfo.getPathToCycle()).containsExactly(topKey); + assertThat(cycleInfo.getCycle()).containsExactly(midKey, cycleKey); + } + + @Test + public void cycleAndErrorAndErrorNoKeepGoing() throws Exception { + cycleAndErrorAndError(false); + } + + @Test + public void cycleAndErrorAndErrorKeepGoing() throws Exception { + cycleAndErrorAndError(true); + } + + @Test + public void testFunctionCrashTrace() throws Exception { + final SkyFunctionName childType = new SkyFunctionName("child", false); + final SkyFunctionName parentType = new SkyFunctionName("parent", false); + + class ChildFunction implements SkyFunction { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + throw new IllegalStateException("I WANT A PONY!!!"); + } + + @Override public String extractTag(SkyKey skyKey) { return null; } + } + + class ParentFunction implements SkyFunction { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + SkyValue dep = env.getValue(new SkyKey(childType, "billy the kid")); + if (dep == null) { + return null; + } + throw new IllegalStateException(); // Should never get here. + } + + @Override public String extractTag(SkyKey skyKey) { return null; } + } + + ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.of( + childType, new ChildFunction(), + parentType, new ParentFunction()); + ParallelEvaluator evaluator = makeEvaluator(new InMemoryGraph(), + skyFunctions, false); + + try { + evaluator.eval(ImmutableList.of(new SkyKey(parentType, "octodad"))); + fail(); + } catch (RuntimeException e) { + assertEquals("I WANT A PONY!!!", e.getCause().getMessage()); + assertEquals("Unrecoverable error while evaluating node 'child:billy the kid' " + + "(requested by nodes 'parent:octodad')", e.getMessage()); + } + } + + private static class SomeOtherErrorException extends Exception { + public SomeOtherErrorException(String msg) { + super(msg); + } + } + + private void unexpectedErrorDep(boolean keepGoing) throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + final SomeOtherErrorException exception = new SomeOtherErrorException("error exception"); + tester.getOrCreate(errorKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + throw new SkyFunctionException(exception, Transience.PERSISTENT) {}; + } + + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + }); + SkyKey topKey = GraphTester.toSkyKey("top"); + tester.getOrCreate(topKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(keepGoing, ImmutableList.of(topKey)); + assertThat(result.keyNames()).isEmpty(); + assertSame(exception, result.getError(topKey).getException()); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(errorKey); + } + + /** + * This and the following three tests are in response a bug: "Skyframe error propagation model is + * problematic". They ensure that exceptions a child throws that a value does not specify it can + * handle in getValueOrThrow do not cause a crash. + */ + @Test + public void unexpectedErrorDepKeepGoing() throws Exception { + unexpectedErrorDep(true); + } + + @Test + public void unexpectedErrorDepNoKeepGoing() throws Exception { + unexpectedErrorDep(false); + } + + private void unexpectedErrorDepOneLevelDown(final boolean keepGoing) throws Exception { + graph = new InMemoryGraph(); + SkyKey errorKey = GraphTester.toSkyKey("my_error_value"); + final SomeErrorException exception = new SomeErrorException("error exception"); + final SomeErrorException topException = new SomeErrorException("top exception"); + final StringValue topValue = new StringValue("top"); + tester.getOrCreate(errorKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException { + throw new GenericFunctionException(exception, Transience.PERSISTENT); + } + + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + }); + SkyKey topKey = GraphTester.toSkyKey("top"); + final SkyKey parentKey = GraphTester.toSkyKey("parent"); + tester.getOrCreate(parentKey).addDependency(errorKey).setComputedValue(CONCATENATE); + tester.getOrCreate(topKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws GenericFunctionException { + try { + if (env.getValueOrThrow(parentKey, SomeErrorException.class) == null) { + return null; + } + } catch (SomeErrorException e) { + assertEquals(e.toString(), exception, e); + } + if (keepGoing) { + return topValue; + } else { + throw new GenericFunctionException(topException, Transience.PERSISTENT); + } + } + @Override + public String extractTag(SkyKey skyKey) { + throw new UnsupportedOperationException(); + } + }); + tester.getOrCreate(topKey).addErrorDependency(errorKey, new StringValue("recovered")) + .setComputedValue(CONCATENATE); + EvaluationResult<StringValue> result = eval(keepGoing, ImmutableList.of(topKey)); + if (!keepGoing) { + assertThat(result.keyNames()).isEmpty(); + assertEquals(topException, result.getError(topKey).getException()); + assertThat(result.getError(topKey).getRootCauses()).containsExactly(topKey); + assertTrue(result.hasError()); + } else { + // result.hasError() is set to true even if the top-level value returned has recovered from + // an error. + assertTrue(result.hasError()); + assertSame(topValue, result.get(topKey)); + } + } + + @Test + public void unexpectedErrorDepOneLevelDownKeepGoing() throws Exception { + unexpectedErrorDepOneLevelDown(true); + } + + @Test + public void unexpectedErrorDepOneLevelDownNoKeepGoing() throws Exception { + unexpectedErrorDepOneLevelDown(false); + } + + /** + * Exercises various situations involving groups of deps that overlap -- request one group, then + * request another group that has a dep in common with the first group. + * + * @param sameFirst whether the dep in common in the two groups should be the first dep. + * @param twoCalls whether the two groups should be requested in two different builder calls. + * @param valuesOrThrow whether the deps should be requested using getValuesOrThrow. + */ + private void sameDepInTwoGroups(final boolean sameFirst, final boolean twoCalls, + final boolean valuesOrThrow) throws Exception { + graph = new InMemoryGraph(); + SkyKey topKey = GraphTester.toSkyKey("top"); + final List<SkyKey> leaves = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + SkyKey leaf = GraphTester.toSkyKey("leaf" + i); + leaves.add(leaf); + tester.set(leaf, new StringValue("leaf" + i)); + } + final SkyKey leaf4 = GraphTester.toSkyKey("leaf4"); + tester.set(leaf4, new StringValue("leaf" + 4)); + tester.getOrCreate(topKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, + InterruptedException { + if (valuesOrThrow) { + env.getValuesOrThrow(leaves, SomeErrorException.class); + } else { + env.getValues(leaves); + } + if (twoCalls && env.valuesMissing()) { + return null; + } + SkyKey first = sameFirst ? leaves.get(0) : leaf4; + SkyKey second = sameFirst ? leaf4 : leaves.get(2); + List<SkyKey> secondRequest = ImmutableList.of(first, second); + if (valuesOrThrow) { + env.getValuesOrThrow(secondRequest, SomeErrorException.class); + } else { + env.getValues(secondRequest); + } + if (env.valuesMissing()) { + return null; + } + return new StringValue("top"); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + eval(/*keepGoing=*/false, topKey); + assertEquals(new StringValue("top"), eval(/*keepGoing=*/false, topKey)); + } + + @Test + public void sameDepInTwoGroups_Same_Two_Throw() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/true, /*valuesOrThrow=*/true); + } + + @Test + public void sameDepInTwoGroups_Same_Two_Deps() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/true, /*valuesOrThrow=*/false); + } + + @Test + public void sameDepInTwoGroups_Same_One_Throw() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/false, /*valuesOrThrow=*/true); + } + + @Test + public void sameDepInTwoGroups_Same_One_Deps() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/true, /*twoCalls=*/false, /*valuesOrThrow=*/false); + } + + @Test + public void sameDepInTwoGroups_Different_Two_Throw() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/true, /*valuesOrThrow=*/true); + } + + @Test + public void sameDepInTwoGroups_Different_Two_Deps() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/true, /*valuesOrThrow=*/false); + } + + @Test + public void sameDepInTwoGroups_Different_One_Throw() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/false, /*valuesOrThrow=*/true); + } + + @Test + public void sameDepInTwoGroups_Different_One_Deps() throws Exception { + sameDepInTwoGroups(/*sameFirst=*/false, /*twoCalls=*/false, /*valuesOrThrow=*/false); + } + + private void getValuesOrThrowWithErrors(boolean keepGoing) throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + final SkyKey errorDep = GraphTester.toSkyKey("errorChild"); + final SomeErrorException childExn = new SomeErrorException("child error"); + tester.getOrCreate(errorDep).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + throw new GenericFunctionException(childExn, Transience.PERSISTENT); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + final List<SkyKey> deps = new ArrayList<>(); + for (int i = 1; i <= 3; i++) { + SkyKey dep = GraphTester.toSkyKey("child" + i); + deps.add(dep); + tester.set(dep, new StringValue("child" + i)); + } + final SomeErrorException parentExn = new SomeErrorException("parent error"); + tester.getOrCreate(parentKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + try { + SkyValue value = env.getValueOrThrow(errorDep, SomeErrorException.class); + if (value == null) { + return null; + } + } catch (SomeErrorException e) { + // Recover from the child error. + } + env.getValues(deps); + if (env.valuesMissing()) { + return null; + } + throw new GenericFunctionException(parentExn, Transience.PERSISTENT); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + EvaluationResult<StringValue> evaluationResult = eval(keepGoing, ImmutableList.of(parentKey)); + assertTrue(evaluationResult.hasError()); + assertEquals(keepGoing ? parentExn : childExn, evaluationResult.getError().getException()); + } + + @Test + public void getValuesOrThrowWithErrors_NoKeepGoing() throws Exception { + getValuesOrThrowWithErrors(/*keepGoing=*/false); + } + + @Test + public void getValuesOrThrowWithErrors_KeepGoing() throws Exception { + getValuesOrThrowWithErrors(/*keepGoing=*/true); + } + + @Test + public void duplicateCycles() throws Exception { + graph = new InMemoryGraph(); + SkyKey grandparentKey = GraphTester.toSkyKey("grandparent"); + SkyKey parentKey1 = GraphTester.toSkyKey("parent1"); + SkyKey parentKey2 = GraphTester.toSkyKey("parent2"); + SkyKey loopKey1 = GraphTester.toSkyKey("loop1"); + SkyKey loopKey2 = GraphTester.toSkyKey("loop2"); + tester.getOrCreate(loopKey1).addDependency(loopKey2); + tester.getOrCreate(loopKey2).addDependency(loopKey1); + tester.getOrCreate(parentKey1).addDependency(loopKey1); + tester.getOrCreate(parentKey2).addDependency(loopKey2); + tester.getOrCreate(grandparentKey).addDependency(parentKey1); + tester.getOrCreate(grandparentKey).addDependency(parentKey2); + + ErrorInfo errorInfo = evalValueInError(grandparentKey); + List<ImmutableList<SkyKey>> cycles = Lists.newArrayList(); + for (CycleInfo cycleInfo : errorInfo.getCycleInfo()) { + cycles.add(cycleInfo.getCycle()); + } + // Skyframe doesn't automatically dedupe cycles that are the same except for entry point. + assertEquals(2, cycles.size()); + int numUniqueCycles = 0; + CycleDeduper<SkyKey> cycleDeduper = new CycleDeduper<SkyKey>(); + for (ImmutableList<SkyKey> cycle : cycles) { + if (cycleDeduper.seen(cycle)) { + numUniqueCycles++; + } + } + assertEquals(1, numUniqueCycles); + } + + @Test + public void signalValueEnqueuedAndEvaluated() throws Exception { + final Set<SkyKey> enqueuedValues = Sets.newConcurrentHashSet(); + final Set<SkyKey> evaluatedValues = Sets.newConcurrentHashSet(); + EvaluationProgressReceiver progressReceiver = new EvaluationProgressReceiver() { + @Override + public void invalidated(SkyValue value, InvalidationState state) { + throw new IllegalStateException(); + } + + @Override + public void enqueueing(SkyKey skyKey) { + enqueuedValues.add(skyKey); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + evaluatedValues.add(skyKey); + } + }; + + EventHandler reporter = new EventHandler() { + @Override + public void handle(Event e) { + throw new IllegalStateException(); + } + }; + + MemoizingEvaluator aug = new InMemoryMemoizingEvaluator( + ImmutableMap.of(GraphTester.NODE_TYPE, tester.getFunction()), new RecordingDifferencer(), + progressReceiver); + SequentialBuildDriver driver = new SequentialBuildDriver(aug); + + tester.getOrCreate("top1").setComputedValue(CONCATENATE) + .addDependency("d1").addDependency("d2"); + tester.getOrCreate("top2").setComputedValue(CONCATENATE).addDependency("d3"); + tester.getOrCreate("top3"); + assertThat(enqueuedValues).isEmpty(); + assertThat(evaluatedValues).isEmpty(); + + tester.set("d1", new StringValue("1")); + tester.set("d2", new StringValue("2")); + tester.set("d3", new StringValue("3")); + + driver.evaluate(ImmutableList.of(GraphTester.toSkyKey("top1")), false, 200, reporter); + assertThat(enqueuedValues) + .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top1", "d1", "d2"))); + assertThat(evaluatedValues) + .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top1", "d1", "d2"))); + enqueuedValues.clear(); + evaluatedValues.clear(); + + driver.evaluate(ImmutableList.of(GraphTester.toSkyKey("top2")), false, 200, reporter); + assertThat(enqueuedValues) + .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top2", "d3"))); + assertThat(evaluatedValues) + .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top2", "d3"))); + enqueuedValues.clear(); + evaluatedValues.clear(); + + driver.evaluate(ImmutableList.of(GraphTester.toSkyKey("top1")), false, 200, reporter); + assertThat(enqueuedValues).isEmpty(); + assertThat(evaluatedValues) + .containsExactlyElementsIn(Arrays.asList(GraphTester.toSkyKeys("top1"))); + } + + public void runDepOnErrorHaltsNoKeepGoingBuildEagerly(boolean childErrorCached, + final boolean handleChildError) throws Exception { + graph = new InMemoryGraph(); + SkyKey parentKey = GraphTester.toSkyKey("parent"); + final SkyKey childKey = GraphTester.toSkyKey("child"); + tester.getOrCreate(childKey).setHasError(/*hasError=*/true); + // The parent should be built exactly twice: once during normal evaluation and once + // during error bubbling. + final AtomicInteger numParentInvocations = new AtomicInteger(0); + tester.getOrCreate(parentKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + int invocations = numParentInvocations.incrementAndGet(); + if (handleChildError) { + try { + SkyValue value = env.getValueOrThrow(childKey, SomeErrorException.class); + // On the first invocation, either the child error should already be cached and not + // propagated, or it should be computed freshly and not propagated. On the second build + // (error bubbling), the child error should be propagated. + assertTrue("bogus non-null value " + value, value == null); + assertEquals("parent incorrectly re-computed during normal evaluation", 1, invocations); + assertFalse("child error not propagated during error bubbling", + env.inErrorBubblingForTesting()); + return value; + } catch (SomeErrorException e) { + assertTrue("child error propagated during normal evaluation", + env.inErrorBubblingForTesting()); + assertEquals(2, invocations); + return null; + } + } else { + if (invocations == 1) { + assertFalse("parent's first computation should be during normal evaluation", + env.inErrorBubblingForTesting()); + return env.getValue(childKey); + } else { + assertEquals(2, invocations); + assertTrue("parent incorrectly re-computed during normal evaluation", + env.inErrorBubblingForTesting()); + return env.getValue(childKey); + } + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + if (childErrorCached) { + // Ensure that the child is already in the graph. + evalValueInError(childKey); + } + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, ImmutableList.of(parentKey)); + assertEquals(2, numParentInvocations.get()); + assertTrue(result.hasError()); + assertEquals(childKey, result.getError().getRootCauseOfException()); + } + + @Test + public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorCachedAndHandled() + throws Exception { + runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/true, + /*handleChildError=*/true); + } + + @Test + public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorCachedAndNotHandled() + throws Exception { + runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/true, + /*handleChildError=*/false); + } + + @Test + public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorFreshAndHandled() throws Exception { + runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/false, + /*handleChildError=*/true); + } + + @Test + public void depOnErrorHaltsNoKeepGoingBuildEagerly_ChildErrorFreshAndNotHandled() + throws Exception { + runDepOnErrorHaltsNoKeepGoingBuildEagerly(/*childErrorCached=*/false, + /*handleChildError=*/false); + } + + @Test + public void raceConditionWithNoKeepGoingErrors_InflightError() throws Exception { + final CountDownLatch errorCommitted = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiterForError = new TrackingAwaiter(); + final CountDownLatch otherDone = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiterForOther = new TrackingAwaiter(); + final SkyKey errorKey = GraphTester.toSkyKey("errorKey"); + final SkyKey otherKey = GraphTester.toSkyKey("otherKey"); + tester.getOrCreate(errorKey).setHasError(true); + final AtomicInteger numOtherInvocations = new AtomicInteger(0); + tester.getOrCreate(otherKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + int invocations = numOtherInvocations.incrementAndGet(); + if (invocations == 1) { + trackingAwaiterForError.awaitLatchAndTrackExceptions(errorCommitted, + "error didn't get committed to the graph in time"); + } + try { + SkyValue value = env.getValueOrThrow(errorKey, SomeErrorException.class); + assertTrue("bogus non-null value " + value, value == null); + assertEquals(1, invocations); + otherDone.countDown(); + throw new GenericFunctionException(new SomeErrorException("other"), + Transience.PERSISTENT); + } catch (SomeErrorException e) { + assertEquals(2, invocations); + return null; + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + graph = new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (key.equals(errorKey) && type == EventType.SET_VALUE && order == Order.AFTER) { + errorCommitted.countDown(); + trackingAwaiterForOther.awaitLatchAndTrackExceptions(otherDone, + "otherKey's SkyFunction didn't finish in time"); + } + } + }); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, + ImmutableList.of(errorKey, otherKey)); + assertEquals(null, graph.get(otherKey)); + assertTrue(result.hasError()); + assertEquals(errorKey, result.getError().getRootCauseOfException()); + } + + @Test + public void raceConditionWithNoKeepGoingErrors_FutureError() throws Exception { + final CountDownLatch errorCommitted = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiterForError = new TrackingAwaiter(); + final CountDownLatch otherStarted = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiterForOther = new TrackingAwaiter(); + final CountDownLatch otherParentSignaled = new CountDownLatch(1); + final TrackingAwaiter trackingAwaiterForOtherParent = new TrackingAwaiter(); + final SkyKey errorParentKey = GraphTester.toSkyKey("errorParentKey"); + final SkyKey errorKey = GraphTester.toSkyKey("errorKey"); + final SkyKey otherParentKey = GraphTester.toSkyKey("otherParentKey"); + final SkyKey otherKey = GraphTester.toSkyKey("otherKey"); + final AtomicInteger numOtherParentInvocations = new AtomicInteger(0); + final AtomicInteger numErrorParentInvocations = new AtomicInteger(0); + tester.getOrCreate(otherParentKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + int invocations = numOtherParentInvocations.incrementAndGet(); + assertEquals("otherParentKey should not be restarted", 1, invocations); + return env.getValue(otherKey); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + tester.getOrCreate(otherKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + otherStarted.countDown(); + trackingAwaiterForError.awaitLatchAndTrackExceptions(errorCommitted, + "error didn't get committed to the graph in time"); + return new StringValue("other"); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + tester.getOrCreate(errorKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + trackingAwaiterForOther.awaitLatchAndTrackExceptions(otherStarted, + "other didn't start in time"); + throw new GenericFunctionException(new SomeErrorException("error"), + Transience.PERSISTENT); + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + tester.getOrCreate(errorParentKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + int invocations = numErrorParentInvocations.incrementAndGet(); + try { + SkyValue value = env.getValueOrThrow(errorKey, SomeErrorException.class); + assertTrue("bogus non-null value " + value, value == null); + if (invocations == 1) { + return null; + } else { + assertFalse(env.inErrorBubblingForTesting()); + fail("RACE CONDITION: errorParentKey was restarted!"); + return null; + } + } catch (SomeErrorException e) { + assertTrue("child error propagated during normal evaluation", + env.inErrorBubblingForTesting()); + assertEquals(2, invocations); + return null; + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + graph = new NotifyingInMemoryGraph(new Listener() { + @Override + public void accept(SkyKey key, EventType type, Order order, Object context) { + if (key.equals(errorKey) && type == EventType.SET_VALUE && order == Order.AFTER) { + errorCommitted.countDown(); + trackingAwaiterForOtherParent.awaitLatchAndTrackExceptions(otherParentSignaled, + "otherParent didn't get signaled in time"); + // We try to give some time for ParallelEvaluator to incorrectly re-evaluate + // 'otherParentKey'. This test case is testing for a real race condition and the 10ms time + // was chosen experimentally to give a true positive rate of 99.8% (without a sleep it + // has a 1% true positive rate). There's no good way to do this without sleeping. We + // *could* introspect ParallelEvaulator's AbstractQueueVisitor to see if the re-evaluation + // has been enqueued, but that's relying on pretty low-level implementation details. + Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS); + } + if (key.equals(otherParentKey) && type == EventType.SIGNAL && order == Order.AFTER) { + otherParentSignaled.countDown(); + } + } + }); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/false, + ImmutableList.of(otherParentKey, errorParentKey)); + assertTrue(result.hasError()); + assertEquals(errorKey, result.getError().getRootCauseOfException()); + } + + @Test + public void cachedErrorsFromKeepGoingUsedOnNoKeepGoing() throws Exception { + graph = new DeterministicInMemoryGraph(); + tester = new GraphTester(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + SkyKey parent1Key = GraphTester.toSkyKey("parent1"); + SkyKey parent2Key = GraphTester.toSkyKey("parent2"); + tester.getOrCreate(parent1Key).addDependency(errorKey).setConstantValue( + new StringValue("parent1")); + tester.getOrCreate(parent2Key).addDependency(errorKey).setConstantValue( + new StringValue("parent2")); + tester.getOrCreate(errorKey).setHasError(true); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(parent1Key)); + assertTrue(result.hasError()); + assertEquals(errorKey, result.getError().getRootCauseOfException()); + result = eval(/*keepGoing=*/false, ImmutableList.of(parent2Key)); + assertTrue(result.hasError()); + assertEquals(errorKey, result.getError(parent2Key).getRootCauseOfException()); + } + + @Test + public void cachedTopLevelErrorsShouldHaltNoKeepGoingBuildEarly() throws Exception { + graph = new DeterministicInMemoryGraph(); + tester = new GraphTester(); + SkyKey errorKey = GraphTester.toSkyKey("error"); + tester.getOrCreate(errorKey).setHasError(true); + EvaluationResult<StringValue> result = eval(/*keepGoing=*/true, ImmutableList.of(errorKey)); + assertTrue(result.hasError()); + assertEquals(errorKey, result.getError().getRootCauseOfException()); + SkyKey rogueKey = GraphTester.toSkyKey("rogue"); + tester.getOrCreate(rogueKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) { + // This SkyFunction could do an arbitrarily bad computation, e.g. loop-forever. So we want + // to make sure that it is never run when we want to fail-fast anyway. + fail("eval call should have already terminated"); + return null; + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + result = eval(/*keepGoing=*/false, ImmutableList.of(errorKey, rogueKey)); + assertTrue(result.hasError()); + assertEquals(errorKey, result.getError(errorKey).getRootCauseOfException()); + assertFalse(result.errorMap().containsKey(rogueKey)); + } + + private void runUnhandledTransitiveErrors(boolean keepGoing, + final boolean explicitlyPropagateError) throws Exception { + graph = new DeterministicInMemoryGraph(); + tester = new GraphTester(); + SkyKey grandparentKey = GraphTester.toSkyKey("grandparent"); + final SkyKey parentKey = GraphTester.toSkyKey("parent"); + final SkyKey childKey = GraphTester.toSkyKey("child"); + final AtomicBoolean errorPropagated = new AtomicBoolean(false); + tester.getOrCreate(grandparentKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + try { + return env.getValueOrThrow(parentKey, SomeErrorException.class); + } catch (SomeErrorException e) { + errorPropagated.set(true); + throw new GenericFunctionException(e, Transience.PERSISTENT); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + tester.getOrCreate(parentKey).setBuilder(new SkyFunction() { + @Override + public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException { + if (explicitlyPropagateError) { + try { + return env.getValueOrThrow(childKey, SomeErrorException.class); + } catch (SomeErrorException e) { + throw new GenericFunctionException(e, childKey); + } + } else { + return env.getValue(childKey); + } + } + + @Override + public String extractTag(SkyKey skyKey) { + return null; + } + }); + tester.getOrCreate(childKey).setHasError(/*hasError=*/true); + EvaluationResult<StringValue> result = eval(keepGoing, ImmutableList.of(grandparentKey)); + assertTrue(result.hasError()); + assertTrue(errorPropagated.get()); + assertEquals(grandparentKey, result.getError().getRootCauseOfException()); + } + + @Test + public void unhandledTransitiveErrorsDuringErrorBubbling_ImplicitPropagation() throws Exception { + runUnhandledTransitiveErrors(/*keepGoing=*/false, /*explicitlyPropagateError=*/false); + } + + @Test + public void unhandledTransitiveErrorsDuringErrorBubbling_ExplicitPropagation() throws Exception { + runUnhandledTransitiveErrors(/*keepGoing=*/false, /*explicitlyPropagateError=*/true); + } + + @Test + public void unhandledTransitiveErrorsDuringNormalEvaluation_ImplicitPropagation() + throws Exception { + runUnhandledTransitiveErrors(/*keepGoing=*/true, /*explicitlyPropagateError=*/false); + } + + @Test + public void unhandledTransitiveErrorsDuringNormalEvaluation_ExplicitPropagation() + throws Exception { + runUnhandledTransitiveErrors(/*keepGoing=*/true, /*explicitlyPropagateError=*/true); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/ReverseDepsUtilTest.java b/src/test/java/com/google/devtools/build/skyframe/ReverseDepsUtilTest.java new file mode 100644 index 0000000000..9183775d58 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/ReverseDepsUtilTest.java @@ -0,0 +1,155 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Test for {@code ReverseDepsUtil}. + */ +@RunWith(Parameterized.class) +public class ReverseDepsUtilTest { + + private static final SkyFunctionName NODE_TYPE = new SkyFunctionName("Type", false); + private final int numElements; + + @Parameters + public static List<Object[]> paramenters() { + List<Object[]> params = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + params.add(new Object[]{i}); + } + return params; + } + + public ReverseDepsUtilTest(int numElements) { + this.numElements = numElements; + } + + private static final ReverseDepsUtil<Example> REVERSE_DEPS_UTIL = new ReverseDepsUtil<Example>() { + @Override + void setReverseDepsObject(Example container, Object object) { + container.reverseDeps = object; + } + + @Override + void setSingleReverseDep(Example container, boolean singleObject) { + container.single = singleObject; + } + + @Override + void setReverseDepsToRemove(Example container, List<SkyKey> object) { + container.reverseDepsToRemove = object; + } + + @Override + Object getReverseDepsObject(Example container) { + return container.reverseDeps; + } + + @Override + boolean isSingleReverseDep(Example container) { + return container.single; + } + + @Override + List<SkyKey> getReverseDepsToRemove(Example container) { + return container.reverseDepsToRemove; + } + }; + + private class Example { + + Object reverseDeps = ImmutableList.of(); + boolean single; + List<SkyKey> reverseDepsToRemove; + } + + @Test + public void testAddAndRemove() { + for (int numRemovals = 0; numRemovals <= numElements; numRemovals++) { + Example example = new Example(); + for (int j = 0; j < numElements; j++) { + REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, j))); + } + // Not a big test but at least check that it does not blow up. + assertThat(REVERSE_DEPS_UTIL.toString(example)).isNotEmpty(); + assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements); + for (int i = 0; i < numRemovals; i++) { + REVERSE_DEPS_UTIL.removeReverseDep(example, new SkyKey(NODE_TYPE, i)); + } + assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements - numRemovals); + assertThat(example.reverseDepsToRemove).isNull(); + } + } + + // Same as testAdditionAndRemoval but we add all the reverse deps in one call. + @Test + public void testAddAllAndRemove() { + for (int numRemovals = 0; numRemovals <= numElements; numRemovals++) { + Example example = new Example(); + List<SkyKey> toAdd = new ArrayList<>(); + for (int j = 0; j < numElements; j++) { + toAdd.add(new SkyKey(NODE_TYPE, j)); + } + REVERSE_DEPS_UTIL.addReverseDeps(example, toAdd); + assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements); + for (int i = 0; i < numRemovals; i++) { + REVERSE_DEPS_UTIL.removeReverseDep(example, new SkyKey(NODE_TYPE, i)); + } + assertThat(REVERSE_DEPS_UTIL.getReverseDeps(example)).hasSize(numElements - numRemovals); + assertThat(example.reverseDepsToRemove).isNull(); + } + } + + @Test + public void testDuplicateCheckOnGetReverseDeps() { + Example example = new Example(); + for (int i = 0; i < numElements; i++) { + REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, i))); + } + // Should only fail when we call getReverseDeps(). + REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, 0))); + try { + REVERSE_DEPS_UTIL.getReverseDeps(example); + assertThat(numElements).is(0); + } catch (Exception expected) { } + } + + @Test + public void testMaybeCheck() { + Example example = new Example(); + for (int i = 0; i < numElements; i++) { + REVERSE_DEPS_UTIL.addReverseDeps(example, Collections.singleton(new SkyKey(NODE_TYPE, i))); + // This should always succeed, since the next element is still not present. + REVERSE_DEPS_UTIL.maybeCheckReverseDepNotPresent(example, new SkyKey(NODE_TYPE, i + 1)); + } + try { + REVERSE_DEPS_UTIL.maybeCheckReverseDepNotPresent(example, new SkyKey(NODE_TYPE, 0)); + // Should only fail if empty or above the checking threshold. + assertThat(numElements == 0 || numElements >= ReverseDepsUtil.MAYBE_CHECK_THRESHOLD).isTrue(); + } catch (Exception expected) { } + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/SomeErrorException.java b/src/test/java/com/google/devtools/build/skyframe/SomeErrorException.java new file mode 100644 index 0000000000..b25cbd3180 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/SomeErrorException.java @@ -0,0 +1,20 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +public class SomeErrorException extends Exception { + public SomeErrorException(String msg) { + super(msg); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/TrackingAwaiter.java b/src/test/java/com/google/devtools/build/skyframe/TrackingAwaiter.java new file mode 100644 index 0000000000..3757583ccb --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/TrackingAwaiter.java @@ -0,0 +1,78 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.base.Throwables; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.util.Pair; + +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** Safely await {@link CountDownLatch}es in tests, storing any exceptions that happen. */ +public class TrackingAwaiter { + private final ConcurrentLinkedQueue<Pair<String, Throwable>> exceptionsThrown = + new ConcurrentLinkedQueue<>(); + + /** + * This method fixes a race condition with simply calling {@link CountDownLatch#await}. If this + * thread is interrupted before {@code latch.await} is called, then {@code latch.await} will throw + * an {@link InterruptedException} without checking the value of the latch at all. This leads to a + * race condition in which this thread will throw an InterruptedException if it is slow calling + * {@code latch.await}, but it will succeed normally otherwise. + * + * <p>To avoid this, we wait for the latch uninterruptibly. In the end, if the latch has in fact + * been released, we do nothing, although the interrupted bit is set, so that the caller can + * decide to throw an InterruptedException if it wants to. If the latch was not released, then + * this was not a race condition, but an honest-to-goodness interrupt, and we propagate the + * exception onward. + */ + public static void waitAndMaybeThrowInterrupt(CountDownLatch latch, String errorMessage) + throws InterruptedException { + if (Uninterruptibles.awaitUninterruptibly(latch, TestUtils.WAIT_TIMEOUT_SECONDS, + TimeUnit.SECONDS)) { + // Latch was released. We can ignore the interrupt state. + return; + } + if (!Thread.currentThread().isInterrupted()) { + // Nobody interrupted us, but latch wasn't released. Failure. + throw new AssertionError(errorMessage); + } else { + // We were interrupted before the latch was released. Propagate this interruption. + throw new InterruptedException(); + } + } + + /** Threadpools can swallow exceptions. Make sure they don't get lost. */ + public void awaitLatchAndTrackExceptions(CountDownLatch latch, String errorMessage) { + try { + waitAndMaybeThrowInterrupt(latch, errorMessage); + } catch (Throwable e) { + // We would expect e to be InterruptedException or AssertionError, but we leave it open so + // that any throwable gets recorded. + exceptionsThrown.add(Pair.of(errorMessage, e)); + // Caller will assert exceptionsThrown is empty at end of test and fail, even if this is + // swallowed. + Throwables.propagate(e); + } + } + + public void assertNoErrors() { + assertThat(exceptionsThrown).isEmpty(); + } +} diff --git a/src/test/java/com/google/devtools/build/skyframe/TrackingInvalidationReceiver.java b/src/test/java/com/google/devtools/build/skyframe/TrackingInvalidationReceiver.java new file mode 100644 index 0000000000..e93098d702 --- /dev/null +++ b/src/test/java/com/google/devtools/build/skyframe/TrackingInvalidationReceiver.java @@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; + +import java.util.Set; + +/** + * A testing utility to keep track of evaluation. + */ +public class TrackingInvalidationReceiver implements EvaluationProgressReceiver { + public final Set<SkyValue> dirty = Sets.newConcurrentHashSet(); + public final Set<SkyValue> deleted = Sets.newConcurrentHashSet(); + public final Set<SkyKey> enqueued = Sets.newConcurrentHashSet(); + public final Set<SkyKey> evaluated = Sets.newConcurrentHashSet(); + + @Override + public void invalidated(SkyValue value, InvalidationState state) { + switch (state) { + case DELETED: + dirty.remove(value); + deleted.add(value); + break; + case DIRTY: + dirty.add(value); + Preconditions.checkState(!deleted.contains(value)); + break; + default: + throw new IllegalStateException(); + } + } + + @Override + public void enqueueing(SkyKey skyKey) { + enqueued.add(skyKey); + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue value, EvaluationState state) { + evaluated.add(skyKey); + switch (state) { + default: + dirty.remove(value); + deleted.remove(value); + break; + } + } + + public void clear() { + dirty.clear(); + deleted.clear(); + enqueued.clear(); + evaluated.clear(); + } +} |