From 79f052118e94c418247c4ea0ba6fcd3a35ca7e0d Mon Sep 17 00:00:00 2001 From: Ulf Adams Date: Fri, 27 Feb 2015 16:40:00 +0000 Subject: Move BuildViewTestCase to the lib.analysis.util package. -- MOS_MIGRATED_REVID=87345558 --- .../build/lib/analysis/util/BuildViewTestCase.java | 1355 ++++++++++++++++++++ 1 file changed, 1355 insertions(+) create mode 100644 src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java (limited to 'src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java') diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java new file mode 100644 index 0000000000..5a8677a3b1 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java @@ -0,0 +1,1355 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.analysis.util; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactOwner; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.MapBasedActionGraph; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.actions.MutableActionGraph; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.actions.util.ActionsTestUtil; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisHooks; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult; +import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredAttributeMapper; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider; +import com.google.devtools.build.lib.analysis.FileConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TopLevelArtifactProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.ConstantRuleVisibility; +import com.google.devtools.build.lib.packages.NoSuchPackageException; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.util.MockToolsConfig; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.skyframe.DiffAwareness; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.testutil.MoreAsserts; +import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.testutil.TestRuleClassProvider; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.ModifiedFileSet; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; + +import org.mockito.Mockito; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +/** + * Common test code that creates a BuildView instance. + */ +public abstract class BuildViewTestCase extends FoundationTestCase { + protected static final int LOADING_PHASE_THREADS = 20; + + protected ConfiguredRuleClassProvider ruleClassProvider; + protected ConfigurationFactory configurationFactory; + protected BuildView view; + + private SequencedSkyframeExecutor skyframeExecutor; + + protected BlazeDirectories directories; + protected BinTools binTools; + + // Note that these configurations are virtual (they use only VFS) + protected BuildConfigurationCollection masterConfig; + protected BuildConfiguration targetConfig; // "target" or "build" config + + protected OptionsParser optionsParser; + private PackageCacheOptions packageCacheOptions; + + protected MockToolsConfig mockToolsConfig; + + protected WorkspaceStatusAction.Factory workspaceStatusActionFactory; + + private MutableActionGraph mutableActionGraph; + + @Override + protected void setUp() throws Exception { + super.setUp(); + AnalysisMock mock = getAnalysisMock(); + directories = new BlazeDirectories(outputBase, outputBase, rootDirectory); + binTools = BinTools.forUnitTesting(directories, TestConstants.EMBEDDED_TOOLS); + mockToolsConfig = new MockToolsConfig(rootDirectory, false); + mock.setupMockClient(mockToolsConfig); + configurationFactory = mock.createConfigurationFactory(); + packageCacheOptions = parsePackageCacheOptions(); + workspaceStatusActionFactory = + new AnalysisTestUtil.DummyWorkspaceStatusActionFactory(directories); + mutableActionGraph = new MapBasedActionGraph(); + ruleClassProvider = getRuleClassProvider(); + skyframeExecutor = SequencedSkyframeExecutor.create(reporter, + new PackageFactory(ruleClassProvider, getEnvironmentExtensions()), + new TimestampGranularityMonitor(BlazeClock.instance()), directories, + workspaceStatusActionFactory, + ruleClassProvider.getBuildInfoFactories(), + ImmutableSet.of(), + ImmutableList.of(), + Predicates.alwaysFalse(), + getPreprocessorFactorySupplier(), + ImmutableMap.of(), + getPrecomputedValues() + ); + skyframeExecutor.preparePackageLoading( + new PathPackageLocator(rootDirectory), ConstantRuleVisibility.PUBLIC, true, "", + UUID.randomUUID()); + useConfiguration(); + setUpSkyframe(); + // Also initializes ResourceManager. + ResourceManager.instance().setAvailableResources(getStartingResources()); + } + + protected AnalysisMock getAnalysisMock() { + try { + Class providerClass = Class.forName(TestConstants.TEST_ANALYSIS_MOCK); + Field instanceField = providerClass.getField("INSTANCE"); + return (AnalysisMock) instanceField.get(null); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + /** Creates or retrieves the rule class provider used in this test. */ + protected ConfiguredRuleClassProvider getRuleClassProvider() { + return TestRuleClassProvider.getRuleClassProvider(); + } + + protected Iterable getEnvironmentExtensions() { + return ImmutableList.of(); + } + + protected ImmutableList getPrecomputedValues() { + return ImmutableList.of(); + } + + protected Preprocessor.Factory.Supplier getPreprocessorFactorySupplier() { + return Preprocessor.Factory.Supplier.NullSupplier.INSTANCE; + } + + protected ResourceSet getStartingResources() { + // Effectively disable ResourceManager by default. + return ResourceSet.createWithRamCpuIo(Double.MAX_VALUE, Double.MAX_VALUE, Double.MAX_VALUE); + } + + protected final BuildConfigurationCollection createConfigurations(String... args) + throws Exception { + optionsParser = OptionsParser.newOptionsParser(Iterables.concat(Arrays.asList( + ExecutionOptions.class, + BuildRequest.BuildRequestOptions.class), + ruleClassProvider.getConfigurationOptions())); + try { + List configurationArgs = new ArrayList<>(); + configurationArgs.add("--experimental_extended_sanity_checks"); + configurationArgs.addAll(getAnalysisMock().getOptionOverrides()); + + optionsParser.parse(configurationArgs); + optionsParser.parse(args); + + configurationFactory.forbidSanityCheck(); + BuildOptions buildOptions = ruleClassProvider.createBuildOptions(optionsParser); + ensureTargetsVisited(buildOptions.getAllLabels().values()); + BuildConfigurationKey key = new BuildConfigurationKey( + buildOptions, directories, + ImmutableMap.of()); + skyframeExecutor.invalidateConfigurationCollection(); + return skyframeExecutor.createConfigurations(configurationFactory, key); + } catch (InvalidConfigurationException | OptionsParsingException e) { + throw new IllegalArgumentException(e); + } + } + + protected Target getTarget(String label) + throws NoSuchPackageException, NoSuchTargetException, + Label.SyntaxException, InterruptedException { + return getTarget(Label.parseAbsolute(label)); + } + + protected Target getTarget(Label label) + throws NoSuchPackageException, NoSuchTargetException, InterruptedException { + return getPackageManager().getTarget(reporter, label); + } + + private void setUpSkyframe() { + PathPackageLocator pkgLocator = PathPackageLocator.create( + packageCacheOptions.packagePath, reporter, rootDirectory, rootDirectory); + skyframeExecutor.preparePackageLoading(pkgLocator, + packageCacheOptions.defaultVisibility, true, + ruleClassProvider.getDefaultsPackageContent(optionsParser), + UUID.randomUUID()); + skyframeExecutor.setDeletedPackages(ImmutableSet.copyOf(packageCacheOptions.deletedPackages)); + } + + protected void setPackageCacheOptions(String... options) throws Exception { + packageCacheOptions = parsePackageCacheOptions(options); + setUpSkyframe(); + } + + private PackageCacheOptions parsePackageCacheOptions(String... options) throws Exception { + OptionsParser parser = OptionsParser.newOptionsParser(PackageCacheOptions.class); + parser.parse("--default_visibility=public"); + parser.parse(options); + return parser.getOptions(PackageCacheOptions.class); + } + + /** Used by skyframe-only tests. */ + protected SequencedSkyframeExecutor getSkyframeExecutor() { + return Preconditions.checkNotNull(skyframeExecutor); + } + + protected PackageManager getPackageManager() { + return skyframeExecutor.getPackageManager(); + } + + protected AnalysisHooks getAnalysisHooks() { + return new AnalysisHooks() { + @Override + public PackageManager getPackageManager() { + return BuildViewTestCase.this.getPackageManager(); + } + + @Override + public ConfiguredTarget getExistingConfiguredTarget(Target target, + BuildConfiguration configuration) { + return view.getExistingConfiguredTarget(target, configuration); + } + + }; + } + + /** + * Invalidates all existing packages. + * @throws InterruptedException + */ + protected void invalidatePackages() throws InterruptedException { + skyframeExecutor.invalidateFilesUnderPathForTesting(ModifiedFileSet.EVERYTHING_MODIFIED, + rootDirectory); + } + + /** + * Sets host and target configuration using the specified options, falling back to the default + * options for unspecified ones, and recreates the build view. + * + * @throws IllegalArgumentException + */ + protected final void useConfiguration(String... args) throws Exception { + masterConfig = createConfigurations(args); + targetConfig = getTargetConfiguration(); + createBuildView(); + } + + /** + * Creates BuildView using current hostConfig/targetConfig values. + * Ensures that hostConfig is either identical to the targetConfig or has + * 'host' short name. + */ + protected final void createBuildView() throws Exception { + Preconditions.checkNotNull(masterConfig); + Preconditions.checkState(getHostConfiguration() == getTargetConfiguration() + || getHostConfiguration().getShortName().equals("host"), + "Host configuration %s does not have name 'host' " + + "and does not match target configuration %s", + getHostConfiguration(), getTargetConfiguration()); + + String defaultsPackageContent = ruleClassProvider.getDefaultsPackageContent(optionsParser); + skyframeExecutor.setupDefaultPackage(defaultsPackageContent); + skyframeExecutor.dropConfiguredTargets(); + + view = new BuildView(directories, getPackageManager(), ruleClassProvider, skyframeExecutor, + binTools, null); + view.setConfigurationsForTesting(masterConfig); + + view.setArtifactRoots( + ImmutableMap.of(PackageIdentifier.createInDefaultRepo(""), rootDirectory)); + simulateLoadingPhase(); + } + + protected CachingAnalysisEnvironment getTestAnalysisEnvironment() { + return new CachingAnalysisEnvironment(view.getArtifactFactory(), + ArtifactOwner.NULL_OWNER, /*isSystemEnv=*/true, /*extendedSanityChecks*/false, reporter, + /*skyframeEnv=*/ null, /*actionsEnabled=*/true, binTools); + } + + /** + * Allows access to the prerequisites of a configured target. This is currently used in some tests + * to reach into the internals of RuleCT for white box testing. In principle, this should not be + * used; instead tests should only assert on properties of the exposed provider instances and / or + * the action graph. + */ + protected Iterable getDirectPrerequisites(ConfiguredTarget target) { + return view.getDirectPrerequisites(target); + } + + /** + * Creates and returns a rule context that is equivalent to the one that was used to create the + * given configured target. + */ + protected RuleContext getRuleContext(ConfiguredTarget target) { + return view.getRuleContextForTesting(target, new StubAnalysisEnvironment()); + } + + /** + * Creates and returns a rule context to use for Skylark tests that is equivalent to the one + * that was used to create the given configured target. + */ + protected RuleContext getRuleContextForSkylark(ConfiguredTarget target) { + // TODO(bazel-team): we need this horrible workaround because CachingAnalysisEnvironment + // only works with StoredErrorEventListener despite the fact it accepts the interface + // ErrorEventListener, so it's not possible to create it with reporter. + // See BuildView.getRuleContextForTesting(). + StoredEventHandler eventHandler = new StoredEventHandler() { + @Override + public synchronized void handle(Event e) { + super.handle(e); + reporter.handle(e); + } + }; + return view.getRuleContextForTesting(target, eventHandler); + } + + /** + * Allows access to the prerequisites of a configured target. This is currently used in some tests + * to reach into the internals of RuleCT for white box testing. In principle, this should not be + * used; instead tests should only assert on properties of the exposed provider instances and / or + * the action graph. + */ + protected List getPrerequisites(ConfiguredTarget target, + String attributeName) { + return getRuleContext(target).getConfiguredTargetMap().get(attributeName); + } + + /** + * Allows access to the prerequisites of a configured target. This is currently used in some tests + * to reach into the internals of RuleCT for white box testing. In principle, this should not be + * used; instead tests should only assert on properties of the exposed provider instances and / or + * the action graph. + */ + protected Iterable getPrerequisites( + ConfiguredTarget target, String attributeName, Class classType) { + return AnalysisUtils.getProviders(getPrerequisites(target, attributeName), classType); + } + + /** + * Allows access to the prerequisites of a configured target. This is currently used in some tests + * to reach into the internals of RuleCT for white box testing. In principle, this should not be + * used; instead tests should only assert on properties of the exposed provider instances and / or + * the action graph. + */ + protected ImmutableList getPrerequisiteArtifacts( + ConfiguredTarget target, String attributeName) { + Set result = new LinkedHashSet<>(); + for (FileProvider provider : getPrerequisites(target, attributeName, FileProvider.class)) { + Iterables.addAll(result, provider.getFilesToBuild()); + } + return ImmutableList.copyOf(result); + } + + protected final Action getGeneratingAction(Artifact artifact) { + Preconditions.checkNotNull(artifact); + Action action = mutableActionGraph.getGeneratingAction(artifact); + if (action != null) { + return action; + } + return view.getActionGraph().getGeneratingAction(artifact); + } + + protected void simulateLoadingPhase() { + try { + ensureTargetsVisited(targetConfig.getAllLabels().values()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected ActionsTestUtil actionsTestUtil() { + return new ActionsTestUtil(view.getActionGraph()); + } + + private Set getTargets(Iterable