diff options
author | 2018-06-28 10:18:39 -0700 | |
---|---|---|
committer | 2018-06-28 10:19:59 -0700 | |
commit | fc6c7425dc86f13d6922254c0efd11eb39b8c215 (patch) | |
tree | f559b4d341c6a764fbe2e9acceacaf3de9ef1787 /src/main/java/com/google/devtools | |
parent | e661f88e58ab73e1f21f16531e8cc78ff7d2cad9 (diff) |
Split common functionality out of ConfiguredTargetQueryEnvironment.
This will allow us to reuse the code for the action graph query in a later CL.
RELNOTES: None
PiperOrigin-RevId: 202500176
Diffstat (limited to 'src/main/java/com/google/devtools')
-rw-r--r-- | src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java | 432 | ||||
-rw-r--r-- | src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java | 438 |
2 files changed, 482 insertions, 388 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java index d9662e24ff..242edb534a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java @@ -15,87 +15,45 @@ package com.google.devtools.build.lib.query2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.MoreExecutors; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; -import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.TargetParsingException; import com.google.devtools.build.lib.cmdline.TargetPattern; -import com.google.devtools.build.lib.collect.compacthashset.CompactHashSet; -import com.google.devtools.build.lib.concurrent.MultisetSemaphore; -import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.Reporter; -import com.google.devtools.build.lib.packages.DependencyFilter; -import com.google.devtools.build.lib.packages.NoSuchTargetException; -import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.RuleTransitionFactory; import com.google.devtools.build.lib.packages.Target; -import com.google.devtools.build.lib.pkgcache.FilteringPolicies; import com.google.devtools.build.lib.pkgcache.PackageManager; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; -import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; import com.google.devtools.build.lib.query2.engine.Callback; import com.google.devtools.build.lib.query2.engine.KeyExtractor; -import com.google.devtools.build.lib.query2.engine.MinDepthUniquifier; import com.google.devtools.build.lib.query2.engine.QueryEnvironment; -import com.google.devtools.build.lib.query2.engine.QueryEvalResult; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; -import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; -import com.google.devtools.build.lib.query2.engine.QueryUtil.MinDepthUniquifierImpl; -import com.google.devtools.build.lib.query2.engine.QueryUtil.MutableKeyExtractorBackedMapImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.ThreadSafeMutableKeyExtractorBackedSetImpl; -import com.google.devtools.build.lib.query2.engine.QueryUtil.UniquifierImpl; -import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; -import com.google.devtools.build.lib.query2.engine.Uniquifier; import com.google.devtools.build.lib.query2.output.AspectResolver; import com.google.devtools.build.lib.query2.output.CqueryOptions; -import com.google.devtools.build.lib.query2.output.QueryOptions; import com.google.devtools.build.lib.rules.AliasConfiguredTarget; import com.google.devtools.build.lib.skyframe.BuildConfigurationValue; import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue; -import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider; -import com.google.devtools.build.lib.skyframe.PackageValue; -import com.google.devtools.build.lib.skyframe.RecursivePackageProviderBackedTargetPatternResolver; -import com.google.devtools.build.lib.skyframe.RecursivePkgValueRootPackageExtractor; -import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; -import com.google.devtools.build.lib.skyframe.TargetPatternValue; -import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.WalkableGraph; -import com.google.devtools.common.options.OptionsParser; -import com.google.devtools.common.options.OptionsParsingException; -import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; import javax.annotation.Nullable; /** * {@link QueryEnvironment} that runs queries over the configured target (analysis) graph. * - * <p>This environment can theoretically be used for multiple queries, but currently is only ever - * used for one over the course of its lifetime. If this ever changed to be used for multiple, the - * {@link ConfiguredTargetAccessor} field should be initialized on a per-query basis not a - * per-environment basis. - * * <p>There is currently a limited way to specify a configuration in the query syntax via {@link * ConfigFunction}. This currently still limits the user to choosing the 'target', 'host', or null * configurations. It shouldn't be terribly difficult to expand this with {@link @@ -105,41 +63,20 @@ import javax.annotation.Nullable; * <p>Aspects are also not supported, but probably should be in some fashion. */ public class ConfiguredTargetQueryEnvironment - extends AbstractBlazeQueryEnvironment<ConfiguredTarget> { - private final BuildConfiguration defaultTargetConfiguration; - private final BuildConfiguration hostConfiguration; - private final String parserPrefix; - protected final PathPackageLocator pkgPath; - private final Supplier<WalkableGraph> walkableGraphSupplier; - private ConfiguredTargetAccessor accessor; - protected WalkableGraph graph; - - private static final Function<SkyKey, ConfiguredTargetKey> SKYKEY_TO_CTKEY = - skyKey -> (ConfiguredTargetKey) skyKey.argument(); - private static final ImmutableList<TargetPatternKey> ALL_PATTERNS; - private final KeyExtractor<ConfiguredTarget, ConfiguredTargetKey> configuredTargetKeyExtractor; - + extends PostAnalysisQueryEnvironment<ConfiguredTarget> { /** Common query functions and cquery specific functions. */ public static final ImmutableList<QueryFunction> FUNCTIONS = populateFunctions(); /** Cquery specific functions. */ public static final ImmutableList<QueryFunction> CQUERY_FUNCTIONS = getCqueryFunctions(); - static { - TargetPattern targetPattern; - try { - targetPattern = TargetPattern.defaultParser().parse("//..."); - } catch (TargetParsingException e) { - throw new IllegalStateException(e); - } - ALL_PATTERNS = - ImmutableList.of( - new TargetPatternKey( - targetPattern, FilteringPolicies.NO_FILTER, false, "", ImmutableSet.of())); - } + private CqueryOptions cqueryOptions; - private RecursivePackageProviderBackedTargetPatternResolver resolver; + private final KeyExtractor<ConfiguredTarget, ConfiguredTargetKey> configuredTargetKeyExtractor; - private CqueryOptions cqueryOptions; + @Override + protected KeyExtractor<ConfiguredTarget, ConfiguredTargetKey> getConfiguredTargetKeyExtractor() { + return configuredTargetKeyExtractor; + } public ConfiguredTargetQueryEnvironment( boolean keepGoing, @@ -151,13 +88,17 @@ public class ConfiguredTargetQueryEnvironment PathPackageLocator pkgPath, Supplier<WalkableGraph> walkableGraphSupplier, Set<Setting> settings) { - super(keepGoing, true, Rule.ALL_LABELS, eventHandler, settings, extraFunctions); - this.defaultTargetConfiguration = defaultTargetConfiguration; - this.hostConfiguration = hostConfiguration; - this.parserPrefix = parserPrefix; - this.pkgPath = pkgPath; - this.walkableGraphSupplier = walkableGraphSupplier; - this.accessor = new ConfiguredTargetAccessor(walkableGraphSupplier.get()); + super( + keepGoing, + eventHandler, + extraFunctions, + defaultTargetConfiguration, + hostConfiguration, + parserPrefix, + pkgPath, + walkableGraphSupplier, + settings, + new ConfiguredTargetAccessor(walkableGraphSupplier.get())); this.configuredTargetKeyExtractor = element -> { try { @@ -199,6 +140,7 @@ public class ConfiguredTargetQueryEnvironment return ImmutableList.of(new ConfigFunction()); } + @Override public ImmutableList<CqueryThreadsafeCallback> getDefaultOutputFormatters( TargetAccessor<ConfiguredTarget> accessor, Reporter reporter, @@ -233,81 +175,6 @@ public class ConfiguredTargetQueryEnvironment } @Override - public QueryEvalResult evaluateQuery( - QueryExpression expr, ThreadSafeOutputFormatterCallback<ConfiguredTarget> callback) - throws QueryException, InterruptedException, IOException { - beforeEvaluateQuery(); - return super.evaluateQuery(expr, callback); - } - - private void beforeEvaluateQuery() throws InterruptedException, QueryException { - graph = walkableGraphSupplier.get(); - GraphBackedRecursivePackageProvider graphBackedRecursivePackageProvider = - new GraphBackedRecursivePackageProvider( - graph, ALL_PATTERNS, pkgPath, new RecursivePkgValueRootPackageExtractor()); - resolver = - new RecursivePackageProviderBackedTargetPatternResolver( - graphBackedRecursivePackageProvider, - eventHandler, - FilteringPolicies.NO_FILTER, - MultisetSemaphore.unbounded()); - checkSettings(settings); - } - - // Check to make sure the settings requested are currently supported by this class - private void checkSettings(Set<Setting> settings) throws QueryException { - if (settings.contains(Setting.NO_NODEP_DEPS) - || settings.contains(Setting.TESTS_EXPRESSION_STRICT)) { - settings = - Sets.difference( - settings, ImmutableSet.of(Setting.NO_HOST_DEPS, Setting.NO_IMPLICIT_DEPS)); - throw new QueryException( - String.format( - "The following filter(s) are not currently supported by configured query: %s", - settings.toString())); - } - } - - public BuildConfiguration getHostConfiguration() { - return hostConfiguration; - } - - @Override - public TargetAccessor<ConfiguredTarget> getAccessor() { - return accessor; - } - - // TODO(bazel-team): It's weird that this untemplated function exists. Fix? Or don't implement? - @Override - public Target getTarget(Label label) - throws TargetNotFoundException, QueryException, InterruptedException { - try { - return ((PackageValue) - walkableGraphSupplier.get().getValue(PackageValue.key(label.getPackageIdentifier()))) - .getPackage() - .getTarget(label.getName()); - } catch (NoSuchTargetException e) { - throw new TargetNotFoundException(e); - } - } - - @Override - public ConfiguredTarget getOrCreate(ConfiguredTarget target) { - return target; - } - - /** - * This method has to exist because {@link AliasConfiguredTarget#getLabel()} returns - * the label of the "actual" target instead of the alias target. Grr. - */ - public static Label getCorrectLabel(ConfiguredTarget target) { - if (target instanceof AliasConfiguredTarget) { - return ((AliasConfiguredTarget) target).getOriginalLabel(); - } - return target.getLabel(); - } - - @Override public QueryTaskFuture<Void> getTargetsMatchingPattern( QueryExpression owner, String pattern, Callback<ConfiguredTarget> callback) { TargetPattern patternToEval; @@ -320,8 +187,6 @@ public class ConfiguredTargetQueryEnvironment return immediateFailedFuture(qe); } return immediateSuccessfulFuture(null); - } catch (InterruptedException ie) { - return immediateCancelledFuture(); } AsyncFunction<TargetParsingException, Void> reportBuildFileErrorAsyncFunction = exn -> { @@ -368,23 +233,13 @@ public class ConfiguredTargetQueryEnvironment return getNullConfiguredTarget(label); } + @Override @Nullable - private ConfiguredTarget getConfiguredTarget(SkyKey key) throws InterruptedException { - ConfiguredTargetValue value = - ((ConfiguredTargetValue) walkableGraphSupplier.get().getValue(key)); + protected ConfiguredTarget getValueFromKey(SkyKey key) throws InterruptedException { + ConfiguredTargetValue value = getConfiguredTargetValue(key); return value == null ? null : value.getConfiguredTarget(); } - private TargetPattern getPattern(String pattern) - throws TargetParsingException, InterruptedException { - TargetPatternKey targetPatternKey = - ((TargetPatternKey) - TargetPatternValue.key( - pattern, TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix) - .argument()); - return targetPatternKey.getParsedPattern(); - } - /** * Processes the targets in {@code targets} with the requested {@code configuration} * @@ -440,159 +295,39 @@ public class ConfiguredTargetQueryEnvironment }; } - @Nullable - private ConfiguredTarget getHostConfiguredTarget(Label label) throws InterruptedException { - return getConfiguredTarget(ConfiguredTargetValue.key(label, hostConfiguration)); + /** + * This method has to exist because {@link AliasConfiguredTarget#getLabel()} returns the label of + * the "actual" target instead of the alias target. Grr. + */ + @Override + public Label getCorrectLabel(ConfiguredTarget target) { + if (target instanceof AliasConfiguredTarget) { + return ((AliasConfiguredTarget) target).getOriginalLabel(); + } + return target.getLabel(); } @Nullable - private ConfiguredTarget getTargetConfiguredTarget(Label label) throws InterruptedException { - return getConfiguredTarget(ConfiguredTargetValue.key(label, defaultTargetConfiguration)); + @Override + protected ConfiguredTarget getHostConfiguredTarget(Label label) throws InterruptedException { + return getValueFromKey(ConfiguredTargetValue.key(label, hostConfiguration)); } @Nullable - private ConfiguredTarget getNullConfiguredTarget(Label label) throws InterruptedException { - return getConfiguredTarget(ConfiguredTargetValue.key(label, null)); - } - @Override - public ThreadSafeMutableSet<ConfiguredTarget> getFwdDeps( - Iterable<ConfiguredTarget> targets, QueryExpressionContext<ConfiguredTarget> context) - throws InterruptedException { - Map<SkyKey, ConfiguredTarget> targetsByKey = new HashMap<>(Iterables.size(targets)); - for (ConfiguredTarget target : targets) { - targetsByKey.put(getSkyKey(target), target); - } - Map<SkyKey, Collection<ConfiguredTarget>> directDeps = - targetifyValues(graph.getDirectDeps(targetsByKey.keySet())); - if (targetsByKey.keySet().size() != directDeps.keySet().size()) { - Iterable<ConfiguredTargetKey> missingTargets = - Sets.difference(targetsByKey.keySet(), directDeps.keySet()) - .stream() - .map(SKYKEY_TO_CTKEY) - .collect(Collectors.toList()); - eventHandler.handle(Event.warn("Targets were missing from graph: " + missingTargets)); - } - ThreadSafeMutableSet<ConfiguredTarget> result = createThreadSafeMutableSet(); - for (Map.Entry<SkyKey, Collection<ConfiguredTarget>> entry : directDeps.entrySet()) { - result.addAll(filterFwdDeps(targetsByKey.get(entry.getKey()), entry.getValue())); - } - return result; - } - - private Collection<ConfiguredTarget> filterFwdDeps( - ConfiguredTarget configTarget, Collection<ConfiguredTarget> rawFwdDeps) - throws InterruptedException { - if (settings.isEmpty()) { - return rawFwdDeps; - } - return getAllowedDeps(configTarget, rawFwdDeps); + protected ConfiguredTarget getTargetConfiguredTarget(Label label) throws InterruptedException { + return getValueFromKey(ConfiguredTargetValue.key(label, defaultTargetConfiguration)); } + @Nullable @Override - public Collection<ConfiguredTarget> getReverseDeps( - Iterable<ConfiguredTarget> targets, QueryExpressionContext<ConfiguredTarget> context) - throws InterruptedException { - Map<SkyKey, ConfiguredTarget> targetsByKey = new HashMap<>(Iterables.size(targets)); - for (ConfiguredTarget target : targets) { - targetsByKey.put(getSkyKey(target), target); - } - Map<SkyKey, Collection<ConfiguredTarget>> reverseDepsByKey = - targetifyValues(graph.getReverseDeps(targetsByKey.keySet())); - if (targetsByKey.size() != reverseDepsByKey.size()) { - Iterable<ConfiguredTargetKey> missingTargets = - Sets.difference(targetsByKey.keySet(), reverseDepsByKey.keySet()) - .stream() - .map(SKYKEY_TO_CTKEY) - .collect(Collectors.toList()); - eventHandler.handle(Event.warn("Targets were missing from graph: " + missingTargets)); - } - Map<ConfiguredTarget, Collection<ConfiguredTarget>> reverseDepsByCT = new HashMap<>(); - for (Map.Entry<SkyKey, Collection<ConfiguredTarget>> entry : reverseDepsByKey.entrySet()) { - reverseDepsByCT.put(targetsByKey.get(entry.getKey()), entry.getValue()); - } - return reverseDepsByCT.isEmpty() ? Collections.emptyList() : filterReverseDeps(reverseDepsByCT); - } - - private Collection<ConfiguredTarget> filterReverseDeps( - Map<ConfiguredTarget, Collection<ConfiguredTarget>> rawReverseDeps) { - Set<ConfiguredTarget> result = CompactHashSet.create(); - for (Map.Entry<ConfiguredTarget, Collection<ConfiguredTarget>> targetAndRdeps : - rawReverseDeps.entrySet()) { - ImmutableSet.Builder<ConfiguredTarget> ruleDeps = ImmutableSet.builder(); - for (ConfiguredTarget parent : targetAndRdeps.getValue()) { - if (parent instanceof RuleConfiguredTarget - && dependencyFilter != DependencyFilter.ALL_DEPS) { - ruleDeps.add(parent); - } else { - result.add(parent); - } - } - result.addAll(getAllowedDeps((targetAndRdeps.getKey()), ruleDeps.build())); - } - return result; - } - - /** - * @param target source target - * @param deps next level of deps to filter - */ - private Collection<ConfiguredTarget> getAllowedDeps( - ConfiguredTarget target, Collection<ConfiguredTarget> deps) { - // It's possible to query on a target that's configured in the host configuration. In those - // cases if --nohost_deps is turned on, we only allow reachable targets that are ALSO in the - // host config. This is somewhat counterintuitive and subject to change in the future but seems - // like the best option right now. - if (settings.contains(Setting.NO_HOST_DEPS)) { - BuildConfiguration currentConfig = getConfiguration(target); - if (currentConfig != null && currentConfig.isHostConfiguration()) { - deps = - deps.stream() - .filter( - dep -> - getConfiguration(dep) != null - && getConfiguration(dep).isHostConfiguration()) - .collect(Collectors.toList()); - } else { - deps = - deps.stream() - .filter( - dep -> - getConfiguration(dep) != null - && !getConfiguration(dep).isHostConfiguration()) - .collect(Collectors.toList()); - } - } - if (settings.contains(Setting.NO_IMPLICIT_DEPS) && target instanceof RuleConfiguredTarget) { - Set<ConfiguredTargetKey> implicitDeps = ((RuleConfiguredTarget) target).getImplicitDeps(); - deps = - deps.stream() - .filter( - dep -> - !implicitDeps.contains( - ConfiguredTargetKey.of(getCorrectLabel(dep), getConfiguration(dep)))) - .collect(Collectors.toList()); - } - return deps; - } - - private Map<SkyKey, Collection<ConfiguredTarget>> targetifyValues( - Map<SkyKey, ? extends Iterable<SkyKey>> input) throws InterruptedException { - Map<SkyKey, Collection<ConfiguredTarget>> result = new HashMap<>(); - for (Map.Entry<SkyKey, ? extends Iterable<SkyKey>> entry : input.entrySet()) { - Collection<ConfiguredTarget> value = new ArrayList<>(); - for (SkyKey key : entry.getValue()) { - if (key.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { - value.add(getConfiguredTarget(key)); - } - } - result.put(entry.getKey(), value); - } - return result; + protected ConfiguredTarget getNullConfiguredTarget(Label label) throws InterruptedException { + return getValueFromKey(ConfiguredTargetValue.key(label, null)); } @Nullable - private BuildConfiguration getConfiguration(ConfiguredTarget target) { + @Override + protected BuildConfiguration getConfiguration(ConfiguredTarget target) { try { return target.getConfigurationKey() == null ? null @@ -603,35 +338,9 @@ public class ConfiguredTargetQueryEnvironment } } - private ConfiguredTargetKey getSkyKey(ConfiguredTarget target) { - return ConfiguredTargetKey.of(target, getConfiguration(target)); - } - @Override - public ThreadSafeMutableSet<ConfiguredTarget> getTransitiveClosure( - ThreadSafeMutableSet<ConfiguredTarget> targets, - QueryExpressionContext<ConfiguredTarget> context) - throws InterruptedException { - return SkyQueryUtils.getTransitiveClosure( - targets, targets1 -> getFwdDeps(targets1, context), createThreadSafeMutableSet()); - } - - @Override - public void buildTransitiveClosure( - QueryExpression caller, ThreadSafeMutableSet<ConfiguredTarget> targetNodes, int maxDepth) - throws QueryException, InterruptedException { - // TODO(bazel-team): implement this. Just needed for error-checking. - } - - @Override - public ImmutableList<ConfiguredTarget> getNodesOnPath( - ConfiguredTarget from, ConfiguredTarget to, QueryExpressionContext<ConfiguredTarget> context) - throws InterruptedException { - return SkyQueryUtils.getNodesOnPath( - from, - to, - targets -> getFwdDeps(targets, context), - configuredTargetKeyExtractor::extractKey); + protected ConfiguredTargetKey getSkyKey(ConfiguredTarget target) { + return ConfiguredTargetKey.of(target, getConfiguration(target)); } @Override @@ -641,58 +350,5 @@ public class ConfiguredTargetQueryEnvironment ConfiguredTarget.class, SkyQueryEnvironment.DEFAULT_THREAD_COUNT); } - - @Override - public <V> MutableMap<ConfiguredTarget, V> createMutableMap() { - return new MutableKeyExtractorBackedMapImpl<>(configuredTargetKeyExtractor); - } - - @Override - public Uniquifier<ConfiguredTarget> createUniquifier() { - return new UniquifierImpl<>( - configuredTargetKeyExtractor, SkyQueryEnvironment.DEFAULT_THREAD_COUNT); - } - - @Override - public MinDepthUniquifier<ConfiguredTarget> createMinDepthUniquifier() { - return new MinDepthUniquifierImpl<>( - configuredTargetKeyExtractor, SkyQueryEnvironment.DEFAULT_THREAD_COUNT); - } - - /** Target patterns are resolved on the fly so no pre-work to be done here. */ - @Override - protected void preloadOrThrow(QueryExpression caller, Collection<String> patterns) {} - - public static QueryOptions parseOptions(String rawOptions) throws QueryException { - List<String> options = new ArrayList<>(Arrays.asList(rawOptions.split(" "))); - OptionsParser parser = OptionsParser.newOptionsParser(QueryOptions.class); - parser.setAllowResidue(false); - try { - parser.parse(options); - } catch (OptionsParsingException e) { - throw new QueryException(e.getMessage()); - } - return parser.getOptions(QueryOptions.class); - } - - @Override - public ThreadSafeMutableSet<ConfiguredTarget> getBuildFiles( - QueryExpression caller, - ThreadSafeMutableSet<ConfiguredTarget> nodes, - boolean buildFiles, - boolean loads, - QueryExpressionContext<ConfiguredTarget> context) - throws QueryException, InterruptedException { - throw new QueryException("buildfiles() doesn't make sense for the configured target graph"); - } - - @Override - public Collection<ConfiguredTarget> getSiblingTargetsInPackage(ConfiguredTarget target) { - throw new UnsupportedOperationException("siblings() not supported"); - } - - - @Override - public void close() {} } diff --git a/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java new file mode 100644 index 0000000000..956abf5605 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/PostAnalysisQueryEnvironment.java @@ -0,0 +1,438 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.cmdline.TargetPattern; +import com.google.devtools.build.lib.collect.compacthashset.CompactHashSet; +import com.google.devtools.build.lib.concurrent.MultisetSemaphore; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.DependencyFilter; +import com.google.devtools.build.lib.packages.NoSuchTargetException; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleTransitionFactory; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.FilteringPolicies; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.query2.engine.KeyExtractor; +import com.google.devtools.build.lib.query2.engine.MinDepthUniquifier; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment; +import com.google.devtools.build.lib.query2.engine.QueryEvalResult; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; +import com.google.devtools.build.lib.query2.engine.QueryUtil.MinDepthUniquifierImpl; +import com.google.devtools.build.lib.query2.engine.QueryUtil.MutableKeyExtractorBackedMapImpl; +import com.google.devtools.build.lib.query2.engine.QueryUtil.UniquifierImpl; +import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; +import com.google.devtools.build.lib.query2.engine.Uniquifier; +import com.google.devtools.build.lib.rules.AliasConfiguredTarget; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; +import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue; +import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider; +import com.google.devtools.build.lib.skyframe.PackageValue; +import com.google.devtools.build.lib.skyframe.RecursivePackageProviderBackedTargetPatternResolver; +import com.google.devtools.build.lib.skyframe.RecursivePkgValueRootPackageExtractor; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.TargetPatternValue; +import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.WalkableGraph; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** + * {@link QueryEnvironment} that runs queries based on results from the analysis phase. + * + * <p>This environment can theoretically be used for multiple queries, but currently is only ever + * used for one over the course of its lifetime. If this ever changed to be used for multiple, the + * {@link TargetAccessor} field should be initialized on a per-query basis not a per-environment + * basis. + * + * <p>Aspects are also not supported, but probably should be in some fashion. + */ +abstract class PostAnalysisQueryEnvironment<T> extends AbstractBlazeQueryEnvironment<T> { + protected final BuildConfiguration defaultTargetConfiguration; + protected final BuildConfiguration hostConfiguration; + private final String parserPrefix; + private final PathPackageLocator pkgPath; + private final Supplier<WalkableGraph> walkableGraphSupplier; + private final TargetAccessor<T> accessor; + protected WalkableGraph graph; + + private static final Function<SkyKey, ConfiguredTargetKey> SKYKEY_TO_CTKEY = + skyKey -> (ConfiguredTargetKey) skyKey.argument(); + private static final ImmutableList<TargetPatternKey> ALL_PATTERNS; + + static { + TargetPattern targetPattern; + try { + targetPattern = TargetPattern.defaultParser().parse("//..."); + } catch (TargetParsingException e) { + throw new IllegalStateException(e); + } + ALL_PATTERNS = + ImmutableList.of( + new TargetPatternKey( + targetPattern, FilteringPolicies.NO_FILTER, false, "", ImmutableSet.of())); + } + + protected RecursivePackageProviderBackedTargetPatternResolver resolver; + + public PostAnalysisQueryEnvironment( + boolean keepGoing, + ExtendedEventHandler eventHandler, + Iterable<QueryFunction> extraFunctions, + BuildConfiguration defaultTargetConfiguration, + BuildConfiguration hostConfiguration, + String parserPrefix, + PathPackageLocator pkgPath, + Supplier<WalkableGraph> walkableGraphSupplier, + Set<Setting> settings, + TargetAccessor<T> targetAccessor) { + super(keepGoing, true, Rule.ALL_LABELS, eventHandler, settings, extraFunctions); + this.defaultTargetConfiguration = defaultTargetConfiguration; + this.hostConfiguration = hostConfiguration; + this.parserPrefix = parserPrefix; + this.pkgPath = pkgPath; + this.walkableGraphSupplier = walkableGraphSupplier; + this.accessor = targetAccessor; + } + + public abstract ImmutableList<CqueryThreadsafeCallback> getDefaultOutputFormatters( + TargetAccessor<ConfiguredTarget> accessor, + Reporter reporter, + SkyframeExecutor skyframeExecutor, + BuildConfiguration hostConfiguration, + @Nullable RuleTransitionFactory trimmingTransitionFactory, + PackageManager packageManager); + + protected abstract KeyExtractor<T, ConfiguredTargetKey> getConfiguredTargetKeyExtractor(); + + @Override + public QueryEvalResult evaluateQuery( + QueryExpression expr, ThreadSafeOutputFormatterCallback<T> callback) + throws QueryException, InterruptedException, IOException { + beforeEvaluateQuery(); + return super.evaluateQuery(expr, callback); + } + + private void beforeEvaluateQuery() throws QueryException { + graph = walkableGraphSupplier.get(); + GraphBackedRecursivePackageProvider graphBackedRecursivePackageProvider = + new GraphBackedRecursivePackageProvider( + graph, ALL_PATTERNS, pkgPath, new RecursivePkgValueRootPackageExtractor()); + resolver = + new RecursivePackageProviderBackedTargetPatternResolver( + graphBackedRecursivePackageProvider, + eventHandler, + FilteringPolicies.NO_FILTER, + MultisetSemaphore.unbounded()); + checkSettings(settings); + } + + // Check to make sure the settings requested are currently supported by this class + private void checkSettings(Set<Setting> settings) throws QueryException { + if (settings.contains(Setting.NO_NODEP_DEPS) + || settings.contains(Setting.TESTS_EXPRESSION_STRICT)) { + settings = + Sets.difference( + settings, ImmutableSet.of(Setting.NO_HOST_DEPS, Setting.NO_IMPLICIT_DEPS)); + throw new QueryException( + String.format( + "The following filter(s) are not currently supported by configured query: %s", + settings.toString())); + } + } + + public BuildConfiguration getHostConfiguration() { + return hostConfiguration; + } + + @Override + public TargetAccessor<T> getAccessor() { + return accessor; + } + + // TODO(bazel-team): It's weird that this untemplated function exists. Fix? Or don't implement? + @Override + public Target getTarget(Label label) throws TargetNotFoundException, InterruptedException { + try { + return ((PackageValue) + walkableGraphSupplier.get().getValue(PackageValue.key(label.getPackageIdentifier()))) + .getPackage() + .getTarget(label.getName()); + } catch (NoSuchTargetException e) { + throw new TargetNotFoundException(e); + } + } + + @Override + public T getOrCreate(T target) { + return target; + } + + /** + * This method has to exist because {@link AliasConfiguredTarget#getLabel()} returns the label of + * the "actual" target instead of the alias target. Grr. + */ + public abstract Label getCorrectLabel(T target); + + @Nullable + protected abstract T getHostConfiguredTarget(Label label) throws InterruptedException; + + @Nullable + protected abstract T getTargetConfiguredTarget(Label label) throws InterruptedException; + + @Nullable + protected abstract T getNullConfiguredTarget(Label label) throws InterruptedException; + + @Nullable + protected ConfiguredTargetValue getConfiguredTargetValue(SkyKey key) throws InterruptedException { + return (ConfiguredTargetValue) walkableGraphSupplier.get().getValue(key); + } + + @Nullable + protected abstract T getValueFromKey(SkyKey key) throws InterruptedException; + + protected TargetPattern getPattern(String pattern) throws TargetParsingException { + TargetPatternKey targetPatternKey = + ((TargetPatternKey) + TargetPatternValue.key( + pattern, TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix) + .argument()); + return targetPatternKey.getParsedPattern(); + } + + @Override + public ThreadSafeMutableSet<T> getFwdDeps(Iterable<T> targets, QueryExpressionContext<T> context) + throws InterruptedException { + Map<SkyKey, T> targetsByKey = Maps.newHashMapWithExpectedSize(Iterables.size(targets)); + for (T target : targets) { + targetsByKey.put(getSkyKey(target), target); + } + Map<SkyKey, Collection<T>> directDeps = + targetifyValues(graph.getDirectDeps(targetsByKey.keySet())); + if (targetsByKey.size() != directDeps.size()) { + Iterable<ConfiguredTargetKey> missingTargets = + Sets.difference(targetsByKey.keySet(), directDeps.keySet()) + .stream() + .map(SKYKEY_TO_CTKEY) + .collect(Collectors.toList()); + eventHandler.handle(Event.warn("Targets were missing from graph: " + missingTargets)); + } + ThreadSafeMutableSet<T> result = createThreadSafeMutableSet(); + for (Map.Entry<SkyKey, Collection<T>> entry : directDeps.entrySet()) { + result.addAll(filterFwdDeps(targetsByKey.get(entry.getKey()), entry.getValue())); + } + return result; + } + + private Collection<T> filterFwdDeps(T configTarget, Collection<T> rawFwdDeps) { + if (settings.isEmpty()) { + return rawFwdDeps; + } + return getAllowedDeps(configTarget, rawFwdDeps); + } + + @Override + public Collection<T> getReverseDeps(Iterable<T> targets, QueryExpressionContext<T> context) + throws InterruptedException { + Map<SkyKey, T> targetsByKey = Maps.newHashMapWithExpectedSize(Iterables.size(targets)); + for (T target : targets) { + targetsByKey.put(getSkyKey(target), target); + } + Map<SkyKey, Collection<T>> reverseDepsByKey = + targetifyValues(graph.getReverseDeps(targetsByKey.keySet())); + if (targetsByKey.size() != reverseDepsByKey.size()) { + Iterable<ConfiguredTargetKey> missingTargets = + Sets.difference(targetsByKey.keySet(), reverseDepsByKey.keySet()) + .stream() + .map(SKYKEY_TO_CTKEY) + .collect(Collectors.toList()); + eventHandler.handle(Event.warn("Targets were missing from graph: " + missingTargets)); + } + Map<T, Collection<T>> reverseDepsByCT = new HashMap<>(); + for (Map.Entry<SkyKey, Collection<T>> entry : reverseDepsByKey.entrySet()) { + reverseDepsByCT.put(targetsByKey.get(entry.getKey()), entry.getValue()); + } + return reverseDepsByCT.isEmpty() ? Collections.emptyList() : filterReverseDeps(reverseDepsByCT); + } + + private Collection<T> filterReverseDeps(Map<T, Collection<T>> rawReverseDeps) { + Set<T> result = CompactHashSet.create(); + for (Map.Entry<T, Collection<T>> targetAndRdeps : rawReverseDeps.entrySet()) { + ImmutableSet.Builder<T> ruleDeps = ImmutableSet.builder(); + for (T parent : targetAndRdeps.getValue()) { + if (parent instanceof RuleConfiguredTarget + && dependencyFilter != DependencyFilter.ALL_DEPS) { + ruleDeps.add(parent); + } else { + result.add(parent); + } + } + result.addAll(getAllowedDeps((targetAndRdeps.getKey()), ruleDeps.build())); + } + return result; + } + + /** + * @param target source target + * @param deps next level of deps to filter + */ + protected Collection<T> getAllowedDeps(T target, Collection<T> deps) { + // It's possible to query on a target that's configured in the host configuration. In those + // cases if --nohost_deps is turned on, we only allow reachable targets that are ALSO in the + // host config. This is somewhat counterintuitive and subject to change in the future but seems + // like the best option right now. + if (settings.contains(Setting.NO_HOST_DEPS)) { + BuildConfiguration currentConfig = getConfiguration(target); + if (currentConfig != null && currentConfig.isHostConfiguration()) { + deps = + deps.stream() + .filter( + dep -> + getConfiguration(dep) != null + && getConfiguration(dep).isHostConfiguration()) + .collect(Collectors.toList()); + } else { + deps = + deps.stream() + .filter( + dep -> + getConfiguration(dep) != null + && !getConfiguration(dep).isHostConfiguration()) + .collect(Collectors.toList()); + } + } + if (settings.contains(Setting.NO_IMPLICIT_DEPS) && target instanceof RuleConfiguredTarget) { + Set<ConfiguredTargetKey> implicitDeps = ((RuleConfiguredTarget) target).getImplicitDeps(); + deps = + deps.stream() + .filter( + dep -> + !implicitDeps.contains( + ConfiguredTargetKey.of(getCorrectLabel(dep), getConfiguration(dep)))) + .collect(Collectors.toList()); + } + return deps; + } + + protected Map<SkyKey, Collection<T>> targetifyValues( + Map<SkyKey, ? extends Iterable<SkyKey>> input) throws InterruptedException { + Map<SkyKey, Collection<T>> result = new HashMap<>(); + for (Map.Entry<SkyKey, ? extends Iterable<SkyKey>> entry : input.entrySet()) { + Collection<T> value = new ArrayList<>(); + for (SkyKey key : entry.getValue()) { + if (key.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) { + value.add(getValueFromKey(key)); + } + } + result.put(entry.getKey(), value); + } + return result; + } + + @Nullable + protected abstract BuildConfiguration getConfiguration(T target); + + protected abstract ConfiguredTargetKey getSkyKey(T target); + + @Override + public ThreadSafeMutableSet<T> getTransitiveClosure( + ThreadSafeMutableSet<T> targets, QueryExpressionContext<T> context) + throws InterruptedException { + return SkyQueryUtils.getTransitiveClosure( + targets, targets1 -> getFwdDeps(targets1, context), createThreadSafeMutableSet()); + } + + @Override + public void buildTransitiveClosure( + QueryExpression caller, ThreadSafeMutableSet<T> targetNodes, int maxDepth) { + // TODO(bazel-team): implement this. Just needed for error-checking. + } + + @Override + public ImmutableList<T> getNodesOnPath(T from, T to, QueryExpressionContext<T> context) + throws InterruptedException { + return SkyQueryUtils.getNodesOnPath( + from, + to, + targets -> getFwdDeps(targets, context), + getConfiguredTargetKeyExtractor()::extractKey); + } + + @Override + public <V> MutableMap<T, V> createMutableMap() { + return new MutableKeyExtractorBackedMapImpl<>(getConfiguredTargetKeyExtractor()); + } + + @Override + public Uniquifier<T> createUniquifier() { + return new UniquifierImpl<>( + getConfiguredTargetKeyExtractor(), SkyQueryEnvironment.DEFAULT_THREAD_COUNT); + } + + @Override + public MinDepthUniquifier<T> createMinDepthUniquifier() { + return new MinDepthUniquifierImpl<>( + getConfiguredTargetKeyExtractor(), SkyQueryEnvironment.DEFAULT_THREAD_COUNT); + } + + /** Target patterns are resolved on the fly so no pre-work to be done here. */ + @Override + protected void preloadOrThrow(QueryExpression caller, Collection<String> patterns) {} + + @Override + public ThreadSafeMutableSet<T> getBuildFiles( + QueryExpression caller, + ThreadSafeMutableSet<T> nodes, + boolean buildFiles, + boolean loads, + QueryExpressionContext<T> context) + throws QueryException { + throw new QueryException("buildfiles() doesn't make sense for the configured target graph"); + } + + @Override + public Collection<T> getSiblingTargetsInPackage(T target) { + throw new UnsupportedOperationException("siblings() not supported"); + } + + @Override + public void close() {} +} |