// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.query2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AsyncFunction;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.cmdline.TargetPattern;
import com.google.devtools.build.lib.cmdline.TargetPattern.Type;
import com.google.devtools.build.lib.collect.compacthashset.CompactHashSet;
import com.google.devtools.build.lib.concurrent.MultisetSemaphore;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.packages.DependencyFilter;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.FilteringPolicies;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
import com.google.devtools.build.lib.query2.engine.Callback;
import com.google.devtools.build.lib.query2.engine.KeyExtractor;
import com.google.devtools.build.lib.query2.engine.MinDepthUniquifier;
import com.google.devtools.build.lib.query2.engine.QueryEnvironment;
import com.google.devtools.build.lib.query2.engine.QueryEvalResult;
import com.google.devtools.build.lib.query2.engine.QueryException;
import com.google.devtools.build.lib.query2.engine.QueryExpression;
import com.google.devtools.build.lib.query2.engine.QueryUtil.MinDepthUniquifierImpl;
import com.google.devtools.build.lib.query2.engine.QueryUtil.MutableKeyExtractorBackedMapImpl;
import com.google.devtools.build.lib.query2.engine.QueryUtil.ThreadSafeMutableKeyExtractorBackedSetImpl;
import com.google.devtools.build.lib.query2.engine.QueryUtil.UniquifierImpl;
import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback;
import com.google.devtools.build.lib.query2.engine.Uniquifier;
import com.google.devtools.build.lib.query2.output.QueryOptions;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue;
import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider;
import com.google.devtools.build.lib.skyframe.PackageValue;
import com.google.devtools.build.lib.skyframe.RecursivePackageProviderBackedTargetPatternResolver;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.skyframe.TargetPatternValue;
import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.WalkableGraph;
import com.google.devtools.common.options.OptionsParser;
import com.google.devtools.common.options.OptionsParsingException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* {@link QueryEnvironment} that runs queries over the configured target (analysis) graph.
*
*
This object can theoretically be used for multiple queries, but currently is only ever used
* for one over the course of its lifetime.
*
*
There is currently no way to specify a configuration in the query syntax. Instead, the default
* configuration that will be used for any raw labels is provided in the constructor of this
* environment. That will probably have to change.
*
*
On the other end, recursive target patterns are not supported.
*
*
Aspects are also not supported, but probably should be in some fashion.
*/
public class ConfiguredTargetQueryEnvironment
extends AbstractBlazeQueryEnvironment {
private final BuildConfiguration defaultTargetConfiguration;
private final BuildConfiguration hostConfiguration;
private final String parserPrefix;
protected final PathPackageLocator pkgPath;
private final Supplier walkableGraphSupplier;
private ConfiguredTargetAccessor accessor;
protected WalkableGraph graph;
private static final Function CT_TO_SKYKEY =
target -> ConfiguredTargetValue.key(target.getLabel(), target.getConfiguration());
private static final Function SKYKEY_TO_CTKEY =
skyKey -> (ConfiguredTargetKey) skyKey.argument();
private static final ImmutableList ALL_PATTERNS;
private static final KeyExtractor
CONFIGURED_TARGET_KEY_EXTRACTOR = ConfiguredTargetKey::of;
/** Common query functions and cquery specific functions. */
public static final ImmutableList FUNCTIONS = populateFunctions();
/** Cquery specific functions. */
public static final ImmutableList CQUERY_FUNCTIONS = getCqueryFunctions();
static {
TargetPattern targetPattern;
try {
targetPattern = TargetPattern.defaultParser().parse("//...");
} catch (TargetParsingException e) {
throw new IllegalStateException(e);
}
ALL_PATTERNS =
ImmutableList.of(
new TargetPatternKey(
targetPattern, FilteringPolicies.NO_FILTER, false, "", ImmutableSet.of()));
}
private RecursivePackageProviderBackedTargetPatternResolver resolver;
public ConfiguredTargetQueryEnvironment(
boolean keepGoing,
ExtendedEventHandler eventHandler,
Iterable extraFunctions,
BuildConfiguration defaultTargetConfiguration,
BuildConfiguration hostConfiguration,
String parserPrefix,
PathPackageLocator pkgPath,
Supplier walkableGraphSupplier,
Set settings) {
super(keepGoing, true, Rule.ALL_LABELS, eventHandler, settings, extraFunctions);
this.defaultTargetConfiguration = defaultTargetConfiguration;
this.hostConfiguration = hostConfiguration;
this.parserPrefix = parserPrefix;
this.pkgPath = pkgPath;
this.walkableGraphSupplier = walkableGraphSupplier;
}
private void beforeEvaluateQuery() throws InterruptedException, QueryException {
graph = walkableGraphSupplier.get();
GraphBackedRecursivePackageProvider graphBackedRecursivePackageProvider =
new GraphBackedRecursivePackageProvider(graph, ALL_PATTERNS, pkgPath);
resolver =
new RecursivePackageProviderBackedTargetPatternResolver(
graphBackedRecursivePackageProvider,
eventHandler,
FilteringPolicies.NO_FILTER,
MultisetSemaphore.unbounded());
accessor = new ConfiguredTargetAccessor(walkableGraphSupplier.get());
checkSettings(settings);
}
private static ImmutableList populateFunctions() {
return new ImmutableList.Builder()
.addAll(QueryEnvironment.DEFAULT_QUERY_FUNCTIONS)
.addAll(getCqueryFunctions())
.build();
}
private static ImmutableList getCqueryFunctions() {
return ImmutableList.of(new ConfigFunction());
}
// Check to make sure the settings requested are currently supported by this class
private void checkSettings(Set settings) throws QueryException {
if (settings.contains(Setting.NO_NODEP_DEPS)
|| settings.contains(Setting.TESTS_EXPRESSION_STRICT)) {
settings =
Sets.difference(
settings, ImmutableSet.of(Setting.NO_HOST_DEPS, Setting.NO_IMPLICIT_DEPS));
throw new QueryException(
String.format(
"The following filter(s) are not currently supported by configured query: %s",
settings.toString()));
}
}
@Nullable
private ConfiguredTarget getConfiguredTarget(SkyKey key) throws InterruptedException {
ConfiguredTargetValue value =
((ConfiguredTargetValue) walkableGraphSupplier.get().getValue(key));
return value == null ? null : value.getConfiguredTarget();
}
private ConfiguredTarget getConfiguredTarget(Label label) throws InterruptedException {
// Try with target configuration.
ConfiguredTarget configuredTarget = getTargetConfiguredTarget(label);
if (configuredTarget != null) {
return configuredTarget;
}
// Try with host configuration (even when --nohost_deps is set in the case that top-level
// targets are configured in the host configuration so we are doing a host-configuration-only
// query).
configuredTarget = getHostConfiguredTarget(label);
if (configuredTarget != null) {
return configuredTarget;
}
// Last chance: source file.
return getNullConfiguredTarget(label);
}
@Nullable
private ConfiguredTarget getHostConfiguredTarget(Label label) throws InterruptedException {
return getConfiguredTarget(ConfiguredTargetValue.key(label, hostConfiguration));
}
@Nullable
private ConfiguredTarget getTargetConfiguredTarget(Label label) throws InterruptedException {
return getConfiguredTarget(ConfiguredTargetValue.key(label, defaultTargetConfiguration));
}
@Nullable
private ConfiguredTarget getNullConfiguredTarget(Label label) throws InterruptedException {
return getConfiguredTarget(ConfiguredTargetValue.key(label, null));
}
@Override
public void close() {}
@Override
public QueryEvalResult evaluateQuery(
QueryExpression expr, ThreadSafeOutputFormatterCallback callback)
throws QueryException, InterruptedException, IOException {
beforeEvaluateQuery();
return super.evaluateQuery(expr, callback);
}
private TargetPattern getPattern(String pattern)
throws TargetParsingException, InterruptedException {
TargetPatternKey targetPatternKey =
((TargetPatternKey)
TargetPatternValue.key(
pattern, TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix)
.argument());
return targetPatternKey.getParsedPattern();
}
@Override
public Collection getSiblingTargetsInPackage(ConfiguredTarget target) {
throw new UnsupportedOperationException("siblings() not supported");
}
@Override
public QueryTaskFuture getTargetsMatchingPattern(
QueryExpression owner, String pattern, Callback callback) {
TargetPattern patternToEval;
try {
patternToEval = getPattern(pattern);
} catch (TargetParsingException tpe) {
try {
reportBuildFileError(owner, tpe.getMessage());
} catch (QueryException qe) {
return immediateFailedFuture(qe);
}
return immediateSuccessfulFuture(null);
} catch (InterruptedException ie) {
return immediateCancelledFuture();
}
AsyncFunction reportBuildFileErrorAsyncFunction =
exn -> {
reportBuildFileError(owner, exn.getMessage());
return Futures.immediateFuture(null);
};
return QueryTaskFutureImpl.ofDelegate(
Futures.catchingAsync(
patternToEval.evalAdaptedForAsync(
resolver,
ImmutableSet.of(),
ImmutableSet.of(),
(Callback)
partialResult -> {
List transformedResult = new ArrayList<>();
for (Target target : partialResult) {
ConfiguredTarget configuredTarget = getConfiguredTarget(target.getLabel());
if (configuredTarget != null) {
transformedResult.add(configuredTarget);
}
}
callback.process(transformedResult);
},
QueryException.class),
TargetParsingException.class,
reportBuildFileErrorAsyncFunction,
MoreExecutors.directExecutor()));
}
/**
* Processes the targets in {@code targets} with the requested {@code configuration}
*
* @param pattern the original pattern that {@code targets} were parsed from. Used for error
* message.
* @param targets the set of {@link ConfiguredTarget}s whose labels represent the targets being
* requested.
* @param configuration the configuration to request {@code targets} in.
* @param callback the callback to receive the results of this method.
* @return {@link QueryTaskCallable} that returns the correctly configured targets.
*/
QueryTaskCallable getConfiguredTargets(
String pattern,
ThreadSafeMutableSet targets,
String configuration,
Callback callback) {
return new QueryTaskCallable() {
@Override
public Void call() throws QueryException, InterruptedException {
List transformedResult = new ArrayList<>();
for (ConfiguredTarget target : targets) {
Label label = target.getLabel();
ConfiguredTarget configuredTarget;
switch (configuration) {
case "\'host\'":
configuredTarget = getHostConfiguredTarget(label);
break;
case "\'target\'":
configuredTarget = getTargetConfiguredTarget(label);
break;
case "\'null\'":
configuredTarget = getNullConfiguredTarget(label);
break;
default:
throw new QueryException(
"the second argument of the config function must be 'target', 'host', or 'null'");
}
if (configuredTarget != null) {
transformedResult.add(configuredTarget);
}
}
if (transformedResult.isEmpty()) {
throw new QueryException(
"No target (in) "
+ pattern
+ " could be found in the "
+ configuration
+ " configuration");
}
callback.process(transformedResult);
return null;
}
};
}
@Override
public ConfiguredTarget getOrCreate(ConfiguredTarget target) {
return target;
}
private Map> targetifyValues(
Map> input) throws InterruptedException {
Map> result = new HashMap<>();
for (Map.Entry> entry : input.entrySet()) {
Collection value = new ArrayList<>();
for (SkyKey key : entry.getValue()) {
if (key.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) {
value.add(getConfiguredTarget(key));
}
}
result.put(entry.getKey(), value);
}
return result;
}
private Collection filterFwdDeps(
ConfiguredTarget configTarget, Collection rawFwdDeps)
throws InterruptedException {
if (!(configTarget instanceof RuleConfiguredTarget) || settings.isEmpty()) {
return rawFwdDeps;
}
return getAllowedDeps(configTarget, rawFwdDeps);
}
/**
* @param target source target
* @param deps next level of deps to filter
*/
private Collection getAllowedDeps(
ConfiguredTarget target, Collection deps) {
// It's possible to query on a target that's configured in the host configuration. In those
// cases if --nohost_deps is turned on, we only allow reachable targets that are ALSO in the
// host config. This is somewhat counterintuitive and subject to change in the future but seems
// like the best option right now.
if (settings.contains(Setting.NO_HOST_DEPS)) {
BuildConfiguration currentConfig = target.getConfiguration();
if (currentConfig != null && currentConfig.isHostConfiguration()) {
deps =
deps.stream()
.filter(
dep ->
dep.getConfiguration() != null
&& dep.getConfiguration().isHostConfiguration())
.collect(Collectors.toList());
} else {
deps =
deps.stream()
.filter(
dep ->
dep.getConfiguration() == null
|| !dep.getConfiguration().isHostConfiguration())
.collect(Collectors.toList());
}
}
if (settings.contains(Setting.NO_IMPLICIT_DEPS) && target instanceof RuleConfiguredTarget) {
Set implicitDeps = ((RuleConfiguredTarget) target).getImplicitDeps();
deps =
deps.stream()
.filter(
dep ->
!implicitDeps.contains(
ConfiguredTargetKey.of(
dep.getLabel(), dep.getConfiguration())))
.collect(Collectors.toList());
}
return deps;
}
@Override
public ThreadSafeMutableSet getFwdDeps(Iterable targets)
throws InterruptedException {
Map targetsByKey = new HashMap<>(Iterables.size(targets));
for (ConfiguredTarget target : targets) {
targetsByKey.put(CT_TO_SKYKEY.apply(target), target);
}
Map> directDeps =
targetifyValues(graph.getDirectDeps(targetsByKey.keySet()));
if (targetsByKey.keySet().size() != directDeps.keySet().size()) {
Iterable missingTargets =
Sets.difference(targetsByKey.keySet(), directDeps.keySet())
.stream()
.map(SKYKEY_TO_CTKEY)
.collect(Collectors.toList());
eventHandler.handle(Event.warn("Targets were missing from graph: " + missingTargets));
}
ThreadSafeMutableSet result = createThreadSafeMutableSet();
for (Entry> entry : directDeps.entrySet()) {
result.addAll(filterFwdDeps(targetsByKey.get(entry.getKey()), entry.getValue()));
}
return result;
}
private Collection filterReverseDeps(
Map> rawReverseDeps) {
Set result = CompactHashSet.create();
for (Map.Entry> targetAndRdeps :
rawReverseDeps.entrySet()) {
ImmutableSet.Builder ruleDeps = ImmutableSet.builder();
for (ConfiguredTarget parent : targetAndRdeps.getValue()) {
if (parent instanceof RuleConfiguredTarget
&& dependencyFilter != DependencyFilter.ALL_DEPS) {
ruleDeps.add(parent);
} else {
result.add(parent);
}
}
result.addAll(getAllowedDeps((targetAndRdeps.getKey()), ruleDeps.build()));
}
return result;
}
@Override
public Collection getReverseDeps(Iterable targets)
throws InterruptedException {
Map targetsByKey = new HashMap<>(Iterables.size(targets));
for (ConfiguredTarget target : targets) {
targetsByKey.put(CT_TO_SKYKEY.apply(target), target);
}
Map> reverseDepsByKey =
targetifyValues(graph.getReverseDeps(targetsByKey.keySet()));
if (targetsByKey.size() != reverseDepsByKey.size()) {
Iterable missingTargets =
Sets.difference(targetsByKey.keySet(), reverseDepsByKey.keySet())
.stream()
.map(SKYKEY_TO_CTKEY)
.collect(Collectors.toList());
eventHandler.handle(Event.warn("Targets were missing from graph: " + missingTargets));
}
Map> reverseDepsByCT = new HashMap<>();
for (Map.Entry> entry : reverseDepsByKey.entrySet()) {
reverseDepsByCT.put(targetsByKey.get(entry.getKey()), entry.getValue());
}
return reverseDepsByCT.isEmpty() ? Collections.emptyList() : filterReverseDeps(reverseDepsByCT);
}
@Override
public ThreadSafeMutableSet getTransitiveClosure(
ThreadSafeMutableSet targets) throws InterruptedException {
return SkyQueryUtils.getTransitiveClosure(
targets, this::getFwdDeps, createThreadSafeMutableSet());
}
@Override
public void buildTransitiveClosure(
QueryExpression caller, ThreadSafeMutableSet targetNodes, int maxDepth)
throws QueryException, InterruptedException {
// TODO(bazel-team): implement this. Just needed for error-checking.
}
@Override
public ImmutableList getNodesOnPath(ConfiguredTarget from, ConfiguredTarget to)
throws InterruptedException {
return SkyQueryUtils.getNodesOnPath(from, to, this::getFwdDeps, ConfiguredTargetKey::of);
}
@Override
public TargetAccessor getAccessor() {
return accessor;
}
// TODO(bazel-team): It's weird that this untemplated function exists. Fix? Or don't implement?
@Override
public Target getTarget(Label label)
throws TargetNotFoundException, QueryException, InterruptedException {
try {
return ((PackageValue)
walkableGraphSupplier.get().getValue(PackageValue.key(label.getPackageIdentifier())))
.getPackage()
.getTarget(label.getName());
} catch (NoSuchTargetException e) {
throw new TargetNotFoundException(e);
}
}
@Override
public ThreadSafeMutableSet createThreadSafeMutableSet() {
return new ThreadSafeMutableKeyExtractorBackedSetImpl<>(
CONFIGURED_TARGET_KEY_EXTRACTOR,
ConfiguredTarget.class,
SkyQueryEnvironment.DEFAULT_THREAD_COUNT);
}
@Override
public MutableMap createMutableMap() {
return new MutableKeyExtractorBackedMapImpl<>(CONFIGURED_TARGET_KEY_EXTRACTOR);
}
@Override
public Uniquifier createUniquifier() {
return new UniquifierImpl<>(
CONFIGURED_TARGET_KEY_EXTRACTOR, SkyQueryEnvironment.DEFAULT_THREAD_COUNT);
}
@Override
public MinDepthUniquifier createMinDepthUniquifier() {
return new MinDepthUniquifierImpl<>(
CONFIGURED_TARGET_KEY_EXTRACTOR, SkyQueryEnvironment.DEFAULT_THREAD_COUNT);
}
@Override
public ThreadSafeMutableSet getBuildFiles(
QueryExpression caller,
ThreadSafeMutableSet nodes,
boolean buildFiles,
boolean subincludes,
boolean loads)
throws QueryException, InterruptedException {
throw new QueryException("buildfiles() doesn't make sense for the configured target graph");
}
@Override
protected void preloadOrThrow(QueryExpression caller, Collection patterns)
throws QueryException, TargetParsingException, InterruptedException {
for (String pattern : patterns) {
if (TargetPattern.defaultParser()
.parse(pattern)
.getType()
.equals(Type.TARGETS_BELOW_DIRECTORY)) {
// TODO(bazel-team): allow recursive patterns if the pattern is present in the graph? We
// could do a mini-eval here to update the graph to contain the necessary nodes for
// GraphBackedRecursivePackageProvider, since all the package loading and directory
// traversal should already be done.
throw new QueryException(
"Recursive pattern '" + pattern + "' is not supported in configured target query");
}
}
}
public static QueryOptions parseOptions(String rawOptions) throws QueryException {
List options = new ArrayList<>(Arrays.asList(rawOptions.split(" ")));
OptionsParser parser = OptionsParser.newOptionsParser(QueryOptions.class);
parser.setAllowResidue(false);
try {
parser.parse(options);
} catch (OptionsParsingException e) {
throw new QueryException(e.getMessage());
}
return parser.getOptions(QueryOptions.class);
}
}