diff options
author | shreyax <shreyax@google.com> | 2018-06-12 15:34:09 -0700 |
---|---|---|
committer | Copybara-Service <copybara-piper@google.com> | 2018-06-12 15:35:36 -0700 |
commit | 159a61134e49c36b7839b22a886809df4e378ee7 (patch) | |
tree | 4de50bf86cd51dd9916674bf99256704e2372ef4 /src/main/java/com/google/devtools/build/lib/query2 | |
parent | 686de84ea61bbc58277e6023222c8230f69d54d5 (diff) |
Rename VariableContext to the more general purpose QueryExpressionContext and thread it through to graph traversal functions. Some other light refactorings as well.
PiperOrigin-RevId: 200292556
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/query2')
33 files changed, 246 insertions, 222 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/query2/AbstractBlazeQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/AbstractBlazeQueryEnvironment.java index 0221aacd75..a378f55e85 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/AbstractBlazeQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/AbstractBlazeQueryEnvironment.java @@ -32,10 +32,10 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment; import com.google.devtools.build.lib.query2.engine.QueryEvalResult; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.QueryUtil; import com.google.devtools.build.lib.query2.engine.QueryUtil.AggregateAllCallback; import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; -import com.google.devtools.build.lib.query2.engine.VariableContext; import java.io.IOException; import java.util.Collection; import java.util.LinkedHashSet; @@ -104,7 +104,11 @@ public abstract class AbstractBlazeQueryEnvironment<T> extends AbstractQueryEnvi */ protected void evalTopLevelInternal(QueryExpression expr, OutputFormatterCallback<T> callback) throws QueryException, InterruptedException { - ((QueryTaskFutureImpl<Void>) eval(expr, VariableContext.<T>empty(), callback)).getChecked(); + ((QueryTaskFutureImpl<Void>) eval(expr, createEmptyContext(), callback)).getChecked(); + } + + protected QueryExpressionContext<T> createEmptyContext() { + return QueryExpressionContext.empty(); } /** diff --git a/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java index 62cc674d31..0b448abe25 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/BlazeQueryEnvironment.java @@ -44,6 +44,7 @@ import com.google.devtools.build.lib.query2.engine.MinDepthUniquifier; import com.google.devtools.build.lib.query2.engine.QueryEvalResult; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.QueryUtil.MinDepthUniquifierImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.MutableKeyExtractorBackedMapImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.ThreadSafeMutableKeyExtractorBackedSetImpl; @@ -243,7 +244,8 @@ public class BlazeQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } @Override - public Collection<Target> getFwdDeps(Iterable<Target> targets) { + public Collection<Target> getFwdDeps( + Iterable<Target> targets, QueryExpressionContext<Target> context) { ThreadSafeMutableSet<Target> result = createThreadSafeMutableSet(); for (Target target : targets) { result.addAll(getTargetsFromNodes(getNode(target).getSuccessors())); @@ -252,7 +254,8 @@ public class BlazeQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } @Override - public Collection<Target> getReverseDeps(Iterable<Target> targets) { + public Collection<Target> getReverseDeps( + Iterable<Target> targets, QueryExpressionContext<Target> context) { ThreadSafeMutableSet<Target> result = createThreadSafeMutableSet(); for (Target target : targets) { result.addAll(getTargetsFromNodes(getNode(target).getPredecessors())); @@ -262,7 +265,7 @@ public class BlazeQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> @Override public ThreadSafeMutableSet<Target> getTransitiveClosure( - ThreadSafeMutableSet<Target> targetNodes) { + ThreadSafeMutableSet<Target> targetNodes, QueryExpressionContext<Target> context) { for (Target node : targetNodes) { checkBuilt(node); } @@ -301,7 +304,8 @@ public class BlazeQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } @Override - public Iterable<Target> getNodesOnPath(Target from, Target to) { + public Iterable<Target> getNodesOnPath( + Target from, Target to, QueryExpressionContext<Target> context) { ImmutableList.Builder<Target> builder = ImmutableList.builder(); for (Node<Target> node : graph.getShortestPath(getNode(from), getNode(to))) { builder.add(node.getLabel()); @@ -389,7 +393,8 @@ public class BlazeQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> final QueryExpression caller, ThreadSafeMutableSet<Target> nodes, boolean buildFiles, - boolean loads) + boolean loads, + QueryExpressionContext<Target> context) throws QueryException { ThreadSafeMutableSet<Target> dependentFiles = createThreadSafeMutableSet(); Set<PackageIdentifier> seenPackages = new HashSet<>(); diff --git a/src/main/java/com/google/devtools/build/lib/query2/ConfigFunction.java b/src/main/java/com/google/devtools/build/lib/query2/ConfigFunction.java index 7a3f6a0b63..20341fceb0 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/ConfigFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/ConfigFunction.java @@ -23,8 +23,8 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunctio import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryTaskFuture; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ThreadSafeMutableSet; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.QueryUtil; -import com.google.devtools.build.lib.query2.engine.VariableContext; import java.util.List; /** @@ -64,7 +64,7 @@ public final class ConfigFunction implements QueryFunction { @SuppressWarnings("unchecked") public <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, List<Argument> args, final Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java index d709e55c18..d9662e24ff 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/ConfiguredTargetQueryEnvironment.java @@ -47,6 +47,7 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment; import com.google.devtools.build.lib.query2.engine.QueryEvalResult; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.QueryUtil.MinDepthUniquifierImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.MutableKeyExtractorBackedMapImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.ThreadSafeMutableKeyExtractorBackedSetImpl; @@ -455,7 +456,8 @@ public class ConfiguredTargetQueryEnvironment } @Override - public ThreadSafeMutableSet<ConfiguredTarget> getFwdDeps(Iterable<ConfiguredTarget> targets) + public ThreadSafeMutableSet<ConfiguredTarget> getFwdDeps( + Iterable<ConfiguredTarget> targets, QueryExpressionContext<ConfiguredTarget> context) throws InterruptedException { Map<SkyKey, ConfiguredTarget> targetsByKey = new HashMap<>(Iterables.size(targets)); for (ConfiguredTarget target : targets) { @@ -488,7 +490,8 @@ public class ConfiguredTargetQueryEnvironment } @Override - public Collection<ConfiguredTarget> getReverseDeps(Iterable<ConfiguredTarget> targets) + public Collection<ConfiguredTarget> getReverseDeps( + Iterable<ConfiguredTarget> targets, QueryExpressionContext<ConfiguredTarget> context) throws InterruptedException { Map<SkyKey, ConfiguredTarget> targetsByKey = new HashMap<>(Iterables.size(targets)); for (ConfiguredTarget target : targets) { @@ -604,12 +607,13 @@ public class ConfiguredTargetQueryEnvironment return ConfiguredTargetKey.of(target, getConfiguration(target)); } - @Override public ThreadSafeMutableSet<ConfiguredTarget> getTransitiveClosure( - ThreadSafeMutableSet<ConfiguredTarget> targets) throws InterruptedException { + ThreadSafeMutableSet<ConfiguredTarget> targets, + QueryExpressionContext<ConfiguredTarget> context) + throws InterruptedException { return SkyQueryUtils.getTransitiveClosure( - targets, this::getFwdDeps, createThreadSafeMutableSet()); + targets, targets1 -> getFwdDeps(targets1, context), createThreadSafeMutableSet()); } @Override @@ -620,10 +624,14 @@ public class ConfiguredTargetQueryEnvironment } @Override - public ImmutableList<ConfiguredTarget> getNodesOnPath(ConfiguredTarget from, ConfiguredTarget to) + public ImmutableList<ConfiguredTarget> getNodesOnPath( + ConfiguredTarget from, ConfiguredTarget to, QueryExpressionContext<ConfiguredTarget> context) throws InterruptedException { return SkyQueryUtils.getNodesOnPath( - from, to, this::getFwdDeps, configuredTargetKeyExtractor::extractKey); + from, + to, + targets -> getFwdDeps(targets, context), + configuredTargetKeyExtractor::extractKey); } @Override @@ -672,7 +680,8 @@ public class ConfiguredTargetQueryEnvironment QueryExpression caller, ThreadSafeMutableSet<ConfiguredTarget> nodes, boolean buildFiles, - boolean loads) + boolean loads, + QueryExpressionContext<ConfiguredTarget> context) throws QueryException, InterruptedException { throw new QueryException("buildfiles() doesn't make sense for the configured target graph"); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/DepsUnboundedVisitor.java b/src/main/java/com/google/devtools/build/lib/query2/DepsUnboundedVisitor.java index 55adaacdfe..cb5e19c196 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/DepsUnboundedVisitor.java +++ b/src/main/java/com/google/devtools/build/lib/query2/DepsUnboundedVisitor.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.query2.engine.Callback; import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.Uniquifier; import com.google.devtools.build.skyframe.SkyKey; import java.util.Map; @@ -47,6 +48,7 @@ class DepsUnboundedVisitor extends AbstractEdgeVisitor<SkyKey> { private final boolean depsNeedFiltering; private final Callback<Target> errorReporter; + private final QueryExpressionContext<Target> context; DepsUnboundedVisitor( SkyQueryEnvironment env, @@ -54,11 +56,13 @@ class DepsUnboundedVisitor extends AbstractEdgeVisitor<SkyKey> { Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore, boolean depsNeedFiltering, - Callback<Target> errorReporter) { + Callback<Target> errorReporter, + QueryExpressionContext<Target> context) { super(env, callback, packageSemaphore); this.validDepUniquifier = validDepUniquifier; this.depsNeedFiltering = depsNeedFiltering; this.errorReporter = errorReporter; + this.context = context; } /** @@ -74,25 +78,34 @@ class DepsUnboundedVisitor extends AbstractEdgeVisitor<SkyKey> { private final MultisetSemaphore<PackageIdentifier> packageSemaphore; private final boolean depsNeedFiltering; private final Callback<Target> errorReporter; + private final QueryExpressionContext<Target> context; Factory( SkyQueryEnvironment env, Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore, boolean depsNeedFiltering, - Callback<Target> errorReporter) { + Callback<Target> errorReporter, + QueryExpressionContext<Target> context) { this.env = env; this.validDepUniquifier = env.createSkyKeyUniquifier(); this.callback = callback; this.packageSemaphore = packageSemaphore; this.depsNeedFiltering = depsNeedFiltering; this.errorReporter = errorReporter; + this.context = context; } @Override public ParallelVisitor<SkyKey, Target> create() { return new DepsUnboundedVisitor( - env, validDepUniquifier, callback, packageSemaphore, depsNeedFiltering, errorReporter); + env, + validDepUniquifier, + callback, + packageSemaphore, + depsNeedFiltering, + errorReporter, + context); } } @@ -110,7 +123,7 @@ class DepsUnboundedVisitor extends AbstractEdgeVisitor<SkyKey> { packageSemaphore.acquireAll(pkgIdsNeededForTargetification); Iterable<Target> deps; try { - deps = env.getFwdDeps(env.makeTargetsFromSkyKeys(keys).values()); + deps = env.getFwdDeps(env.makeTargetsFromSkyKeys(keys).values(), context); } finally { packageSemaphore.releaseAll(pkgIdsNeededForTargetification); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/ParallelSkyQueryUtils.java b/src/main/java/com/google/devtools/build/lib/query2/ParallelSkyQueryUtils.java index 1fe3bbf97f..c95a653e29 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/ParallelSkyQueryUtils.java +++ b/src/main/java/com/google/devtools/build/lib/query2/ParallelSkyQueryUtils.java @@ -28,10 +28,10 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryTaskFut import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ThreadSafeMutableSet; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.QueryUtil; import com.google.devtools.build.lib.query2.engine.QueryUtil.AggregateAllCallback; import com.google.devtools.build.lib.query2.engine.Uniquifier; -import com.google.devtools.build.lib.query2.engine.VariableContext; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyKey; import java.util.Collection; @@ -61,7 +61,7 @@ public class ParallelSkyQueryUtils { static QueryTaskFuture<Void> getAllRdepsUnboundedParallel( SkyQueryEnvironment env, QueryExpression expression, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore) { return env.eval( @@ -79,7 +79,7 @@ public class ParallelSkyQueryUtils { SkyQueryEnvironment env, QueryExpression expression, int depth, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore) { return env.eval( @@ -98,7 +98,7 @@ public class ParallelSkyQueryUtils { SkyQueryEnvironment env, QueryExpression expression, Predicate<SkyKey> universe, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore) { return env.eval( @@ -111,7 +111,7 @@ public class ParallelSkyQueryUtils { static QueryTaskFuture<Predicate<SkyKey>> getDTCSkyKeyPredicateFuture( SkyQueryEnvironment env, QueryExpression expression, - VariableContext<Target> context, + QueryExpressionContext<Target> context, int processResultsBatchSize, int concurrencyLevel) { QueryTaskFuture<ThreadSafeMutableSet<Target>> universeValueFuture = @@ -144,7 +144,7 @@ public class ParallelSkyQueryUtils { QueryExpression expression, int depth, Predicate<SkyKey> universe, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore) { return env.eval( @@ -173,7 +173,7 @@ public class ParallelSkyQueryUtils { static QueryTaskFuture<Void> getDepsUnboundedParallel( SkyQueryEnvironment env, QueryExpression expression, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback, MultisetSemaphore<PackageIdentifier> packageSemaphore, boolean depsNeedFiltering, @@ -183,7 +183,7 @@ public class ParallelSkyQueryUtils { context, ParallelVisitor.createParallelVisitorCallback( new DepsUnboundedVisitor.Factory( - env, callback, packageSemaphore, depsNeedFiltering, errorReporter))); + env, callback, packageSemaphore, depsNeedFiltering, errorReporter, context))); } static class DepAndRdep { diff --git a/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java index 3cffb9a1ca..7dfb1b841f 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java @@ -23,7 +23,7 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunctio import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryTaskFuture; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; -import com.google.devtools.build.lib.query2.engine.VariableContext; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.List; import java.util.stream.Collectors; @@ -60,7 +60,7 @@ public class RBuildFilesFunction implements QueryFunction { @SuppressWarnings("unchecked") // Cast from <Target> to <T>. This will only be used with <Target>. public <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, List<Argument> args, Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/RdepsToAllRdepsQueryExpressionMapper.java b/src/main/java/com/google/devtools/build/lib/query2/RdepsToAllRdepsQueryExpressionMapper.java new file mode 100644 index 0000000000..3146ef6654 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/query2/RdepsToAllRdepsQueryExpressionMapper.java @@ -0,0 +1,58 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.query2; + +import com.google.devtools.build.lib.cmdline.TargetPattern; +import com.google.devtools.build.lib.query2.engine.AllRdepsFunction; +import com.google.devtools.build.lib.query2.engine.FunctionExpression; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionMapper; +import com.google.devtools.build.lib.query2.engine.RdepsFunction; +import com.google.devtools.build.lib.query2.engine.TargetLiteral; +import java.util.List; + +/** + * A {@link QueryExpressionMapper} that transforms each occurrence of an expression of the form + * {@literal 'rdeps(<universeScope>, <T>)'} to {@literal 'allrdeps(<T>)'}. The latter is more + * efficient. + */ +class RdepsToAllRdepsQueryExpressionMapper extends QueryExpressionMapper<Void> { + private final TargetPattern.Parser targetPatternParser; + private final String absoluteUniverseScopePattern; + + RdepsToAllRdepsQueryExpressionMapper( + TargetPattern.Parser targetPatternParser, String universeScopePattern) { + this.targetPatternParser = targetPatternParser; + this.absoluteUniverseScopePattern = targetPatternParser.absolutize(universeScopePattern); + } + + @Override + public QueryExpression visit(FunctionExpression functionExpression, Void context) { + if (functionExpression.getFunction().getName().equals(new RdepsFunction().getName())) { + List<Argument> args = functionExpression.getArgs(); + QueryExpression universeExpression = args.get(0).getExpression(); + if (universeExpression instanceof TargetLiteral) { + TargetLiteral literalUniverseExpression = (TargetLiteral) universeExpression; + String absolutizedUniverseExpression = + targetPatternParser.absolutize(literalUniverseExpression.getPattern()); + if (absolutizedUniverseExpression.equals(absoluteUniverseScopePattern)) { + List<Argument> argsTail = args.subList(1, functionExpression.getArgs().size()); + return new FunctionExpression(new AllRdepsFunction(), argsTail); + } + } + } + return super.visit(functionExpression, context); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java index 382a8d2f73..a53f1d610d 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/SkyQueryEnvironment.java @@ -61,31 +61,26 @@ import com.google.devtools.build.lib.packages.NoSuchThingException; import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Target; -import com.google.devtools.build.lib.pkgcache.FilteringPolicy; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; -import com.google.devtools.build.lib.pkgcache.RecursivePackageProvider; import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; import com.google.devtools.build.lib.profiler.AutoProfiler; import com.google.devtools.build.lib.query2.engine.AllRdepsFunction; import com.google.devtools.build.lib.query2.engine.Callback; -import com.google.devtools.build.lib.query2.engine.FunctionExpression; import com.google.devtools.build.lib.query2.engine.KeyExtractor; import com.google.devtools.build.lib.query2.engine.MinDepthUniquifier; import com.google.devtools.build.lib.query2.engine.OutputFormatterCallback; import com.google.devtools.build.lib.query2.engine.QueryEvalResult; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.QueryExpressionContext; import com.google.devtools.build.lib.query2.engine.QueryExpressionMapper; import com.google.devtools.build.lib.query2.engine.QueryUtil.MinDepthUniquifierImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.MutableKeyExtractorBackedMapImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.ThreadSafeMutableKeyExtractorBackedSetImpl; import com.google.devtools.build.lib.query2.engine.QueryUtil.UniquifierImpl; -import com.google.devtools.build.lib.query2.engine.RdepsFunction; import com.google.devtools.build.lib.query2.engine.StreamableQueryEnvironment; -import com.google.devtools.build.lib.query2.engine.TargetLiteral; import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; import com.google.devtools.build.lib.query2.engine.Uniquifier; -import com.google.devtools.build.lib.query2.engine.VariableContext; import com.google.devtools.build.lib.skyframe.BlacklistedPackagePrefixesValue; import com.google.devtools.build.lib.skyframe.ContainingPackageLookupFunction; import com.google.devtools.build.lib.skyframe.GraphBackedRecursivePackageProvider; @@ -270,22 +265,13 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> new ThreadFactoryBuilder().setNameFormat("QueryEnvironment %d").build())); } resolver = - createTargetPatternResolver( + new RecursivePackageProviderBackedTargetPatternResolver( graphBackedRecursivePackageProvider, eventHandler, TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, packageSemaphore); } - protected RecursivePackageProviderBackedTargetPatternResolver createTargetPatternResolver( - RecursivePackageProvider graphBackedRecursivePackageProvider, - ExtendedEventHandler eventHandler, - FilteringPolicy policy, - MultisetSemaphore<PackageIdentifier> packageSemaphore) { - return new RecursivePackageProviderBackedTargetPatternResolver( - graphBackedRecursivePackageProvider, eventHandler, policy, packageSemaphore); - } - protected MultisetSemaphore<PackageIdentifier> makeFreshPackageMultisetSemaphore() { return MultisetSemaphore.unbounded(); } @@ -320,41 +306,6 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } } - /** - * A {@link QueryExpressionMapper} that transforms each occurrence of an expression of the form - * {@literal 'rdeps(<universeScope>, <T>)'} to {@literal 'allrdeps(<T>)'}. The latter is more - * efficient. - */ - protected static class RdepsToAllRdepsQueryExpressionMapper extends QueryExpressionMapper<Void> { - protected final TargetPattern.Parser targetPatternParser; - private final String absoluteUniverseScopePattern; - - protected RdepsToAllRdepsQueryExpressionMapper( - TargetPattern.Parser targetPatternParser, - String universeScopePattern) { - this.targetPatternParser = targetPatternParser; - this.absoluteUniverseScopePattern = targetPatternParser.absolutize(universeScopePattern); - } - - @Override - public QueryExpression visit(FunctionExpression functionExpression, Void context) { - if (functionExpression.getFunction().getName().equals(new RdepsFunction().getName())) { - List<Argument> args = functionExpression.getArgs(); - QueryExpression universeExpression = args.get(0).getExpression(); - if (universeExpression instanceof TargetLiteral) { - TargetLiteral literalUniverseExpression = (TargetLiteral) universeExpression; - String absolutizedUniverseExpression = - targetPatternParser.absolutize(literalUniverseExpression.getPattern()); - if (absolutizedUniverseExpression.equals(absoluteUniverseScopePattern)) { - List<Argument> argsTail = args.subList(1, functionExpression.getArgs().size()); - return new FunctionExpression(new AllRdepsFunction(), argsTail); - } - } - } - return super.visit(functionExpression, context); - } - } - @Override public final QueryExpression transformParsedQuery(QueryExpression queryExpression) { QueryExpressionMapper<Void> mapper = getQueryExpressionMapper(); @@ -482,7 +433,8 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } @Override - public ThreadSafeMutableSet<Target> getFwdDeps(Iterable<Target> targets) + public ThreadSafeMutableSet<Target> getFwdDeps( + Iterable<Target> targets, QueryExpressionContext<Target> context) throws InterruptedException { Map<SkyKey, Target> targetsByKey = Maps.newHashMapWithExpectedSize(Iterables.size(targets)); for (Target target : targets) { @@ -519,7 +471,9 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } @Override - public Collection<Target> getReverseDeps(Iterable<Target> targets) throws InterruptedException { + public Collection<Target> getReverseDeps( + Iterable<Target> targets, QueryExpressionContext<Target> context) + throws InterruptedException { return getReverseDepsOfTransitiveTraversalKeys(Iterables.transform(targets, TARGET_TO_SKY_KEY)); } @@ -571,16 +525,18 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } @Override - public ThreadSafeMutableSet<Target> getTransitiveClosure(ThreadSafeMutableSet<Target> targets) + public ThreadSafeMutableSet<Target> getTransitiveClosure( + ThreadSafeMutableSet<Target> targets, QueryExpressionContext<Target> context) throws InterruptedException { return SkyQueryUtils.getTransitiveClosure( - targets, this::getFwdDeps, createThreadSafeMutableSet()); + targets, targets1 -> getFwdDeps(targets1, context), createThreadSafeMutableSet()); } @Override - public ImmutableList<Target> getNodesOnPath(Target from, Target to) - throws InterruptedException { - return SkyQueryUtils.getNodesOnPath(from, to, this::getFwdDeps, Target::getLabel); + public ImmutableList<Target> getNodesOnPath( + Target from, Target to, QueryExpressionContext<Target> context) throws InterruptedException { + return SkyQueryUtils.getNodesOnPath( + from, to, targets -> getFwdDeps(targets, context), Target::getLabel); } private <R> ListenableFuture<R> safeSubmit(Callable<R> callable) { @@ -603,7 +559,7 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> @Override public QueryTaskFuture<Void> eval( final QueryExpression expr, - final VariableContext<Target> context, + final QueryExpressionContext<Target> context, final Callback<Target> callback) { // TODO(bazel-team): As in here, use concurrency for the async #eval of other QueryEnvironment // implementations. @@ -739,7 +695,11 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> @ThreadSafe @Override public ThreadSafeMutableSet<Target> getBuildFiles( - QueryExpression caller, ThreadSafeMutableSet<Target> nodes, boolean buildFiles, boolean loads) + QueryExpression caller, + ThreadSafeMutableSet<Target> nodes, + boolean buildFiles, + boolean loads, + QueryExpressionContext<Target> context) throws QueryException, InterruptedException { ThreadSafeMutableSet<Target> dependentFiles = createThreadSafeMutableSet(); Set<PackageIdentifier> seenPackages = new HashSet<>(); @@ -1220,7 +1180,7 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> @Override public QueryTaskFuture<Void> getAllRdepsUnboundedParallel( QueryExpression expression, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback) { return ParallelSkyQueryUtils.getAllRdepsUnboundedParallel( this, expression, context, callback, packageSemaphore); @@ -1231,15 +1191,14 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> public QueryTaskFuture<Void> getAllRdepsBoundedParallel( QueryExpression expression, int depth, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback) { return ParallelSkyQueryUtils.getAllRdepsBoundedParallel( this, expression, depth, context, callback, packageSemaphore); } protected QueryTaskFuture<Predicate<SkyKey>> getUniverseDTCSkyKeyPredicateFuture( - QueryExpression universe, - VariableContext<Target> context) { + QueryExpression universe, QueryExpressionContext<Target> context) { return ParallelSkyQueryUtils.getDTCSkyKeyPredicateFuture( this, universe, @@ -1253,7 +1212,7 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> public QueryTaskFuture<Void> getRdepsUnboundedParallel( QueryExpression expression, QueryExpression universe, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback) { return transformAsync( getUniverseDTCSkyKeyPredicateFuture(universe, context), @@ -1264,7 +1223,7 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> @Override public QueryTaskFuture<Void> getDepsUnboundedParallel( QueryExpression expression, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback, Callback<Target> errorReporter) { return ParallelSkyQueryUtils.getDepsUnboundedParallel( @@ -1283,7 +1242,7 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> QueryExpression expression, int depth, QueryExpression universe, - VariableContext<Target> context, + QueryExpressionContext<Target> context, Callback<Target> callback) { return transformAsync( getUniverseDTCSkyKeyPredicateFuture(universe, context), @@ -1318,32 +1277,4 @@ public class SkyQueryEnvironment extends AbstractBlazeQueryEnvironment<Target> } } } - - /** Pair of a key and a depth, useful for driving usages of {@link MinDepthUniquifier}. */ - public static class KeyAtDepth { - public final SkyKey key; - public final int depth; - - public KeyAtDepth(SkyKey key, int depth) { - this.key = key; - this.depth = depth; - } - - @Override - public int hashCode() { - // N.B. - We deliberately use a garbage-free hashCode implementation (rather than e.g. - // Objects#hash). This method is very hot during large visitations done by - // ParallelSkyQueryUtils. - return 31 * key.hashCode() + Integer.hashCode(depth); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof KeyAtDepth)) { - return false; - } - KeyAtDepth other = (KeyAtDepth) obj; - return key.equals(other.key) && depth == other.depth; - } - } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AbstractQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AbstractQueryEnvironment.java index 818813c75c..6971d07801 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AbstractQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AbstractQueryEnvironment.java @@ -118,7 +118,7 @@ public abstract class AbstractQueryEnvironment<T> implements QueryEnvironment<T> @Override public QueryTaskFuture<Void> eval( - QueryExpression expr, VariableContext<T> context, final Callback<T> callback) { + QueryExpression expr, QueryExpressionContext<T> context, final Callback<T> callback) { // Not all QueryEnvironment implementations embrace the async+streaming evaluation framework. In // particular, the streaming callbacks employed by functions like 'deps' use // QueryEnvironment#buildTransitiveClosure. So if the implementation of that method does some diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java index e0b5a45acd..f3c80b2ee4 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java @@ -54,7 +54,7 @@ public class AllPathsFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<Argument> args, final Callback<T> callback) { @@ -79,14 +79,14 @@ public class AllPathsFunction implements QueryFunction { env.buildTransitiveClosure(expression, fromValue, Integer.MAX_VALUE); - Set<T> reachableFromX = env.getTransitiveClosure(fromValue); + Set<T> reachableFromX = env.getTransitiveClosure(fromValue, context); Predicate<T> reachable = Predicates.in(reachableFromX); Uniquifier<T> uniquifier = env.createUniquifier(); Collection<T> result = uniquifier.unique(intersection(reachableFromX, toValue)); callback.process(result); Collection<T> worklist = result; while (!worklist.isEmpty()) { - Iterable<T> reverseDeps = env.getReverseDeps(worklist); + Iterable<T> reverseDeps = env.getReverseDeps(worklist, context); worklist = uniquifier.unique(Iterables.filter(reverseDeps, reachable)); callback.process(worklist); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java index 6292b7f969..a90c8eba7b 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java @@ -53,7 +53,7 @@ public class AllRdepsFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, List<Argument> args, Callback<T> callback) { @@ -81,7 +81,7 @@ public class AllRdepsFunction implements QueryFunction { final QueryEnvironment<T> env, QueryExpression expression, final Predicate<T> universe, - VariableContext<T> context, + QueryExpressionContext<T> context, final Callback<T> callback, final int depth) { final MinDepthUniquifier<T> minDepthUniquifier = env.createMinDepthUniquifier(); @@ -104,7 +104,8 @@ public class AllRdepsFunction implements QueryFunction { Iterables.addAll( next, env.getReverseDeps( - minDepthUniquifier.uniqueAtDepthLessThanOrEqualTo(currentInUniverse, i))); + minDepthUniquifier.uniqueAtDepthLessThanOrEqualTo(currentInUniverse, i), + context)); callback.process(currentInUniverse); if (next.isEmpty()) { // Exit when there are no more nodes to visit. diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java index 3b4968806e..5f4f82e5a8 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BinaryOperatorExpression.java @@ -57,7 +57,7 @@ public class BinaryOperatorExpression extends QueryExpression { @Override public <T> QueryTaskFuture<Void> eval( - QueryEnvironment<T> env, VariableContext<T> context, Callback<T> callback) { + QueryEnvironment<T> env, QueryExpressionContext<T> context, Callback<T> callback) { switch (operator) { case PLUS: case UNION: @@ -82,7 +82,7 @@ public class BinaryOperatorExpression extends QueryExpression { private static <T> QueryTaskFuture<Void> evalPlus( ImmutableList<QueryExpression> operands, QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, Callback<T> callback) { ArrayList<QueryTaskFuture<Void>> queryTasks = new ArrayList<>(operands.size()); for (QueryExpression operand : operands) { @@ -92,13 +92,13 @@ public class BinaryOperatorExpression extends QueryExpression { } /** - * Evaluates an expression of the form "e1 - e2 - ... - eK" by noting its equivalence to - * "e1 - (e2 + ... + eK)" and evaluating the subexpressions on the right-hand-side separately. + * Evaluates an expression of the form "e1 - e2 - ... - eK" by noting its equivalence to "e1 - (e2 + * + ... + eK)" and evaluating the subexpressions on the right-hand-side separately. */ private static <T> QueryTaskFuture<Void> evalMinus( final ImmutableList<QueryExpression> operands, final QueryEnvironment<T> env, - final VariableContext<T> context, + final QueryExpressionContext<T> context, final Callback<T> callback) { QueryTaskFuture<ThreadSafeMutableSet<T>> lhsValueFuture = QueryUtil.evalAll(env, context, operands.get(0)); @@ -131,7 +131,7 @@ public class BinaryOperatorExpression extends QueryExpression { private <T> QueryTaskFuture<Void> evalIntersect( final QueryEnvironment<T> env, - final VariableContext<T> context, + final QueryExpressionContext<T> context, final Callback<T> callback) { // For each right-hand side operand, intersection cannot be performed in a streaming manner; the // entire result of that operand is needed. So, in order to avoid pinning too much in memory at diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java index ac55028c17..06371f856c 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java @@ -42,7 +42,7 @@ public class BuildFilesFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<Argument> args, final Callback<T> callback) { @@ -56,9 +56,10 @@ public class BuildFilesFunction implements QueryFunction { throws QueryException, InterruptedException { ThreadSafeMutableSet<T> result = env.createThreadSafeMutableSet(); Iterables.addAll(result, partialResult); - callback.process(uniquifier.unique( - env.getBuildFiles( - expression, result, /* BUILD */ true, /* load */ true))); + callback.process( + uniquifier.unique( + env.getBuildFiles( + expression, result, /*buildFiles=*/ true, /*loads=*/ true, context))); } }); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java index 803c975d90..190e29137e 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java @@ -48,13 +48,11 @@ final class DepsFunction implements QueryFunction { return ImmutableList.of(ArgumentType.EXPRESSION, ArgumentType.INTEGER); } - /** - * Breadth-first search from the arguments. - */ + /** Breadth-first search from the arguments. */ @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<Argument> args, final Callback<T> callback) { @@ -92,7 +90,7 @@ final class DepsFunction implements QueryFunction { minDepthUniquifier.uniqueAtDepthLessThanOrEqualTo(current, i); callback.process(toProcess); current = env.createThreadSafeMutableSet(); - Iterables.addAll(current, env.getFwdDeps(toProcess)); + Iterables.addAll(current, env.getFwdDeps(toProcess, context)); if (current.isEmpty()) { // Exit when there are no more nodes to visit. break; diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java index 04c4358b4e..c5bd36509b 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/FunctionExpression.java @@ -46,7 +46,7 @@ public class FunctionExpression extends QueryExpression { @Override public <T> QueryTaskFuture<Void> eval( - QueryEnvironment<T> env, VariableContext<T> context, Callback<T> callback) { + QueryEnvironment<T> env, QueryExpressionContext<T> context, Callback<T> callback) { return function.eval(env, context, this, args, callback); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java index edd0935fc7..161fe5b330 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java @@ -55,7 +55,7 @@ public class LabelsFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, final List<Argument> args, final Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java index 336f94ff0c..7709ce75fe 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LetExpression.java @@ -68,7 +68,7 @@ public class LetExpression extends QueryExpression { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - final VariableContext<T> context, + final QueryExpressionContext<T> context, final Callback<T> callback) { if (!NAME_PATTERN.matcher(varName).matches()) { return env.immediateFailedFuture( @@ -78,7 +78,7 @@ public class LetExpression extends QueryExpression { QueryUtil.evalAll(env, context, varExpr); Function<ThreadSafeMutableSet<T>, QueryTaskFuture<Void>> evalBodyAsyncFunction = varValue -> { - VariableContext<T> bodyContext = VariableContext.with(context, varName, varValue); + QueryExpressionContext<T> bodyContext = context.with(varName, varValue); return env.eval(bodyExpr, bodyContext, callback); }; return env.transformAsync(varValueFuture, evalBodyAsyncFunction); diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java index e70f8b696b..716a05cb98 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java @@ -38,7 +38,7 @@ public class LoadFilesFunction implements QueryEnvironment.QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<QueryEnvironment.Argument> args, final Callback<T> callback) { @@ -52,12 +52,10 @@ public class LoadFilesFunction implements QueryEnvironment.QueryFunction { throws QueryException, InterruptedException { ThreadSafeMutableSet<T> result = env.createThreadSafeMutableSet(); Iterables.addAll(result, partialResult); - callback.process(uniquifier.unique( - env.getBuildFiles( - expression, - result, - /* BUILD */ false, - /* load */ true))); + callback.process( + uniquifier.unique( + env.getBuildFiles( + expression, result, /*buildFiles=*/ false, /*loads=*/ true, context))); } }); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java index da460e2c5e..4be94bba4b 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java @@ -112,18 +112,19 @@ public interface QueryEnvironment<T> { Iterable<ArgumentType> getArgumentTypes(); /** - * Returns a {@link QueryTaskFuture} representing the asynchronous application of this - * {@link QueryFunction} to the given {@code args}, feeding the results to the given - * {@code callback}. + * Returns a {@link QueryTaskFuture} representing the asynchronous application of this {@link + * QueryFunction} to the given {@code args}, feeding the results to the given {@code callback}. * * @param env the query environment this function is evaluated in. * @param expression the expression being evaluated. + * @param context the context relevant to the expression being evaluated. Contains the variable + * bindings from {@link LetExpression}s. * @param args the input arguments. These are type-checked against the specification returned by * {@link #getArgumentTypes} and {@link #getMandatoryArguments} */ <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, List<Argument> args, Callback<T> callback); @@ -159,16 +160,19 @@ public interface QueryEnvironment<T> { T getOrCreate(T target); /** Returns the direct forward dependencies of the specified targets. */ - Iterable<T> getFwdDeps(Iterable<T> targets) throws InterruptedException; + Iterable<T> getFwdDeps(Iterable<T> targets, QueryExpressionContext<T> context) + throws InterruptedException; /** Returns the direct reverse dependencies of the specified targets. */ - Iterable<T> getReverseDeps(Iterable<T> targets) throws InterruptedException; + Iterable<T> getReverseDeps(Iterable<T> targets, QueryExpressionContext<T> context) + throws InterruptedException; /** * Returns the forward transitive closure of all of the targets in "targets". Callers must ensure * that {@link #buildTransitiveClosure} has been called for the relevant subgraph. */ - ThreadSafeMutableSet<T> getTransitiveClosure(ThreadSafeMutableSet<T> targets) + ThreadSafeMutableSet<T> getTransitiveClosure( + ThreadSafeMutableSet<T> targets, QueryExpressionContext<T> context) throws InterruptedException; /** @@ -185,11 +189,12 @@ public interface QueryEnvironment<T> { int maxDepth) throws QueryException, InterruptedException; /** Returns the ordered sequence of nodes on some path from "from" to "to". */ - Iterable<T> getNodesOnPath(T from, T to) throws InterruptedException; + Iterable<T> getNodesOnPath(T from, T to, QueryExpressionContext<T> context) + throws InterruptedException; /** - * Returns a {@link QueryTaskFuture} representing the asynchronous evaluation of the given - * {@code expr} and passing of the results to the given {@code callback}. + * Returns a {@link QueryTaskFuture} representing the asynchronous evaluation of the given {@code + * expr} and passing of the results to the given {@code callback}. * * <p>Note that this method should guarantee that the callback does not see repeated elements. * @@ -197,7 +202,7 @@ public interface QueryEnvironment<T> { * @param callback The caller callback to notify when results are available */ QueryTaskFuture<Void> eval( - QueryExpression expr, VariableContext<T> context, Callback<T> callback); + QueryExpression expr, QueryExpressionContext<T> context, Callback<T> callback); /** * An asynchronous computation of part of a query evaluation. @@ -402,7 +407,9 @@ public interface QueryEnvironment<T> { QueryExpression caller, ThreadSafeMutableSet<T> nodes, boolean buildFiles, - boolean loads) throws QueryException, InterruptedException; + boolean loads, + QueryExpressionContext<T> context) + throws QueryException, InterruptedException; /** * Returns an object that can be used to query information about targets. Implementations should diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java index 3a9afe8303..0326df9bcc 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpression.java @@ -62,18 +62,14 @@ public abstract class QueryExpression { * specified environment, notifying the callback with a result. Note that it is allowed to notify * the callback with partial results instead of just one final result. * - * <p>Failures resulting from evaluation of an ill-formed query cause - * QueryException to be thrown. + * <p>Failures resulting from evaluation of an ill-formed query cause QueryException to be thrown. * - * <p>The reporting of failures arising from errors in BUILD files depends on - * the --keep_going flag. If enabled (the default), then QueryException is - * thrown. If disabled, evaluation will stumble on to produce a (possibly - * inaccurate) result, but a result nonetheless. + * <p>The reporting of failures arising from errors in BUILD files depends on the --keep_going + * flag. If enabled (the default), then QueryException is thrown. If disabled, evaluation will + * stumble on to produce a (possibly inaccurate) result, but a result nonetheless. */ public abstract <T> QueryTaskFuture<Void> eval( - QueryEnvironment<T> env, - VariableContext<T> context, - Callback<T> callback); + QueryEnvironment<T> env, QueryExpressionContext<T> context, Callback<T> callback); /** * Collects all target patterns that are referenced anywhere within this query expression and adds diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/VariableContext.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpressionContext.java index 9feb85242f..1971918225 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/VariableContext.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryExpressionContext.java @@ -16,19 +16,20 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; - import java.util.Map; import java.util.Set; - import javax.annotation.Nullable; -/** An immutable context of variable bindings for variables introduced by {@link LetExpression}s. */ +/** + * An immutable context, including variable bindings for variables introduced by {@link + * LetExpression}s. + */ @Immutable @ThreadSafe -public class VariableContext<T> { - private final ImmutableMap<String, Set<T>> context; +public class QueryExpressionContext<T> { + protected final ImmutableMap<String, Set<T>> context; - private VariableContext(ImmutableMap<String, Set<T>> context) { + protected QueryExpressionContext(ImmutableMap<String, Set<T>> context) { this.context = context; } @@ -41,21 +42,21 @@ public class VariableContext<T> { return context.get(name); } - /** Returns a {@link VariableContext} with no variables defined. */ - public static <T> VariableContext<T> empty() { - return new VariableContext<>(ImmutableMap.<String, Set<T>>of()); + /** Returns a {@link QueryExpressionContext} with no variables defined. */ + public static <T> QueryExpressionContext<T> empty() { + return new QueryExpressionContext<>(ImmutableMap.<String, Set<T>>of()); } - /** - * Returns a {@link VariableContext} that has all the same bindings as the given - * {@code variableContext} and also the binding of {@code name} to {@code value}. + * Returns a {@link QueryExpressionContext} that has all the same bindings as the given {@code + * variableContext} and also the binding of {@code name} to {@code value}. */ - static <T> VariableContext<T> with( - VariableContext<T> variableContext, - String name, - Set<T> value) { + protected QueryExpressionContext<T> with(String name, Set<T> value) { + return new QueryExpressionContext<>(withNewVariable(name, value)); + } + + protected final ImmutableMap<String, Set<T>> withNewVariable(String name, Set<T> value) { ImmutableMap.Builder<String, Set<T>> newContextBuilder = ImmutableMap.builder(); - for (Map.Entry<String, Set<T>> entry : variableContext.context.entrySet()) { + for (Map.Entry<String, Set<T>> entry : context.entrySet()) { if (!entry.getKey().equals(name)) { // The binding of 'name' to 'value' should override any existing binding of name in // 'variableContext'. These are the semantics we want in order for nested let-expressions @@ -64,7 +65,12 @@ public class VariableContext<T> { } } newContextBuilder.put(name, value); - return new VariableContext<>(newContextBuilder.build()); + return newContextBuilder.build(); + } + + @Override + public String toString() { + return "QueryExpressionContext: " + context; } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryUtil.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryUtil.java index e860cb9060..fce662fcef 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryUtil.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryUtil.java @@ -121,7 +121,7 @@ public final class QueryUtil { * <p>Should only be used by QueryExpressions when it is the only way of achieving correctness. */ public static <T> QueryTaskFuture<ThreadSafeMutableSet<T>> evalAll( - QueryEnvironment<T> env, VariableContext<T> context, QueryExpression expr) { + QueryEnvironment<T> env, QueryExpressionContext<T> context, QueryExpression expr) { final AggregateAllCallback<T, ThreadSafeMutableSet<T>> callback = newAggregateAllCallback(env); return env.whenSucceedsCall( env.eval(expr, context, callback), diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java index e40960a843..6f30313b3d 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RdepsFunction.java @@ -53,7 +53,7 @@ public final class RdepsFunction extends AllRdepsFunction { @Override public <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, List<Argument> args, Callback<T> callback) { @@ -81,7 +81,7 @@ public final class RdepsFunction extends AllRdepsFunction { private static <T> QueryTaskFuture<Void> evalWithBoundedDepth( QueryEnvironment<T> env, QueryExpression rdepsFunctionExpressionForErrorMessages, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression argumentExpression, int depth, QueryExpression universeExpression, @@ -94,15 +94,14 @@ public final class RdepsFunction extends AllRdepsFunction { try { env.buildTransitiveClosure( rdepsFunctionExpressionForErrorMessages, universeValue, Integer.MAX_VALUE); - universe = Predicates.in(env.getTransitiveClosure(universeValue)); + universe = Predicates.in(env.getTransitiveClosure(universeValue, context)); } catch (InterruptedException e) { return env.immediateCancelledFuture(); } catch (QueryException e) { return env.immediateFailedFuture(e); } - return AllRdepsFunction.eval( - env, argumentExpression, universe, context, callback, depth); + return AllRdepsFunction.eval(env, argumentExpression, universe, context, callback, depth); }; return env.transformAsync(universeValueFuture, evalInUniverseAsyncFunction); diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java index 136e0a6273..f6f748c8e1 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java @@ -34,7 +34,7 @@ public abstract class RegexFilterExpression implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, final List<Argument> args, Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java index 3f883c144f..853972e8e7 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SetExpression.java @@ -48,7 +48,7 @@ public class SetExpression extends QueryExpression { @Override public <T> QueryTaskFuture<Void> eval( - QueryEnvironment<T> env, VariableContext<T> context, Callback<T> callback) { + QueryEnvironment<T> env, QueryExpressionContext<T> context, Callback<T> callback) { ArrayList<QueryTaskFuture<Void>> queryTasks = new ArrayList<>(words.size()); for (TargetLiteral expr : words) { queryTasks.add(env.eval(expr, context, callback)); diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SiblingsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SiblingsFunction.java index 5d03bfd949..4a8ea7f7ca 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SiblingsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SiblingsFunction.java @@ -48,7 +48,7 @@ public class SiblingsFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<Argument> args, final Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java index 995e288f29..882cbc9b12 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java @@ -52,7 +52,7 @@ class SomeFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<Argument> args, final Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java index 93650d5fbb..4f436d1db2 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java @@ -53,7 +53,7 @@ class SomePathFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, final QueryExpression expression, List<Argument> args, final Callback<T> callback) { @@ -83,7 +83,7 @@ class SomePathFunction implements QueryFunction { for (T x : uniquifier.unique(fromValue)) { ThreadSafeMutableSet<T> xSet = env.createThreadSafeMutableSet(); xSet.add(x); - ThreadSafeMutableSet<T> xtc = env.getTransitiveClosure(xSet); + ThreadSafeMutableSet<T> xtc = env.getTransitiveClosure(xSet, context); SetView<T> result; if (xtc.size() > toValue.size()) { result = Sets.intersection(toValue, xtc); @@ -91,7 +91,7 @@ class SomePathFunction implements QueryFunction { result = Sets.intersection(xtc, toValue); } if (!result.isEmpty()) { - callback.process(env.getNodesOnPath(x, result.iterator().next())); + callback.process(env.getNodesOnPath(x, result.iterator().next(), context)); return null; } uniquifier.unique(xtc); diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/StreamableQueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/StreamableQueryEnvironment.java index f36bee0063..07cf9a226c 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/StreamableQueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/StreamableQueryEnvironment.java @@ -22,30 +22,28 @@ public interface StreamableQueryEnvironment<T> extends QueryEnvironment<T> { QueryTaskFuture<Void> getAllRdepsBoundedParallel( QueryExpression expression, int depth, - VariableContext<T> context, + QueryExpressionContext<T> context, Callback<T> callback); QueryTaskFuture<Void> getAllRdepsUnboundedParallel( - QueryExpression expression, - VariableContext<T> context, - Callback<T> callback); + QueryExpression expression, QueryExpressionContext<T> context, Callback<T> callback); QueryTaskFuture<Void> getRdepsBoundedParallel( QueryExpression expression, int depth, QueryExpression universe, - VariableContext<T> context, + QueryExpressionContext<T> context, Callback<T> callback); QueryTaskFuture<Void> getRdepsUnboundedParallel( QueryExpression expression, QueryExpression universe, - VariableContext<T> context, + QueryExpressionContext<T> context, Callback<T> callback); QueryTaskFuture<Void> getDepsUnboundedParallel( QueryExpression expression, - VariableContext<T> context, + QueryExpressionContext<T> context, Callback<T> callback, Callback<T> errorReporter); } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java index 4e3a8e7eb2..6f9976537c 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TargetLiteral.java @@ -45,7 +45,7 @@ public final class TargetLiteral extends QueryExpression { } private <T> QueryTaskFuture<Void> evalVarReference( - QueryEnvironment<T> env, VariableContext<T> context, Callback<T> callback) { + QueryEnvironment<T> env, QueryExpressionContext<T> context, Callback<T> callback) { String varName = LetExpression.getNameFromReference(pattern); Set<T> value = context.get(varName); if (value == null) { @@ -64,7 +64,7 @@ public final class TargetLiteral extends QueryExpression { @Override public <T> QueryTaskFuture<Void> eval( - QueryEnvironment<T> env, VariableContext<T> context, Callback<T> callback) { + QueryEnvironment<T> env, QueryExpressionContext<T> context, Callback<T> callback) { if (isVariableReference()) { return evalVarReference(env, context, callback); } else { diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java index 7119cb47ad..ff00be44b5 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java @@ -63,7 +63,7 @@ public class TestsFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - VariableContext<T> context, + QueryExpressionContext<T> context, QueryExpression expression, List<Argument> args, final Callback<T> callback) { diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java index 072013371d..c76d0da170 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java @@ -56,7 +56,7 @@ public class VisibleFunction implements QueryFunction { @Override public <T> QueryTaskFuture<Void> eval( final QueryEnvironment<T> env, - final VariableContext<T> context, + final QueryExpressionContext<T> context, QueryExpression expression, final List<Argument> args, final Callback<T> callback) { |