diff options
author | Nathan Harmata <nharmata@google.com> | 2016-09-08 15:26:04 +0000 |
---|---|---|
committer | Damien Martin-Guillerez <dmarting@google.com> | 2016-09-09 09:02:50 +0000 |
commit | bfa369fa8eb19bc18a570cfc7733ffa90c3671fe (patch) | |
tree | 350b0a38e8e7e554830922ae3ac323e79eec972e /src/main/java/com | |
parent | c9ee30213b8b59953ac993789cc1899716a8faf8 (diff) |
Boilerplate for parallel evaluation of FunctionExpression query expression nodes.
--
MOS_MIGRATED_REVID=132558031
Diffstat (limited to 'src/main/java/com')
13 files changed, 163 insertions, 2 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java index 9de4769654..3077b689a0 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.query2; import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.query2.engine.Callback; import com.google.devtools.build.lib.query2.engine.QueryEnvironment; @@ -24,6 +25,7 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.ThreadSafeCallback; import com.google.devtools.build.lib.query2.engine.VariableContext; import com.google.devtools.build.lib.vfs.PathFragment; @@ -80,4 +82,15 @@ public class RBuildFilesFunction implements QueryFunction { .getRBuildFiles( Collections2.transform(args, ARGUMENT_TO_PATH_FRAGMENT), (Callback<Target>) callback); } + + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java index 68b92d31a9..b491097eb7 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java @@ -18,6 +18,7 @@ import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -81,6 +82,17 @@ public class AllPathsFunction implements QueryFunction { } } + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** * Returns a (new, mutable, unordered) set containing the intersection of the * two specified sets. diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java index a39efd9a97..0da05aaa1d 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java @@ -17,6 +17,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -102,4 +103,15 @@ public class AllRdepsFunction implements QueryFunction { }); } } + + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java index acc7917b7e..0b6a595487 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.collect.CompactHashSet; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; @@ -65,6 +66,17 @@ class BuildFilesFunction implements QueryFunction { } @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + + @Override public int getMandatoryArguments() { return 1; } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java index ecc8d8edb6..5de84a195c 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -83,4 +84,15 @@ final class DepsFunction implements QueryFunction { } }); } + + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java index 140e732437..5a918cb889 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -80,4 +81,15 @@ class LabelsFunction implements QueryFunction { } }); } + + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java index 6b6866ebd6..5370b4a7d3 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java @@ -15,8 +15,9 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.collect.CompactHashSet; - +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import java.util.List; import java.util.Set; @@ -65,6 +66,17 @@ class LoadFilesFunction implements QueryEnvironment.QueryFunction { } @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + + @Override public int getMandatoryArguments() { return 1; } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java index 3abe19be2b..f415b41a1a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; import java.util.Collection; import java.util.List; import java.util.Set; @@ -120,6 +121,21 @@ public interface QueryEnvironment<T> { QueryExpression expression, List<Argument> args, Callback<T> callback) throws QueryException, InterruptedException; + + /** + * Same as {@link #eval(QueryEnvironment, VariableContext, QueryExpression, List, Callback)}, + * except that this {@link QueryFunction} may use {@code executorService} to achieve + * parallelism. + * + * <p>The caller must ensure that {@code env} is thread safe. + */ + <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException; } /** diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java index 26779082a9..44a40c5aa5 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -64,6 +65,17 @@ public abstract class RegexFilterExpression implements QueryFunction { QueryUtil.filteredCallback(callback, matchFilter)); } + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** * Returns string for the given target that must be matched against pattern. * May return null, in which case matching is guaranteed to fail. diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java index 8560992f93..c482c2ec7a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -70,4 +71,15 @@ class SomeFunction implements QueryFunction { throw new QueryException(expression, "argument set is empty"); } } + + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java index 23988bf2fc..fd5a527224 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -85,4 +86,15 @@ class SomePathFunction implements QueryFunction { } callback.process(ImmutableSet.<T>of()); } + + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java index 77dc6b4899..ac83b78be1 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -85,6 +86,17 @@ class TestsFunction implements QueryFunction { }); } + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** * Decides whether to include a test in a test_suite or not. * @param testTags Collection of all tags exhibited by a given test. @@ -143,7 +155,7 @@ class TestsFunction implements QueryFunction { * A closure over the temporary state needed to compute the expression. This makes the evaluation * thread-safe, as long as instances of this class are used only within a single thread. */ - private final class Closure<T> { + private static final class Closure<T> { private final QueryExpression expression; /** A dynamically-populated mapping from test_suite rules to their tests. */ private final Map<T, Set<T>> testsInSuite = new HashMap<>(); diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java index f69f134bc1..2506d85514 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -70,6 +71,17 @@ public class VisibleFunction implements QueryFunction { }); } + @Override + public <T> void parEval( + QueryEnvironment<T> env, + VariableContext<T> context, + QueryExpression expression, + List<Argument> args, + ThreadSafeCallback<T> callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** Returns true if {@code target} is visible to all targets in {@code toSet}. */ private static <T> boolean visibleToAll(QueryEnvironment<T> env, Set<T> toSet, T target) throws QueryException, InterruptedException { |