From bfa369fa8eb19bc18a570cfc7733ffa90c3671fe Mon Sep 17 00:00:00 2001 From: Nathan Harmata Date: Thu, 8 Sep 2016 15:26:04 +0000 Subject: Boilerplate for parallel evaluation of FunctionExpression query expression nodes. -- MOS_MIGRATED_REVID=132558031 --- .../devtools/build/lib/query2/RBuildFilesFunction.java | 13 +++++++++++++ .../build/lib/query2/engine/AllPathsFunction.java | 12 ++++++++++++ .../build/lib/query2/engine/AllRdepsFunction.java | 12 ++++++++++++ .../build/lib/query2/engine/BuildFilesFunction.java | 12 ++++++++++++ .../devtools/build/lib/query2/engine/DepsFunction.java | 12 ++++++++++++ .../devtools/build/lib/query2/engine/LabelsFunction.java | 12 ++++++++++++ .../build/lib/query2/engine/LoadFilesFunction.java | 14 +++++++++++++- .../build/lib/query2/engine/QueryEnvironment.java | 16 ++++++++++++++++ .../build/lib/query2/engine/RegexFilterExpression.java | 12 ++++++++++++ .../devtools/build/lib/query2/engine/SomeFunction.java | 12 ++++++++++++ .../build/lib/query2/engine/SomePathFunction.java | 12 ++++++++++++ .../devtools/build/lib/query2/engine/TestsFunction.java | 14 +++++++++++++- .../build/lib/query2/engine/VisibleFunction.java | 12 ++++++++++++ 13 files changed, 163 insertions(+), 2 deletions(-) (limited to 'src/main/java/com/google/devtools/build/lib/query2') diff --git a/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java index 9de4769654..3077b689a0 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/RBuildFilesFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.query2; import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.query2.engine.Callback; import com.google.devtools.build.lib.query2.engine.QueryEnvironment; @@ -24,6 +25,7 @@ import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.engine.ThreadSafeCallback; import com.google.devtools.build.lib.query2.engine.VariableContext; import com.google.devtools.build.lib.vfs.PathFragment; @@ -80,4 +82,15 @@ public class RBuildFilesFunction implements QueryFunction { .getRBuildFiles( Collections2.transform(args, ARGUMENT_TO_PATH_FRAGMENT), (Callback) callback); } + + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java index 68b92d31a9..b491097eb7 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllPathsFunction.java @@ -18,6 +18,7 @@ import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -81,6 +82,17 @@ public class AllPathsFunction implements QueryFunction { } } + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** * Returns a (new, mutable, unordered) set containing the intersection of the * two specified sets. diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java index a39efd9a97..0da05aaa1d 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/AllRdepsFunction.java @@ -17,6 +17,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -102,4 +103,15 @@ public class AllRdepsFunction implements QueryFunction { }); } } + + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java index acc7917b7e..0b6a595487 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/BuildFilesFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.collect.CompactHashSet; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; @@ -64,6 +65,17 @@ class BuildFilesFunction implements QueryFunction { }); } + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + @Override public int getMandatoryArguments() { return 1; diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java index ecc8d8edb6..5de84a195c 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/DepsFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -83,4 +84,15 @@ final class DepsFunction implements QueryFunction { } }); } + + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java index 140e732437..5a918cb889 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LabelsFunction.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -80,4 +81,15 @@ class LabelsFunction implements QueryFunction { } }); } + + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java index 6b6866ebd6..5370b4a7d3 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/LoadFilesFunction.java @@ -15,8 +15,9 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.collect.CompactHashSet; - +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import java.util.List; import java.util.Set; @@ -64,6 +65,17 @@ class LoadFilesFunction implements QueryEnvironment.QueryFunction { }); } + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + @Override public int getMandatoryArguments() { return 1; diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java index 3abe19be2b..f415b41a1a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/QueryEnvironment.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; import java.util.Collection; import java.util.List; import java.util.Set; @@ -120,6 +121,21 @@ public interface QueryEnvironment { QueryExpression expression, List args, Callback callback) throws QueryException, InterruptedException; + + /** + * Same as {@link #eval(QueryEnvironment, VariableContext, QueryExpression, List, Callback)}, + * except that this {@link QueryFunction} may use {@code executorService} to achieve + * parallelism. + * + *

The caller must ensure that {@code env} is thread safe. + */ + void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException; } /** diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java index 26779082a9..44a40c5aa5 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/RegexFilterExpression.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -64,6 +65,17 @@ public abstract class RegexFilterExpression implements QueryFunction { QueryUtil.filteredCallback(callback, matchFilter)); } + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** * Returns string for the given target that must be matched against pattern. * May return null, in which case matching is guaranteed to fail. diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java index 8560992f93..c482c2ec7a 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomeFunction.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -70,4 +71,15 @@ class SomeFunction implements QueryFunction { throw new QueryException(expression, "argument set is empty"); } } + + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java index 23988bf2fc..fd5a527224 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/SomePathFunction.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.collect.Sets.SetView; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -85,4 +86,15 @@ class SomePathFunction implements QueryFunction { } callback.process(ImmutableSet.of()); } + + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } } diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java index 77dc6b4899..ac83b78be1 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/TestsFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -85,6 +86,17 @@ class TestsFunction implements QueryFunction { }); } + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** * Decides whether to include a test in a test_suite or not. * @param testTags Collection of all tags exhibited by a given test. @@ -143,7 +155,7 @@ class TestsFunction implements QueryFunction { * A closure over the temporary state needed to compute the expression. This makes the evaluation * thread-safe, as long as instances of this class are used only within a single thread. */ - private final class Closure { + private static final class Closure { private final QueryExpression expression; /** A dynamically-populated mapping from test_suite rules to their tests. */ private final Map> testsInSuite = new HashMap<>(); diff --git a/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java b/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java index f69f134bc1..2506d85514 100644 --- a/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/query2/engine/VisibleFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.query2.engine; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Argument; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.ArgumentType; import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; @@ -70,6 +71,17 @@ public class VisibleFunction implements QueryFunction { }); } + @Override + public void parEval( + QueryEnvironment env, + VariableContext context, + QueryExpression expression, + List args, + ThreadSafeCallback callback, + ListeningExecutorService executorService) throws QueryException, InterruptedException { + eval(env, context, expression, args, callback); + } + /** Returns true if {@code target} is visible to all targets in {@code toSet}. */ private static boolean visibleToAll(QueryEnvironment env, Set toSet, T target) throws QueryException, InterruptedException { -- cgit v1.2.3