Add a mechanism for bounding the number of Packages SkyQueryEnvironment's expensive parallel operations can operate on at once.

--
MOS_MIGRATED_REVID=138779172

1 files changed, 25 insertions, 15 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java
index 62926c8830..007109860c 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/RecursivePackageProviderBackedTargetPatternResolver.java
@@ -29,6 +29,7 @@ import com.google.devtools.build.lib.cmdline.ResolvedTargets;
 import com.google.devtools.build.lib.cmdline.TargetParsingException;
 import com.google.devtools.build.lib.cmdline.TargetPatternResolver;
 import com.google.devtools.build.lib.concurrent.MoreFutures;
+import com.google.devtools.build.lib.concurrent.MultisetSemaphore;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
@@ -65,16 +66,19 @@ public class RecursivePackageProviderBackedTargetPatternResolver
   private final EventHandler eventHandler;
   private final FilteringPolicy policy;
   private final ExecutorService executor;
+  private final MultisetSemaphore<PackageIdentifier> packageSemaphore;
 
   public RecursivePackageProviderBackedTargetPatternResolver(
       RecursivePackageProvider recursivePackageProvider,
       EventHandler eventHandler,
       FilteringPolicy policy,
-      ExecutorService executor) {
+      ExecutorService executor,
+      MultisetSemaphore<PackageIdentifier> packageSemaphore) {
     this.recursivePackageProvider = recursivePackageProvider;
     this.eventHandler = eventHandler;
     this.policy = policy;
     this.executor = executor;
+    this.packageSemaphore = packageSemaphore;
   }
 
   @Override
@@ -215,22 +219,28 @@ public class RecursivePackageProviderBackedTargetPatternResolver
       tasks.add(executor.submit(new Callable<Void>() {
           @Override
           public Void call() throws E, TargetParsingException, InterruptedException {
-            Iterable<ResolvedTargets<Target>> resolvedTargets =
-                bulkGetTargetsInPackage(originalPattern, pkgIdBatch, NO_FILTER).values();
-            List<Target> filteredTargets = new ArrayList<>(calculateSize(resolvedTargets));
-            for (ResolvedTargets<Target> targets : resolvedTargets) {
-              for (Target target : targets.getTargets()) {
-                // Perform the no-targets-found check before applying the filtering policy
-                // so we only return the error if the input directory's subtree really
-                // contains no targets.
-                foundTarget.set(true);
-                if (actualPolicy.shouldRetain(target, false)) {
-                  filteredTargets.add(target);
+            ImmutableSet<PackageIdentifier> pkgIdBatchSet = ImmutableSet.copyOf(pkgIdBatch);
+            packageSemaphore.acquireAll(pkgIdBatchSet);
+            try {
+              Iterable<ResolvedTargets<Target>> resolvedTargets =
+                  bulkGetTargetsInPackage(originalPattern, pkgIdBatch, NO_FILTER).values();
+              List<Target> filteredTargets = new ArrayList<>(calculateSize(resolvedTargets));
+              for (ResolvedTargets<Target> targets : resolvedTargets) {
+                for (Target target : targets.getTargets()) {
+                  // Perform the no-targets-found check before applying the filtering policy
+                  // so we only return the error if the input directory's subtree really
+                  // contains no targets.
+                  foundTarget.set(true);
+                  if (actualPolicy.shouldRetain(target, false)) {
+                    filteredTargets.add(target);
+                  }
                 }
               }
-            }
-            synchronized (callbackLock) {
-              callback.process(filteredTargets);
+              synchronized (callbackLock) {
+                callback.process(filteredTargets);
+              }
+            } finally {
+              packageSemaphore.releaseAll(pkgIdBatchSet);
             }
             return null;
           }