Add --local_test_jobs to limit local test concurrency

Users have asked for ways to control the concurrency level of their
local tests. They can do it right now using the --local_resources
option, but that is unintuitive and affects the parallelism of
non-test actions.

This option changes the kind of resources obtained for local tests. If
set, the CPU, RAM, and IO dimensions for local tests will not be used,
and a new localTestCount dimension will be used, where the capacity is
equal to the option's value, and each local test consumes one unit.

--
MOS_MIGRATED_REVID=87177698

1 files changed, 44 insertions, 28 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
index bb2aea9fcc..db6ed118f6 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
@@ -33,14 +33,14 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
- * CPU/RAM resource manager. Used to keep track of resources consumed by the Blaze action execution
- * threads and throttle them when necessary.
+ * Used to keep track of resources consumed by the Blaze action execution threads and throttle them
+ * when necessary.
  *
- * <p>Threads which are known to consume a significant amount of the local CPU or RAM resources
- * should call {@link #acquireResources} method. This method will check whether requested resources
- * are available and will either mark them as used and allow thread to proceed or will block the
- * thread until requested resources will become available. When thread completes it task, it must
- * release allocated resources by calling {@link #releaseResources} method.
+ * <p>Threads which are known to consume a significant amount of resources should call
+ * {@link #acquireResources} method. This method will check whether requested resources are
+ * available and will either mark them as used and allow the thread to proceed or will block the
+ * thread until requested resources will become available. When the thread completes its task, it
+ * must release allocated resources by calling {@link #releaseResources} method.
  *
  * <p>Available resources can be calculated using one of three ways:
  * <ol>
@@ -100,9 +100,9 @@ public class ResourceManager {
   // Please note that this value is purely empirical - we assume that generally
   // requested resources are somewhat pessimistic and thread would end up
   // using less than requested amount.
-  private final static double MIN_NECESSARY_CPU_RATIO = 0.6;
-  private final static double MIN_NECESSARY_RAM_RATIO = 1.0;
-  private final static double MIN_NECESSARY_IO_RATIO = 1.0;
+  private static final double MIN_NECESSARY_CPU_RATIO = 0.6;
+  private static final double MIN_NECESSARY_RAM_RATIO = 1.0;
+  private static final double MIN_NECESSARY_IO_RATIO = 1.0;
 
   // List of blocked threads. Associated CountDownLatch object will always
   // be initialized to 1 during creation in the acquire() method.
@@ -129,6 +129,9 @@ public class ResourceManager {
   // definition in the ResourceSet class.
   private double usedIo;
 
+  // Used local test count. Corresponds to the local test count definition in the ResourceSet class.
+  private int usedLocalTestCount;
+
   // Specifies how much of the RAM in staticResources we should allow to be used.
   public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67;
   private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE;
@@ -138,7 +141,7 @@ public class ResourceManager {
 
   private ResourceManager() {
     FINE = LOG.isLoggable(Level.FINE);
-    requestList = new LinkedList<Pair<ResourceSet, CountDownLatch>>();
+    requestList = new LinkedList<>();
   }
 
   @VisibleForTesting public static ResourceManager instanceForTestingOnly() {
@@ -154,6 +157,7 @@ public class ResourceManager {
     usedCpu = 0;
     usedRam = 0;
     usedIo = 0;
+    usedLocalTestCount = 0;
     for (Pair<ResourceSet, CountDownLatch> request : requestList) {
       // CountDownLatch can be set only to 0 or 1.
       request.second.countDown();
@@ -220,7 +224,7 @@ public class ResourceManager {
    */
   public void acquireResources(ActionMetadata owner, ResourceSet resources)
       throws InterruptedException {
-    Preconditions.checkArgument(resources != null);
+    Preconditions.checkNotNull(resources);
     long startTime = Profiler.nanoTimeMaybe();
     CountDownLatch latch = null;
     try {
@@ -231,7 +235,7 @@ public class ResourceManager {
       }
     } finally {
       threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0
-          || resources.getIoUsage() != 0);
+          || resources.getIoUsage() != 0 || resources.getLocalTestCount() != 0);
       acquired(owner);
 
       // Profile acquisition only if it waited for resource to become available.
@@ -255,7 +259,8 @@ public class ResourceManager {
     }
 
     if (acquired) {
-      threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0);
+      threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0
+          || resources.getIoUsage() != 0 || resources.getLocalTestCount() != 0);
       acquired(owner);
     }
 
@@ -266,13 +271,15 @@ public class ResourceManager {
     usedCpu += resources.getCpuUsage();
     usedRam += resources.getMemoryMb();
     usedIo += resources.getIoUsage();
+    usedLocalTestCount += resources.getLocalTestCount();
   }
 
   /**
    * Return true if any resources have been claimed through this manager.
    */
   public synchronized boolean inUse() {
-    return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || requestList.size() > 0;
+    return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || usedLocalTestCount != 0
+        || requestList.size() > 0;
   }
 
 
@@ -353,16 +360,18 @@ public class ResourceManager {
     usedCpu -= resources.getCpuUsage();
     usedRam -= resources.getMemoryMb();
     usedIo -= resources.getIoUsage();
+    usedLocalTestCount -= resources.getLocalTestCount();
 
     // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being
     // e.g. 1E-15. So if it is small enough, we set it to 0. But maybe there is a better solution.
-    if (usedCpu < 0.0001) {
+    double epsilon = 0.0001;
+    if (usedCpu < epsilon) {
       usedCpu = 0;
     }
-    if (usedRam < 0.0001) {
+    if (usedRam < epsilon) {
       usedRam = 0;
     }
-    if (usedIo < 0.0001) {
+    if (usedIo < epsilon) {
       usedIo = 0;
     }
     if (requestList.size() > 0) {
@@ -393,7 +402,7 @@ public class ResourceManager {
     Preconditions.checkNotNull(availableResources);
     // Comparison below is robust, since any calculation errors will be fixed
     // by the release() method.
-    if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0) {
+    if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0 && usedLocalTestCount == 0) {
       return true;
     }
     // Use only MIN_NECESSARY_???_RATIO of the resource value to check for
@@ -404,10 +413,12 @@ public class ResourceManager {
     double cpu = resources.getCpuUsage() * MIN_NECESSARY_CPU_RATIO;
     double ram = resources.getMemoryMb() * MIN_NECESSARY_RAM_RATIO;
     double io = resources.getIoUsage() * MIN_NECESSARY_IO_RATIO;
+    int localTestCount = resources.getLocalTestCount();
 
     double availableCpu = availableResources.getCpuUsage();
     double availableRam = availableResources.getMemoryMb();
     double availableIo = availableResources.getIoUsage();
+    int availableLocalTestCount = availableResources.getLocalTestCount();
 
     // Resources are considered available if any one of the conditions below is true:
     // 1) If resource is not requested at all, it is available.
@@ -416,28 +427,33 @@ public class ResourceManager {
     // ensure that at any given time, at least one thread is able to acquire
     // resources even if it requests more than available.
     // 3) If used resource amount is less than total available resource amount.
-    return (cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu) &&
-        (ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam) &&
-        (io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo);
+    boolean cpuIsAvailable = cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu;
+    boolean ramIsAvailable = ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam;
+    boolean ioIsAvailable = io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo;
+    boolean localTestCountIsAvailable = localTestCount == 0 || usedLocalTestCount == 0
+        || usedLocalTestCount + localTestCount <= availableLocalTestCount;
+    return cpuIsAvailable && ramIsAvailable && ioIsAvailable && localTestCountIsAvailable;
   }
 
   private synchronized void updateAvailableResources(boolean useFreeReading) {
     Preconditions.checkNotNull(staticResources);
     if (useFreeReading && isAutoSensingEnabled()) {
-      availableResources = ResourceSet.createWithRamCpuIo(
+      availableResources = ResourceSet.create(
           usedRam + freeReading.getFreeMb(),
           usedCpu + freeReading.getAvgFreeCpu(),
-          staticResources.getIoUsage());
+          staticResources.getIoUsage(),
+          staticResources.getLocalTestCount());
       if(FINE) {
         LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB,"
             + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU");
       }
       processWaitingThreads();
     } else {
-      availableResources = ResourceSet.createWithRamCpuIo(
+      availableResources = ResourceSet.create(
           staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0,
           staticResources.getCpuUsage(),
-          staticResources.getIoUsage());
+          staticResources.getIoUsage(),
+          staticResources.getLocalTestCount());
       processWaitingThreads();
     }
   }
@@ -466,7 +482,7 @@ public class ResourceManager {
   }
 
   @VisibleForTesting
-  synchronized boolean isAvailable(double ram, double cpu, double io) {
-    return areResourcesAvailable(ResourceSet.createWithRamCpuIo(ram, cpu, io));
+  synchronized boolean isAvailable(double ram, double cpu, double io, int localTestCount) {
+    return areResourcesAvailable(ResourceSet.create(ram, cpu, io, localTestCount));
   }
 }