Remove autosense logic.

Polling the machine load can never work, because the following
scenarios are quite common:

* Tasks that are faster than the poll cycle time.

* Tasks whose CPU and/or RAM consumption changes over the lifetime of
  the task.

--
MOS_MIGRATED_REVID=90990445

2 files changed, 9 insertions, 269 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
index 7255b5d49e..b43a1e2ff7 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java
@@ -17,11 +17,7 @@ package com.google.devtools.build.lib.actions;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Splitter;
 import com.google.common.io.Files;
-import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
-import com.google.devtools.build.lib.util.BlazeClock;
-import com.google.devtools.build.lib.util.Clock;
-import com.google.devtools.build.lib.util.LoggingUtil;
 import com.google.devtools.build.lib.util.ProcMeminfoParser;
 
 import java.io.File;
@@ -29,8 +25,6 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 /**
  * This class estimates the local host's resource capacity.
@@ -38,93 +32,6 @@ import java.util.logging.Logger;
 @ThreadCompatible
 public final class LocalHostCapacity {
 
-  private static final Logger LOG = Logger.getLogger(LocalHostCapacity.class.getName());
-
-  /**
-   * Stores parsed /proc/stat CPU time counters.
-   * See {@link LocalHostCapacity#getCpuTimes(String)} for details.
-   */
-  @Immutable
-  private final static class CpuTimes {
-    private final long idleJiffies;
-    private final long totalJiffies;
-
-    CpuTimes(long idleJiffies, long totalJiffies) {
-      this.idleJiffies = idleJiffies;
-      this.totalJiffies = totalJiffies;
-    }
-
-    /**
-     * Return idle CPU ratio using current and previous CPU readings or 0 if
-     * ratio is undefined.
-     */
-    double getIdleRatio(CpuTimes prevTimes) {
-      if (prevTimes.totalJiffies == 0 || totalJiffies == prevTimes.totalJiffies) {
-        return 0;
-      }
-      return ((double)(idleJiffies - prevTimes.idleJiffies) /
-          (double)(totalJiffies - prevTimes.totalJiffies));
-    }
-  }
-
-  /**
-   * Used to store available local CPU and RAM resources information.
-   * See {@link LocalHostCapacity#getFreeResources(FreeResources)} for details.
-   */
-  public static final class FreeResources {
-
-    private final Clock clock;
-    private final CpuTimes cpuTimes;
-    private final long lastTimestamp;
-    private final double freeCpu;
-    private final double freeMb;
-    private final long interval;
-
-    private FreeResources(Clock localClock, ProcMeminfoParser memInfo, String statContent,
-                          FreeResources prevStats) {
-      clock = localClock;
-      lastTimestamp = localClock.nanoTime();
-      freeMb = ProcMeminfoParser.kbToMb(memInfo.getFreeRamKb());
-      cpuTimes = getCpuTimes(statContent);
-      if (prevStats == null) {
-        interval = 0;
-        freeCpu = 0.0;
-      } else {
-        interval = lastTimestamp - prevStats.lastTimestamp;
-        freeCpu = getLocalHostCapacity().getCpuUsage() * cpuTimes.getIdleRatio(prevStats.cpuTimes);
-      }
-    }
-
-    /**
-     * Returns amount of available RAM in MB.
-     */
-    public double getFreeMb() { return freeMb; }
-
-    /**
-     * Returns average available CPU resources (as a fraction of the CPU core,
-     * so one fully CPU-bound thread should consume exactly 1.0 CPU resource).
-     */
-    public double getAvgFreeCpu() { return freeCpu; }
-
-    /**
-     * Returns interval in ms between CPU load measurements used to calculate
-     * average available CPU resources.
-     */
-    public long getInterval() { return interval / 1000000; }
-
-    /**
-     * Returns age of available resource data in ms.
-     */
-    public long getReadingAge() {
-      return (clock.nanoTime() - lastTimestamp) / 1000000;
-    }
-  }
-
-  // Disables getFreeResources() if error occured during reading or parsing
-  // /proc/* information.
-  @VisibleForTesting
-  static boolean isDisabled;
-
   // If /proc/* information is not available, assume 3000 MB and 2 CPUs.
   private static ResourceSet DEFAULT_RESOURCES = ResourceSet.create(3000.0, 2.0, 1.0,
       Integer.MAX_VALUE);
@@ -148,21 +55,6 @@ public final class LocalHostCapacity {
     return localHostCapacity;
   }
 
-  /**
-   * Returns new FreeResources object populated with free RAM information from
-   * /proc/meminfo and CPU load information from the /proc/stat. First call
-   * should be made with null parameter to instantiate new FreeResources object.
-   * Subsequent calls will use information inside it to calculate average CPU
-   * load over the time between calls and to calculate amount of free CPU
-   * resources and generate new FreeResources() instance.
-   *
-   * If information is not available due to error, functionality will be disabled
-   * and method will always return null.
-   */
-  public static FreeResources getFreeResources(FreeResources stats) {
-    return getFreeResources(BlazeClock.instance(), "/proc/meminfo", "/proc/stat", stats);
-  }
-
   private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings();
 
   @VisibleForTesting
@@ -212,35 +104,6 @@ public final class LocalHostCapacity {
     return coresPerCpu;
   }
 
-  /**
-   * Parses cpu line of the /proc/stats, calculates number of idle and total
-   * CPU jiffies and returns CpuTimes instance with that information.
-   *
-   * Total CPU time includes <b>all</b> time reported to be spent by the CPUs,
-   * including so-called "stolen" time - time spent by other VMs on the same
-   * workstation.
-   */
-  private static CpuTimes getCpuTimes(String statContent) {
-    String[] cpuStats = statContent.substring(0, statContent.indexOf('\n')).trim().split(" +");
-    // Supported versions of /proc/stat (Linux kernel 2.6.x) must contain either
-    // 9 or 10 fields:
-    //   "cpu" utime ultime stime idle iowait irq softirq steal(since 2.6.11) 0
-    // We are interested in total time (sum of all columns) and idle time.
-    if (cpuStats.length < 9 | cpuStats.length > 10) {
-      throw new IllegalArgumentException("Unrecognized /proc/stat format");
-    }
-    if (!cpuStats[0].equals("cpu")) {
-      throw new IllegalArgumentException("/proc/stat does not start with cpu keyword");
-    }
-    long idleCpuJiffies = Long.parseLong(cpuStats[4]); // "idle" column.
-    long totalJiffies = 0;
-    for (int i = 1; i < cpuStats.length; i++) {
-      totalJiffies += Long.parseLong(cpuStats[i]);
-    }
-    long totalCpuJiffies = totalJiffies;
-    return new CpuTimes(idleCpuJiffies, totalCpuJiffies);
-  }
-
   @VisibleForTesting
   static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) {
     try {
@@ -259,45 +122,20 @@ public final class LocalHostCapacity {
           1.0,
           Integer.MAX_VALUE);
     } catch (IOException | IllegalArgumentException e) {
-      disableProcFsUse(e);
       return DEFAULT_RESOURCES;
     }
   }
 
-  @VisibleForTesting
-  static FreeResources getFreeResources(Clock localClock, String meminfoFile, String statFile,
-                                        FreeResources prevStats) {
-    if (isDisabled) { return null; }
-    try {
-      String statContent = readContent(statFile);
-      return new FreeResources(localClock, new ProcMeminfoParser(meminfoFile),
-                               statContent, prevStats);
-    } catch (IOException | IllegalArgumentException e) {
-      disableProcFsUse(e);
-      return null;
-    }
-  }
-
   /**
    * For testing purposes only. Do not use it.
    */
   @VisibleForTesting
   static void setLocalHostCapacity(ResourceSet resources) {
     localHostCapacity = resources;
-    isDisabled = false;
   }
 
   private static String readContent(String filename) throws IOException {
     return Files.toString(new File(filename), Charset.defaultCharset());
   }
 
-  /**
-   * Disables use of /proc filesystem. Called internally when unexpected
-   * exception is caught.
-   */
-  private static void disableProcFsUse(Throwable cause) {
-    LoggingUtil.logToRemote(Level.WARNING, "Unable to read system load or capacity", cause);
-    LOG.log(Level.WARNING, "Unable to read system load or capacity", cause);
-    isDisabled = true;
-  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
index 8e6220c143..05be3370ea 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java
@@ -20,17 +20,12 @@ import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
 import com.google.devtools.build.lib.profiler.Profiler;
 import com.google.devtools.build.lib.profiler.ProfilerTask;
-import com.google.devtools.build.lib.util.LoggingUtil;
 import com.google.devtools.build.lib.util.Pair;
 
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.concurrent.CountDownLatch;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 /**
  * Used to keep track of resources consumed by the Blaze action execution threads and throttle them
@@ -52,10 +47,6 @@ import java.util.logging.Logger;
  *     amount of the total available memory and will limit itself to the number of effective cores
  *     and 2/3 of the available memory. For details, please look at the {@link
  *     LocalHostCapacity#getLocalHostCapacity} method.
- * <li>Blaze will periodically (every 3 seconds) poll {@code /proc/meminfo} and {@code /proc/stat}
- *     information to obtain how much RAM and CPU resources are currently idle at that moment. For
- *     calculation details, please look at the {@link LocalHostCapacity#getFreeResources}
- *     implementation.
  * </ol>
  *
  * <p>The resource manager also allows a slight overallocation of the resources to account for the
@@ -67,9 +58,6 @@ import java.util.logging.Logger;
 @ThreadSafe
 public class ResourceManager {
 
-  private static final Logger LOG = Logger.getLogger(ResourceManager.class.getName());
-  private final boolean FINE;
-
   private EventBus eventBus;
 
   private final ThreadLocal<Boolean> threadLocked = new ThreadLocal<Boolean>() {
@@ -114,7 +102,6 @@ public class ResourceManager {
   private ResourceSet staticResources = null;
 
   private ResourceSet availableResources = null;
-  private LocalHostCapacity.FreeResources freeReading = null;
 
   // Used amount of CPU capacity (where 1.0 corresponds to the one fully
   // occupied CPU core. Corresponds to the CPU resource definition in the
@@ -136,11 +123,7 @@ public class ResourceManager {
   public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67;
   private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE;
 
-  // Timer responsible for the periodic polling of the current system load.
-  private Timer timer = null;
-
   private ResourceManager() {
-    FINE = LOG.isLoggable(Level.FINE);
     requestList = new LinkedList<>();
   }
 
@@ -150,8 +133,8 @@ public class ResourceManager {
 
   /**
    * Resets resource manager state and releases all thread locks.
-   * Note - it does not reset auto-sensing or available resources. Use
-   * separate call to setAvailableResoures() or to setAutoSensing().
+   * Note - it does not reset available resources. Use
+   * separate call to setAvailableResoures().
    */
   public synchronized void resetResourceUsage() {
     usedCpu = 0;
@@ -168,17 +151,16 @@ public class ResourceManager {
   /**
    * Sets available resources using given resource set. Must be called
    * at least once before using resource manager.
-   * <p>
-   * Method will also disable auto-sensing if it was enabled.
    */
   public synchronized void setAvailableResources(ResourceSet resources) {
     Preconditions.checkNotNull(resources);
     staticResources = resources;
-    setAutoSensing(false);
-  }
-
-  public synchronized boolean isAutoSensingEnabled() {
-    return timer != null;
+    availableResources = ResourceSet.create(
+        staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0,
+        staticResources.getCpuUsage(),
+        staticResources.getIoUsage(),
+        staticResources.getLocalTestCount());
+    processWaitingThreads();
   }
 
   /**
@@ -190,35 +172,6 @@ public class ResourceManager {
   }
 
   /**
-   * Enables or disables secondary resource allocation algorithm that will
-   * periodically (when needed but at most once per 3 seconds) checks real
-   * amount of available memory (based on /proc/meminfo) and current CPU load
-   * (based on 1 second difference of /proc/stat) and allows additional resource
-   * acquisition if previous requests were overly pessimistic.
-   */
-  public synchronized void setAutoSensing(boolean enable) {
-    // Create new Timer instance only if it does not exist already.
-    if (enable && !isAutoSensingEnabled()) {
-      Profiler.instance().logEvent(ProfilerTask.INFO, "Enable auto sensing");
-      if(refreshFreeResources()) {
-        timer = new Timer("AutoSenseTimer", true);
-        timer.schedule(new TimerTask() {
-          @Override public void run() { refreshFreeResources(); }
-        }, 3000, 3000);
-      }
-    } else if (!enable) {
-      if (isAutoSensingEnabled()) {
-        Profiler.instance().logEvent(ProfilerTask.INFO, "Disable auto sensing");
-        timer.cancel();
-        timer = null;
-      }
-      if (staticResources != null) {
-        updateAvailableResources(false);
-      }
-    }
-  }
-
-  /**
    * Acquires requested resource set. Will block if resource is not available.
    * NB! This method must be thread-safe!
    */
@@ -315,7 +268,7 @@ public class ResourceManager {
   }
 
   /**
-   * Releases previously requested resource set.
+   * Releases previously requested resource =.
    *
    * <p>NB! This method must be thread-safe!
    */
@@ -342,17 +295,6 @@ public class ResourceManager {
     Pair<ResourceSet, CountDownLatch> request =
       new Pair<>(resources, new CountDownLatch(1));
     requestList.add(request);
-
-    // If we use auto sensing and there has not been an update within last
-    // 30 seconds, something has gone really wrong - disable it.
-    if (isAutoSensingEnabled() && freeReading.getReadingAge() > 30000) {
-      LoggingUtil.logToRemote(Level.WARNING, "Free resource readings were " +
-          "not updated for 30 seconds - auto-sensing is disabled",
-          new IllegalStateException());
-      LOG.warning("Free resource readings were not updated for 30 seconds - "
-          + "auto-sensing is disabled");
-      setAutoSensing(false);
-    }
     return request.second;
   }
 
@@ -435,46 +377,6 @@ public class ResourceManager {
     return cpuIsAvailable && ramIsAvailable && ioIsAvailable && localTestCountIsAvailable;
   }
 
-  private synchronized void updateAvailableResources(boolean useFreeReading) {
-    Preconditions.checkNotNull(staticResources);
-    if (useFreeReading && isAutoSensingEnabled()) {
-      availableResources = ResourceSet.create(
-          usedRam + freeReading.getFreeMb(),
-          usedCpu + freeReading.getAvgFreeCpu(),
-          staticResources.getIoUsage(),
-          staticResources.getLocalTestCount());
-      if(FINE) {
-        LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB,"
-            + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU");
-      }
-      processWaitingThreads();
-    } else {
-      availableResources = ResourceSet.create(
-          staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0,
-          staticResources.getCpuUsage(),
-          staticResources.getIoUsage(),
-          staticResources.getLocalTestCount());
-      processWaitingThreads();
-    }
-  }
-
-  /**
-   * Called by the timer thread to update system load information.
-   *
-   * @return true if update was successful and false if error was detected and
-   *         autosensing was disabled.
-   */
-  private boolean refreshFreeResources() {
-    freeReading = LocalHostCapacity.getFreeResources(freeReading);
-    if (freeReading == null) { // Unable to read or parse /proc/* information.
-      LOG.warning("Unable to obtain system load - autosensing is disabled");
-      setAutoSensing(false);
-      return false;
-    }
-    updateAvailableResources(
-        freeReading.getInterval() >= 1000 && freeReading.getInterval() <= 10000);
-    return true;
-  }
 
   @VisibleForTesting
   synchronized int getWaitCount() {