diff options
author | 2015-04-13 15:18:26 +0000 | |
---|---|---|
committer | 2015-04-14 14:30:25 +0000 | |
commit | 3c380eb68aca20791a2c12dd2ccb140ef56c0931 (patch) | |
tree | c79711b739b4aa68d0d2065dac1499ea48463840 /src/main/java/com/google | |
parent | 37c64ee5236d35536b5b003697b6fcff4f457e00 (diff) |
Remove autosense logic.
Polling the machine load can never work, because the following
scenarios are quite common:
* Tasks that are faster than the poll cycle time.
* Tasks whose CPU and/or RAM consumption changes over the lifetime of
the task.
--
MOS_MIGRATED_REVID=90990445
Diffstat (limited to 'src/main/java/com/google')
4 files changed, 11 insertions, 284 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java index 7255b5d49e..b43a1e2ff7 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java +++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java @@ -17,11 +17,7 @@ package com.google.devtools.build.lib.actions; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.io.Files; -import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; -import com.google.devtools.build.lib.util.BlazeClock; -import com.google.devtools.build.lib.util.Clock; -import com.google.devtools.build.lib.util.LoggingUtil; import com.google.devtools.build.lib.util.ProcMeminfoParser; import java.io.File; @@ -29,8 +25,6 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.HashSet; import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; /** * This class estimates the local host's resource capacity. @@ -38,93 +32,6 @@ import java.util.logging.Logger; @ThreadCompatible public final class LocalHostCapacity { - private static final Logger LOG = Logger.getLogger(LocalHostCapacity.class.getName()); - - /** - * Stores parsed /proc/stat CPU time counters. - * See {@link LocalHostCapacity#getCpuTimes(String)} for details. - */ - @Immutable - private final static class CpuTimes { - private final long idleJiffies; - private final long totalJiffies; - - CpuTimes(long idleJiffies, long totalJiffies) { - this.idleJiffies = idleJiffies; - this.totalJiffies = totalJiffies; - } - - /** - * Return idle CPU ratio using current and previous CPU readings or 0 if - * ratio is undefined. - */ - double getIdleRatio(CpuTimes prevTimes) { - if (prevTimes.totalJiffies == 0 || totalJiffies == prevTimes.totalJiffies) { - return 0; - } - return ((double)(idleJiffies - prevTimes.idleJiffies) / - (double)(totalJiffies - prevTimes.totalJiffies)); - } - } - - /** - * Used to store available local CPU and RAM resources information. - * See {@link LocalHostCapacity#getFreeResources(FreeResources)} for details. - */ - public static final class FreeResources { - - private final Clock clock; - private final CpuTimes cpuTimes; - private final long lastTimestamp; - private final double freeCpu; - private final double freeMb; - private final long interval; - - private FreeResources(Clock localClock, ProcMeminfoParser memInfo, String statContent, - FreeResources prevStats) { - clock = localClock; - lastTimestamp = localClock.nanoTime(); - freeMb = ProcMeminfoParser.kbToMb(memInfo.getFreeRamKb()); - cpuTimes = getCpuTimes(statContent); - if (prevStats == null) { - interval = 0; - freeCpu = 0.0; - } else { - interval = lastTimestamp - prevStats.lastTimestamp; - freeCpu = getLocalHostCapacity().getCpuUsage() * cpuTimes.getIdleRatio(prevStats.cpuTimes); - } - } - - /** - * Returns amount of available RAM in MB. - */ - public double getFreeMb() { return freeMb; } - - /** - * Returns average available CPU resources (as a fraction of the CPU core, - * so one fully CPU-bound thread should consume exactly 1.0 CPU resource). - */ - public double getAvgFreeCpu() { return freeCpu; } - - /** - * Returns interval in ms between CPU load measurements used to calculate - * average available CPU resources. - */ - public long getInterval() { return interval / 1000000; } - - /** - * Returns age of available resource data in ms. - */ - public long getReadingAge() { - return (clock.nanoTime() - lastTimestamp) / 1000000; - } - } - - // Disables getFreeResources() if error occured during reading or parsing - // /proc/* information. - @VisibleForTesting - static boolean isDisabled; - // If /proc/* information is not available, assume 3000 MB and 2 CPUs. private static ResourceSet DEFAULT_RESOURCES = ResourceSet.create(3000.0, 2.0, 1.0, Integer.MAX_VALUE); @@ -148,21 +55,6 @@ public final class LocalHostCapacity { return localHostCapacity; } - /** - * Returns new FreeResources object populated with free RAM information from - * /proc/meminfo and CPU load information from the /proc/stat. First call - * should be made with null parameter to instantiate new FreeResources object. - * Subsequent calls will use information inside it to calculate average CPU - * load over the time between calls and to calculate amount of free CPU - * resources and generate new FreeResources() instance. - * - * If information is not available due to error, functionality will be disabled - * and method will always return null. - */ - public static FreeResources getFreeResources(FreeResources stats) { - return getFreeResources(BlazeClock.instance(), "/proc/meminfo", "/proc/stat", stats); - } - private static final Splitter NEWLINE_SPLITTER = Splitter.on('\n').omitEmptyStrings(); @VisibleForTesting @@ -212,35 +104,6 @@ public final class LocalHostCapacity { return coresPerCpu; } - /** - * Parses cpu line of the /proc/stats, calculates number of idle and total - * CPU jiffies and returns CpuTimes instance with that information. - * - * Total CPU time includes <b>all</b> time reported to be spent by the CPUs, - * including so-called "stolen" time - time spent by other VMs on the same - * workstation. - */ - private static CpuTimes getCpuTimes(String statContent) { - String[] cpuStats = statContent.substring(0, statContent.indexOf('\n')).trim().split(" +"); - // Supported versions of /proc/stat (Linux kernel 2.6.x) must contain either - // 9 or 10 fields: - // "cpu" utime ultime stime idle iowait irq softirq steal(since 2.6.11) 0 - // We are interested in total time (sum of all columns) and idle time. - if (cpuStats.length < 9 | cpuStats.length > 10) { - throw new IllegalArgumentException("Unrecognized /proc/stat format"); - } - if (!cpuStats[0].equals("cpu")) { - throw new IllegalArgumentException("/proc/stat does not start with cpu keyword"); - } - long idleCpuJiffies = Long.parseLong(cpuStats[4]); // "idle" column. - long totalJiffies = 0; - for (int i = 1; i < cpuStats.length; i++) { - totalJiffies += Long.parseLong(cpuStats[i]); - } - long totalCpuJiffies = totalJiffies; - return new CpuTimes(idleCpuJiffies, totalCpuJiffies); - } - @VisibleForTesting static ResourceSet getLocalHostCapacity(String cpuinfoFile, String meminfoFile) { try { @@ -259,45 +122,20 @@ public final class LocalHostCapacity { 1.0, Integer.MAX_VALUE); } catch (IOException | IllegalArgumentException e) { - disableProcFsUse(e); return DEFAULT_RESOURCES; } } - @VisibleForTesting - static FreeResources getFreeResources(Clock localClock, String meminfoFile, String statFile, - FreeResources prevStats) { - if (isDisabled) { return null; } - try { - String statContent = readContent(statFile); - return new FreeResources(localClock, new ProcMeminfoParser(meminfoFile), - statContent, prevStats); - } catch (IOException | IllegalArgumentException e) { - disableProcFsUse(e); - return null; - } - } - /** * For testing purposes only. Do not use it. */ @VisibleForTesting static void setLocalHostCapacity(ResourceSet resources) { localHostCapacity = resources; - isDisabled = false; } private static String readContent(String filename) throws IOException { return Files.toString(new File(filename), Charset.defaultCharset()); } - /** - * Disables use of /proc filesystem. Called internally when unexpected - * exception is caught. - */ - private static void disableProcFsUse(Throwable cause) { - LoggingUtil.logToRemote(Level.WARNING, "Unable to read system load or capacity", cause); - LOG.log(Level.WARNING, "Unable to read system load or capacity", cause); - isDisabled = true; - } } diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java index 8e6220c143..05be3370ea 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java @@ -20,17 +20,12 @@ import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; -import com.google.devtools.build.lib.util.LoggingUtil; import com.google.devtools.build.lib.util.Pair; import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.CountDownLatch; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Used to keep track of resources consumed by the Blaze action execution threads and throttle them @@ -52,10 +47,6 @@ import java.util.logging.Logger; * amount of the total available memory and will limit itself to the number of effective cores * and 2/3 of the available memory. For details, please look at the {@link * LocalHostCapacity#getLocalHostCapacity} method. - * <li>Blaze will periodically (every 3 seconds) poll {@code /proc/meminfo} and {@code /proc/stat} - * information to obtain how much RAM and CPU resources are currently idle at that moment. For - * calculation details, please look at the {@link LocalHostCapacity#getFreeResources} - * implementation. * </ol> * * <p>The resource manager also allows a slight overallocation of the resources to account for the @@ -67,9 +58,6 @@ import java.util.logging.Logger; @ThreadSafe public class ResourceManager { - private static final Logger LOG = Logger.getLogger(ResourceManager.class.getName()); - private final boolean FINE; - private EventBus eventBus; private final ThreadLocal<Boolean> threadLocked = new ThreadLocal<Boolean>() { @@ -114,7 +102,6 @@ public class ResourceManager { private ResourceSet staticResources = null; private ResourceSet availableResources = null; - private LocalHostCapacity.FreeResources freeReading = null; // Used amount of CPU capacity (where 1.0 corresponds to the one fully // occupied CPU core. Corresponds to the CPU resource definition in the @@ -136,11 +123,7 @@ public class ResourceManager { public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67; private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE; - // Timer responsible for the periodic polling of the current system load. - private Timer timer = null; - private ResourceManager() { - FINE = LOG.isLoggable(Level.FINE); requestList = new LinkedList<>(); } @@ -150,8 +133,8 @@ public class ResourceManager { /** * Resets resource manager state and releases all thread locks. - * Note - it does not reset auto-sensing or available resources. Use - * separate call to setAvailableResoures() or to setAutoSensing(). + * Note - it does not reset available resources. Use + * separate call to setAvailableResoures(). */ public synchronized void resetResourceUsage() { usedCpu = 0; @@ -168,17 +151,16 @@ public class ResourceManager { /** * Sets available resources using given resource set. Must be called * at least once before using resource manager. - * <p> - * Method will also disable auto-sensing if it was enabled. */ public synchronized void setAvailableResources(ResourceSet resources) { Preconditions.checkNotNull(resources); staticResources = resources; - setAutoSensing(false); - } - - public synchronized boolean isAutoSensingEnabled() { - return timer != null; + availableResources = ResourceSet.create( + staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0, + staticResources.getCpuUsage(), + staticResources.getIoUsage(), + staticResources.getLocalTestCount()); + processWaitingThreads(); } /** @@ -190,35 +172,6 @@ public class ResourceManager { } /** - * Enables or disables secondary resource allocation algorithm that will - * periodically (when needed but at most once per 3 seconds) checks real - * amount of available memory (based on /proc/meminfo) and current CPU load - * (based on 1 second difference of /proc/stat) and allows additional resource - * acquisition if previous requests were overly pessimistic. - */ - public synchronized void setAutoSensing(boolean enable) { - // Create new Timer instance only if it does not exist already. - if (enable && !isAutoSensingEnabled()) { - Profiler.instance().logEvent(ProfilerTask.INFO, "Enable auto sensing"); - if(refreshFreeResources()) { - timer = new Timer("AutoSenseTimer", true); - timer.schedule(new TimerTask() { - @Override public void run() { refreshFreeResources(); } - }, 3000, 3000); - } - } else if (!enable) { - if (isAutoSensingEnabled()) { - Profiler.instance().logEvent(ProfilerTask.INFO, "Disable auto sensing"); - timer.cancel(); - timer = null; - } - if (staticResources != null) { - updateAvailableResources(false); - } - } - } - - /** * Acquires requested resource set. Will block if resource is not available. * NB! This method must be thread-safe! */ @@ -315,7 +268,7 @@ public class ResourceManager { } /** - * Releases previously requested resource set. + * Releases previously requested resource =. * * <p>NB! This method must be thread-safe! */ @@ -342,17 +295,6 @@ public class ResourceManager { Pair<ResourceSet, CountDownLatch> request = new Pair<>(resources, new CountDownLatch(1)); requestList.add(request); - - // If we use auto sensing and there has not been an update within last - // 30 seconds, something has gone really wrong - disable it. - if (isAutoSensingEnabled() && freeReading.getReadingAge() > 30000) { - LoggingUtil.logToRemote(Level.WARNING, "Free resource readings were " + - "not updated for 30 seconds - auto-sensing is disabled", - new IllegalStateException()); - LOG.warning("Free resource readings were not updated for 30 seconds - " - + "auto-sensing is disabled"); - setAutoSensing(false); - } return request.second; } @@ -435,46 +377,6 @@ public class ResourceManager { return cpuIsAvailable && ramIsAvailable && ioIsAvailable && localTestCountIsAvailable; } - private synchronized void updateAvailableResources(boolean useFreeReading) { - Preconditions.checkNotNull(staticResources); - if (useFreeReading && isAutoSensingEnabled()) { - availableResources = ResourceSet.create( - usedRam + freeReading.getFreeMb(), - usedCpu + freeReading.getAvgFreeCpu(), - staticResources.getIoUsage(), - staticResources.getLocalTestCount()); - if(FINE) { - LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB," - + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU"); - } - processWaitingThreads(); - } else { - availableResources = ResourceSet.create( - staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0, - staticResources.getCpuUsage(), - staticResources.getIoUsage(), - staticResources.getLocalTestCount()); - processWaitingThreads(); - } - } - - /** - * Called by the timer thread to update system load information. - * - * @return true if update was successful and false if error was detected and - * autosensing was disabled. - */ - private boolean refreshFreeResources() { - freeReading = LocalHostCapacity.getFreeResources(freeReading); - if (freeReading == null) { // Unable to read or parse /proc/* information. - LOG.warning("Unable to obtain system load - autosensing is disabled"); - setAutoSensing(false); - return false; - } - updateAvailableResources( - freeReading.getInterval() >= 1000 && freeReading.getInterval() <= 10000); - return true; - } @VisibleForTesting synchronized int getWaitCount() { diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 77339bf641..65ad7a7550 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -417,8 +417,6 @@ public class ExecutionTool { skyframeExecutor.getOutputDirtyFilesAndClear(), skyframeExecutor.getModifiedFilesDuringPreviousBuildAndClear())); - // Disable system load polling (noop if it was not enabled). - ResourceManager.instance().setAutoSensing(false); executor.executionPhaseEnding(); for (ActionContextProvider actionContextProvider : actionContextProviders) { actionContextProvider.executionPhaseEnding(); @@ -763,11 +761,6 @@ public class ExecutionTool { } else { resources = LocalHostCapacity.getLocalHostCapacity(); resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage); - if (options.useResourceAutoSense) { - getReporter().handle( - Event.warn("Not using resource autosense due to known responsiveness issues")); - } - ResourceManager.instance().setAutoSensing(/*autosense=*/false); } resourceMgr.setAvailableResources(ResourceSet.create( diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java index a12ece9f51..93d51618a2 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java @@ -152,15 +152,10 @@ public class ExecutionOptions extends OptionsBase { + "to use its default timeouts for that category.") public Map<TestTimeout, Integer> testTimeout; - @Option(name = "resource_autosense", defaultValue = "false", category = "strategy", - help = "Periodically (every 3 seconds) poll system CPU load and available memory " - + "and allow execution of build commands if system has sufficient idle CPU and " - + "free RAM resources. By default this option is disabled, and Blaze will rely on " - + "approximation algorithms based on the total amount of available memory and number " - + "of CPU cores.") + help = "This flag has no effect, and is deprecated") public boolean useResourceAutoSense; @Option(name = "ram_utilization_factor", @@ -176,7 +171,6 @@ public class ExecutionOptions extends OptionsBase { + "higher or much lower than specified. " + "Note also that this option does not affect the amount of memory that the Blaze " + "server itself will use. " - + "Also, this option has no effect if --resource_autosense is enabled." ) public int ramUtilizationPercentage; @@ -188,7 +182,7 @@ public class ExecutionOptions extends OptionsBase { + "and number of CPU cores available for the locally executed build actions. It would also " + "assume default I/O capabilities of the local workstation (1.0). This options allows to " + "explicitly set all 3 values. Note, that if this option is used, Blaze will ignore " - + "both --ram_utilization_factor and --resource_autosense values.", + + "--ram_utilization_factor.", converter = ResourceSet.ResourceSetConverter.class ) public ResourceSet availableResources; |