diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib')
8 files changed, 145 insertions, 49 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java index 7f375d4c25..7255b5d49e 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java +++ b/src/main/java/com/google/devtools/build/lib/actions/LocalHostCapacity.java @@ -126,7 +126,8 @@ public final class LocalHostCapacity { static boolean isDisabled; // If /proc/* information is not available, assume 3000 MB and 2 CPUs. - private static ResourceSet DEFAULT_RESOURCES = ResourceSet.createWithRamCpuIo(3000.0, 2.0, 1.0); + private static ResourceSet DEFAULT_RESOURCES = ResourceSet.create(3000.0, 2.0, 1.0, + Integer.MAX_VALUE); private LocalHostCapacity() {} @@ -252,10 +253,11 @@ public final class LocalHostCapacity { boolean hyperthreading = (logicalCpuCount != totalCores); double ramMb = ProcMeminfoParser.kbToMb(memInfo.getTotalKb()); final double EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU = 0.6; - return ResourceSet.createWithRamCpuIo( + return ResourceSet.create( ramMb, logicalCpuCount * (hyperthreading ? EFFECTIVE_CPUS_PER_HYPERTHREADED_CPU : 1.0), - 1.0); + 1.0, + Integer.MAX_VALUE); } catch (IOException | IllegalArgumentException e) { disableProcFsUse(e); return DEFAULT_RESOURCES; diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java index bb2aea9fcc..db6ed118f6 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceManager.java @@ -33,14 +33,14 @@ import java.util.logging.Level; import java.util.logging.Logger; /** - * CPU/RAM resource manager. Used to keep track of resources consumed by the Blaze action execution - * threads and throttle them when necessary. + * Used to keep track of resources consumed by the Blaze action execution threads and throttle them + * when necessary. * - * <p>Threads which are known to consume a significant amount of the local CPU or RAM resources - * should call {@link #acquireResources} method. This method will check whether requested resources - * are available and will either mark them as used and allow thread to proceed or will block the - * thread until requested resources will become available. When thread completes it task, it must - * release allocated resources by calling {@link #releaseResources} method. + * <p>Threads which are known to consume a significant amount of resources should call + * {@link #acquireResources} method. This method will check whether requested resources are + * available and will either mark them as used and allow the thread to proceed or will block the + * thread until requested resources will become available. When the thread completes its task, it + * must release allocated resources by calling {@link #releaseResources} method. * * <p>Available resources can be calculated using one of three ways: * <ol> @@ -100,9 +100,9 @@ public class ResourceManager { // Please note that this value is purely empirical - we assume that generally // requested resources are somewhat pessimistic and thread would end up // using less than requested amount. - private final static double MIN_NECESSARY_CPU_RATIO = 0.6; - private final static double MIN_NECESSARY_RAM_RATIO = 1.0; - private final static double MIN_NECESSARY_IO_RATIO = 1.0; + private static final double MIN_NECESSARY_CPU_RATIO = 0.6; + private static final double MIN_NECESSARY_RAM_RATIO = 1.0; + private static final double MIN_NECESSARY_IO_RATIO = 1.0; // List of blocked threads. Associated CountDownLatch object will always // be initialized to 1 during creation in the acquire() method. @@ -129,6 +129,9 @@ public class ResourceManager { // definition in the ResourceSet class. private double usedIo; + // Used local test count. Corresponds to the local test count definition in the ResourceSet class. + private int usedLocalTestCount; + // Specifies how much of the RAM in staticResources we should allow to be used. public static final int DEFAULT_RAM_UTILIZATION_PERCENTAGE = 67; private int ramUtilizationPercentage = DEFAULT_RAM_UTILIZATION_PERCENTAGE; @@ -138,7 +141,7 @@ public class ResourceManager { private ResourceManager() { FINE = LOG.isLoggable(Level.FINE); - requestList = new LinkedList<Pair<ResourceSet, CountDownLatch>>(); + requestList = new LinkedList<>(); } @VisibleForTesting public static ResourceManager instanceForTestingOnly() { @@ -154,6 +157,7 @@ public class ResourceManager { usedCpu = 0; usedRam = 0; usedIo = 0; + usedLocalTestCount = 0; for (Pair<ResourceSet, CountDownLatch> request : requestList) { // CountDownLatch can be set only to 0 or 1. request.second.countDown(); @@ -220,7 +224,7 @@ public class ResourceManager { */ public void acquireResources(ActionMetadata owner, ResourceSet resources) throws InterruptedException { - Preconditions.checkArgument(resources != null); + Preconditions.checkNotNull(resources); long startTime = Profiler.nanoTimeMaybe(); CountDownLatch latch = null; try { @@ -231,7 +235,7 @@ public class ResourceManager { } } finally { threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0 - || resources.getIoUsage() != 0); + || resources.getIoUsage() != 0 || resources.getLocalTestCount() != 0); acquired(owner); // Profile acquisition only if it waited for resource to become available. @@ -255,7 +259,8 @@ public class ResourceManager { } if (acquired) { - threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0); + threadLocked.set(resources.getCpuUsage() != 0 || resources.getMemoryMb() != 0 + || resources.getIoUsage() != 0 || resources.getLocalTestCount() != 0); acquired(owner); } @@ -266,13 +271,15 @@ public class ResourceManager { usedCpu += resources.getCpuUsage(); usedRam += resources.getMemoryMb(); usedIo += resources.getIoUsage(); + usedLocalTestCount += resources.getLocalTestCount(); } /** * Return true if any resources have been claimed through this manager. */ public synchronized boolean inUse() { - return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || requestList.size() > 0; + return usedCpu != 0.0 || usedRam != 0.0 || usedIo != 0.0 || usedLocalTestCount != 0 + || requestList.size() > 0; } @@ -353,16 +360,18 @@ public class ResourceManager { usedCpu -= resources.getCpuUsage(); usedRam -= resources.getMemoryMb(); usedIo -= resources.getIoUsage(); + usedLocalTestCount -= resources.getLocalTestCount(); // TODO(bazel-team): (2010) rounding error can accumulate and value below can end up being // e.g. 1E-15. So if it is small enough, we set it to 0. But maybe there is a better solution. - if (usedCpu < 0.0001) { + double epsilon = 0.0001; + if (usedCpu < epsilon) { usedCpu = 0; } - if (usedRam < 0.0001) { + if (usedRam < epsilon) { usedRam = 0; } - if (usedIo < 0.0001) { + if (usedIo < epsilon) { usedIo = 0; } if (requestList.size() > 0) { @@ -393,7 +402,7 @@ public class ResourceManager { Preconditions.checkNotNull(availableResources); // Comparison below is robust, since any calculation errors will be fixed // by the release() method. - if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0) { + if (usedCpu == 0.0 && usedRam == 0.0 && usedIo == 0.0 && usedLocalTestCount == 0) { return true; } // Use only MIN_NECESSARY_???_RATIO of the resource value to check for @@ -404,10 +413,12 @@ public class ResourceManager { double cpu = resources.getCpuUsage() * MIN_NECESSARY_CPU_RATIO; double ram = resources.getMemoryMb() * MIN_NECESSARY_RAM_RATIO; double io = resources.getIoUsage() * MIN_NECESSARY_IO_RATIO; + int localTestCount = resources.getLocalTestCount(); double availableCpu = availableResources.getCpuUsage(); double availableRam = availableResources.getMemoryMb(); double availableIo = availableResources.getIoUsage(); + int availableLocalTestCount = availableResources.getLocalTestCount(); // Resources are considered available if any one of the conditions below is true: // 1) If resource is not requested at all, it is available. @@ -416,28 +427,33 @@ public class ResourceManager { // ensure that at any given time, at least one thread is able to acquire // resources even if it requests more than available. // 3) If used resource amount is less than total available resource amount. - return (cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu) && - (ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam) && - (io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo); + boolean cpuIsAvailable = cpu == 0.0 || usedCpu == 0.0 || usedCpu + cpu <= availableCpu; + boolean ramIsAvailable = ram == 0.0 || usedRam == 0.0 || usedRam + ram <= availableRam; + boolean ioIsAvailable = io == 0.0 || usedIo == 0.0 || usedIo + io <= availableIo; + boolean localTestCountIsAvailable = localTestCount == 0 || usedLocalTestCount == 0 + || usedLocalTestCount + localTestCount <= availableLocalTestCount; + return cpuIsAvailable && ramIsAvailable && ioIsAvailable && localTestCountIsAvailable; } private synchronized void updateAvailableResources(boolean useFreeReading) { Preconditions.checkNotNull(staticResources); if (useFreeReading && isAutoSensingEnabled()) { - availableResources = ResourceSet.createWithRamCpuIo( + availableResources = ResourceSet.create( usedRam + freeReading.getFreeMb(), usedCpu + freeReading.getAvgFreeCpu(), - staticResources.getIoUsage()); + staticResources.getIoUsage(), + staticResources.getLocalTestCount()); if(FINE) { LOG.fine("Free resources: " + Math.round(freeReading.getFreeMb()) + " MB," + Math.round(freeReading.getAvgFreeCpu() * 100) + "% CPU"); } processWaitingThreads(); } else { - availableResources = ResourceSet.createWithRamCpuIo( + availableResources = ResourceSet.create( staticResources.getMemoryMb() * this.ramUtilizationPercentage / 100.0, staticResources.getCpuUsage(), - staticResources.getIoUsage()); + staticResources.getIoUsage(), + staticResources.getLocalTestCount()); processWaitingThreads(); } } @@ -466,7 +482,7 @@ public class ResourceManager { } @VisibleForTesting - synchronized boolean isAvailable(double ram, double cpu, double io) { - return areResourcesAvailable(ResourceSet.createWithRamCpuIo(ram, cpu, io)); + synchronized boolean isAvailable(double ram, double cpu, double io, int localTestCount) { + return areResourcesAvailable(ResourceSet.create(ram, cpu, io, localTestCount)); } } diff --git a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java index 65d2d1453f..f4f1327bb1 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ResourceSet.java @@ -33,7 +33,7 @@ import java.util.NoSuchElementException; public class ResourceSet { /** For actions that consume negligible resources. */ - public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0); + public static final ResourceSet ZERO = new ResourceSet(0.0, 0.0, 0.0, 0); /** The amount of real memory (resident set size). */ private final double memoryMb; @@ -41,20 +41,49 @@ public class ResourceSet { /** The number of CPUs, or fractions thereof. */ private final double cpuUsage; + /** The number of local tests. */ + private final int localTestCount; + /** * Relative amount of used I/O resources (with 1.0 being total available amount on an "average" * workstation. */ private final double ioUsage; - private ResourceSet(double memoryMb, double cpuUsage, double ioUsage) { + private ResourceSet(double memoryMb, double cpuUsage, double ioUsage, int localTestCount) { this.memoryMb = memoryMb; this.cpuUsage = cpuUsage; this.ioUsage = ioUsage; + this.localTestCount = localTestCount; } + /** + * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, and ioUsage, and + * with 0.0 for localTestCount. Use this method in action resource definitions when they aren't + * local tests. + */ public static ResourceSet createWithRamCpuIo(double memoryMb, double cpuUsage, double ioUsage) { - return new ResourceSet(memoryMb, cpuUsage, ioUsage); + return new ResourceSet(memoryMb, cpuUsage, ioUsage, 0); + } + + /** + * Returns a new ResourceSet with the provided value for localTestCount, and 0.0 for memoryMb, + * cpuUsage, and ioUsage. Use this method in action resource definitions when they are local tests + * that acquire no local resources. + */ + public static ResourceSet createWithLocalTestCount(int localTestCount) { + return new ResourceSet(0.0, 0.0, 0.0, localTestCount); + } + + /** + * Returns a new ResourceSet with the provided values for memoryMb, cpuUsage, ioUsage, and + * localTestCount. Most action resource definitions should use + * {@link #createWithRamCpuIo(double, double, double)} or {@link #createWithLocalTestCount(int)}. + * Use this method primarily when constructing ResourceSets that represent available resources. + */ + public static ResourceSet create(double memoryMb, double cpuUsage, double ioUsage, + int localTestCount) { + return new ResourceSet(memoryMb, cpuUsage, ioUsage, localTestCount); } /** Returns the amount of real memory (resident set size) used in MB. */ @@ -83,6 +112,11 @@ public class ResourceSet { return ioUsage; } + /** Returns the local test count used. */ + public int getLocalTestCount() { + return localTestCount; + } + public static class ResourceSetConverter implements Converter<ResourceSet> { private static final Splitter SPLITTER = Splitter.on(','); @@ -99,11 +133,9 @@ public class ResourceSet { if (memoryMb <= 0.0 || cpuUsage <= 0.0 || ioUsage <= 0.0) { throw new OptionsParsingException("All resource values must be positive"); } - return createWithRamCpuIo(memoryMb, cpuUsage, ioUsage); - } catch (NumberFormatException nfe) { + return create(memoryMb, cpuUsage, ioUsage, Integer.MAX_VALUE); + } catch (NumberFormatException | NoSuchElementException nfe) { throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nfe); - } catch (NoSuchElementException nsee) { - throw new OptionsParsingException("Expected exactly 3 comma-separated float values", nsee); } } diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java index 18a03ad8bd..0b56fa06d8 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java @@ -460,6 +460,13 @@ public class BuildRequest implements OptionsClassProvider { } /** + * Returns the set of execution options specified for this request. + */ + public ExecutionOptions getExecutionOptions() { + return getOptions(ExecutionOptions.class); + } + + /** * Returns the human-readable description of the non-default options * for this build request. */ @@ -497,6 +504,18 @@ public class BuildRequest implements OptionsClassProvider { String.format("High value for --jobs: %d. You may run into memory issues", jobs)); } + int localTestJobs = getExecutionOptions().localTestJobs; + if (localTestJobs < 0) { + throw new InvalidConfigurationException(String.format( + "Invalid parameter for --local_test_jobs: %d. Only values 0 or greater are " + + "allowed.", localTestJobs)); + } + if (localTestJobs > jobs) { + warnings.add( + String.format("High value for --local_test_jobs: %d. This exceeds the value for --jobs: " + + "%d. Only up to %d local tests will run concurrently.", localTestJobs, jobs, jobs)); + } + // Validate other BuildRequest options. if (getBuildOptions().verboseExplanations && getBuildOptions().explanationPath == null) { warnings.add("--verbose_explanations has no effect when --explain=<file> is not enabled"); diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 0c8d404935..b248b05c7f 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -42,6 +42,7 @@ import com.google.devtools.build.lib.actions.Executor.ActionContext; import com.google.devtools.build.lib.actions.ExecutorInitException; import com.google.devtools.build.lib.actions.LocalHostCapacity; import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.ResourceSet; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.TestExecException; import com.google.devtools.build.lib.actions.cache.ActionCache; @@ -760,11 +761,12 @@ public class ExecutionTool { private void configureResourceManager(BuildRequest request) { ResourceManager resourceMgr = ResourceManager.instance(); ExecutionOptions options = request.getOptions(ExecutionOptions.class); + ResourceSet resources; if (options.availableResources != null) { - resourceMgr.setAvailableResources(options.availableResources); + resources = options.availableResources; resourceMgr.setRamUtilizationPercentage(100); } else { - resourceMgr.setAvailableResources(LocalHostCapacity.getLocalHostCapacity()); + resources = LocalHostCapacity.getLocalHostCapacity(); resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage); if (options.useResourceAutoSense) { getReporter().handle( @@ -772,6 +774,14 @@ public class ExecutionTool { } ResourceManager.instance().setAutoSensing(/*autosense=*/false); } + + resourceMgr.setAvailableResources(ResourceSet.create( + resources.getMemoryMb(), + resources.getCpuUsage(), + resources.getIoUsage(), + request.getExecutionOptions().usingLocalTestJobs() + ? request.getExecutionOptions().localTestJobs : Integer.MAX_VALUE + )); } /** diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java index 58e360bfe0..a12ece9f51 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java @@ -192,4 +192,17 @@ public class ExecutionOptions extends OptionsBase { converter = ResourceSet.ResourceSetConverter.class ) public ResourceSet availableResources; + + @Option(name = "local_test_jobs", + defaultValue = "0", + category = "testing", + help = "The max number of local test jobs to run concurrently. " + + "0 means local resources will limit the number of local test jobs to run " + + "concurrently instead. Setting this greater than the value for --jobs is ineffectual." + ) + public int localTestJobs; + + public boolean usingLocalTestJobs() { + return localTestJobs != 0; + } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java index 72970a83ae..d0ff9573fc 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/rules/test/StandaloneTestStrategy.java @@ -86,7 +86,7 @@ public class StandaloneTestStrategy extends TestStrategy { Spawn spawn = new BaseSpawn(getArgs(action), env, action.getTestProperties().getExecutionInfo(), action, - action.getTestProperties().getLocalResourceUsage()); + action.getTestProperties().getLocalResourceUsage(executionOptions.usingLocalTestJobs())); Executor executor = actionExecutionContext.getExecutor(); @@ -97,7 +97,8 @@ public class StandaloneTestStrategy extends TestStrategy { fileOutErr = new FileOutErr(action.getTestLog().getPath(), action.resolve(actionExecutionContext.getExecutor().getExecRoot()).getTestStderr()); - resources = action.getTestProperties().getLocalResourceUsage(); + resources = action.getTestProperties() + .getLocalResourceUsage(executionOptions.usingLocalTestJobs()); ResourceManager.instance().acquireResources(action, resources); TestResultData data = execute( actionExecutionContext.withFileOutErr(fileOutErr), spawn, action); diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java index 0834ca9464..0e0befc4d2 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestTargetProperties.java @@ -38,11 +38,12 @@ public class TestTargetProperties { /** * Resources used by local tests of various sizes. */ - private static final ResourceSet SMALL_RESOURCES = ResourceSet.createWithRamCpuIo(20, 0.9, 0.00); - private static final ResourceSet MEDIUM_RESOURCES = ResourceSet.createWithRamCpuIo(100, 0.9, 0.1); - private static final ResourceSet LARGE_RESOURCES = ResourceSet.createWithRamCpuIo(300, 0.8, 0.1); - private static final ResourceSet ENORMOUS_RESOURCES = - ResourceSet.createWithRamCpuIo(800, 0.7, 0.4); + private static final ResourceSet SMALL_RESOURCES = ResourceSet.create(20, 0.9, 0.00, 1); + private static final ResourceSet MEDIUM_RESOURCES = ResourceSet.create(100, 0.9, 0.1, 1); + private static final ResourceSet LARGE_RESOURCES = ResourceSet.create(300, 0.8, 0.1, 1); + private static final ResourceSet ENORMOUS_RESOURCES = ResourceSet.create(800, 0.7, 0.4, 1); + private static final ResourceSet LOCAL_TEST_JOBS_BASED_RESOURCES = + ResourceSet.createWithLocalTestCount(1); private static ResourceSet getResourceSetFromSize(TestSize size) { switch (size) { @@ -115,8 +116,10 @@ public class TestTargetProperties { return isExternal; } - public ResourceSet getLocalResourceUsage() { - return TestTargetProperties.getResourceSetFromSize(size); + public ResourceSet getLocalResourceUsage(boolean usingLocalTestJobs) { + return usingLocalTestJobs + ? LOCAL_TEST_JOBS_BASED_RESOURCES + : TestTargetProperties.getResourceSetFromSize(size); } /** |