diff options
author | ulfjack <ulfjack@google.com> | 2018-08-01 01:04:56 -0700 |
---|---|---|
committer | Copybara-Service <copybara-piper@google.com> | 2018-08-01 01:06:47 -0700 |
commit | c6876545a6b3170f535e70977c4b633f2a7f9092 (patch) | |
tree | 17f3ac5be5ca0e3cb1049fe38f1257c033df3e42 /src/main | |
parent | 6e7fae11cb22a41a4cd73d5dead93aa7567b5eda (diff) |
Open source SpawnMetrics
PiperOrigin-RevId: 206893284
Diffstat (limited to 'src/main')
3 files changed, 252 insertions, 9 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnExecutedEvent.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnExecutedEvent.java index c9edc10a85..0da2779665 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/SpawnExecutedEvent.java +++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnExecutedEvent.java @@ -13,23 +13,19 @@ // limitations under the License. package com.google.devtools.build.lib.actions; -import java.time.Duration; /** This event is fired during the build, when a subprocess is executed. */ public class SpawnExecutedEvent { private final ActionAnalysisMetadata actionMetadata; private final int exitCode; - private final Duration totalTime; private final SpawnResult result; public SpawnExecutedEvent( ActionAnalysisMetadata actionMetadata, int exitCode, - Duration totalTime, SpawnResult result) { this.actionMetadata = actionMetadata; this.exitCode = exitCode; - this.totalTime = totalTime; this.result = result; } @@ -43,11 +39,6 @@ public class SpawnExecutedEvent { return exitCode; } - /** Returns the total time of the subprocess; may include network round trip. */ - public Duration getTotalTime() { - return totalTime; - } - /** Returns the distributor reply. */ public SpawnResult getSpawnResult() { return result; diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java new file mode 100644 index 0000000000..3ac5504360 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnMetrics.java @@ -0,0 +1,248 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.actions; + +import com.google.common.base.Joiner; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** Timing, size, and memory statistics for a Spawn execution. */ +public final class SpawnMetrics { + /** Any non important stats < than 10% will not be shown in the summary. */ + private static final double STATS_SHOW_THRESHOLD = 0.10; + + /** Represents a zero cost/null statistic. */ + public static final SpawnMetrics EMPTY = + new SpawnMetrics( + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + 0L, + 0L, + 0L); + + public static SpawnMetrics forLocalExecution(Duration wallTime) { + return new SpawnMetrics( + wallTime, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + Duration.ZERO, + wallTime, + Duration.ZERO, + 0L, + 0L, + 0L); + } + + private final Duration totalTime; + private final Duration parseTime; + private final Duration fetchTime; + private final Duration remoteQueueTime; + private final Duration uploadTime; + private final Duration setupTime; + private final Duration executionWallTime; + private final Duration retryTime; + private final Duration networkTime; + private final long inputBytes; + private final long inputFiles; + private final long memoryEstimateBytes; + + public SpawnMetrics( + Duration totalTime, + Duration parseTime, + Duration networkTime, + Duration fetchTime, + Duration remoteQueueTime, + Duration setupTime, + Duration uploadTime, + Duration executionWallTime, + Duration retryTime, + long inputBytes, + long inputFiles, + long memoryEstimateBytes) { + this.totalTime = totalTime; + this.parseTime = parseTime; + this.networkTime = networkTime; + this.fetchTime = fetchTime; + this.remoteQueueTime = remoteQueueTime; + this.setupTime = setupTime; + this.uploadTime = uploadTime; + this.executionWallTime = executionWallTime; + this.retryTime = retryTime; + this.inputBytes = inputBytes; + this.inputFiles = inputFiles; + this.memoryEstimateBytes = memoryEstimateBytes; + } + + /** + * Total (measured locally) wall time spent running a spawn. This should be at least as large as + * all the other times summed together. + */ + public Duration totalTime() { + return totalTime; + } + + /** + * Total time spent getting on network. This includes time getting network-side errors and the + * time of the round-trip, found by taking the difference of wall time here and the server time + * reported by the RPC. This is 0 for locally executed spawns. + */ + public Duration networkTime() { + return networkTime; + } + + /** + * Total time waiting in remote queues. Includes queue time for any failed attempts. This is 0 for + * locally executed spawns. + */ + public Duration remoteQueueTime() { + return remoteQueueTime; + } + + /** The time spent transferring files to the backends. This is 0 for locally executed spawns. */ + public Duration uploadTime() { + return uploadTime; + } + + /** + * The time required to setup the environment in which the spawn is run. This may be 0 for locally + * executed spawns, or may include time to setup a sandbox or other environment. Does not include + * failed attempts. + */ + public Duration setupTime() { + return setupTime; + } + + /** Time spent running the subprocess. */ + public Duration executionWallTime() { + return executionWallTime; + } + + /** + * The time taken to convert the spawn into a network request, e.g., collecting runfiles, and + * digests for all input files. + */ + public Duration parseTime() { + return parseTime; + } + + /** Total time spent fetching remote outputs. */ + public Duration fetchTime() { + return fetchTime; + } + + /** Time spent in previous failed attempts. Does not include queue time. */ + public Duration retryTime() { + return retryTime; + } + + /** Any time that is not measured by a more specific component, out of {@code totalTime()}. */ + public Duration otherTime() { + return totalTime + .minus(parseTime) + .minus(networkTime) + .minus(remoteQueueTime) + .minus(uploadTime) + .minus(setupTime) + .minus(executionWallTime) + .minus(fetchTime) + .minus(retryTime); + } + + /** Total size in bytes of inputs or 0 if unavailable. */ + public long inputBytes() { + return inputBytes; + } + + /** Total number of input files or 0 if unavailable. */ + public long inputFiles() { + return inputFiles; + } + + /** Estimated memory usage or 0 if unavailable. */ + public long memoryEstimate() { + return memoryEstimateBytes; + } + + /** + * Generates a String representation of the stats. + * + * @param total total time used to compute the percentages + * @param summary whether to exclude input file count and sizes, and memory estimates + */ + public String toString(Duration total, boolean summary) { + StringBuilder sb = new StringBuilder(); + sb.append("("); + sb.append(prettyPercentage(totalTime, total)); + sb.append(" of the time): ["); + List<String> stats = new ArrayList<>(8); + addStatToString(stats, "parse", !summary, parseTime, total); + addStatToString(stats, "queue", true, remoteQueueTime, total); + addStatToString(stats, "network", !summary, networkTime, total); + addStatToString(stats, "upload", !summary, uploadTime, total); + addStatToString(stats, "setup", true, setupTime, total); + addStatToString(stats, "process", true, executionWallTime, total); + addStatToString(stats, "fetch", !summary, fetchTime, total); + addStatToString(stats, "retry", !summary, retryTime, total); + addStatToString(stats, "other", !summary, otherTime(), total); + if (!summary) { + stats.add("input files: " + inputFiles); + stats.add("input bytes: " + inputBytes); + stats.add("memory bytes: " + memoryEstimateBytes); + } + Joiner.on(", ").appendTo(sb, stats); + sb.append("]"); + return sb.toString(); + } + + /** + * Add to {@code strings} the string representation of {@code name} component. If {@code + * forceShow} is set to false it will only show if it is above certain threshold. + */ + private static void addStatToString( + List<String> strings, String name, boolean forceShow, Duration time, Duration totalTime) { + if (forceShow || isAboveThreshold(time, totalTime)) { + strings.add(name + ": " + prettyPercentage(time, totalTime)); + } + } + + private static boolean isAboveThreshold(Duration time, Duration totalTime) { + return totalTime.toMillis() > 0 + && (((float) time.toMillis() / totalTime.toMillis()) >= STATS_SHOW_THRESHOLD); + } + + /** + * Converts relative duration to the percentage string. + * + * @return formatted percentage string or "N/A" if result is undefined + */ + private static String prettyPercentage(Duration duration, Duration total) { + // Duration.toMillis() != 0 does not imply !Duration.isZero() (due to truncation). + if (total.toMillis() == 0) { + // Return "not available" string if total is 0 and result is undefined. + return "N/A"; + } + return String.format("%.2f%%", duration.toMillis() * 100.0 / total.toMillis()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java b/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java index 2eaca3a55d..65a8417042 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java +++ b/src/main/java/com/google/devtools/build/lib/actions/SpawnResult.java @@ -182,6 +182,10 @@ public interface SpawnResult { */ Optional<Long> getNumInvoluntaryContextSwitches(); + default SpawnMetrics getMetrics() { + return SpawnMetrics.forLocalExecution(getWallTime().orElse(Duration.ZERO)); + } + /** Whether the spawn result was a cache hit. */ boolean isCacheHit(); |