diff options
Diffstat (limited to 'src')
9 files changed, 330 insertions, 22 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtil.java b/src/main/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtil.java index 0aabf2ba00..7f6d9fe663 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtil.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtil.java @@ -57,6 +57,7 @@ public final class ProcessWrapperUtil { private Optional<String> stderrPath; private Optional<Duration> timeout; private Optional<Duration> killDelay; + private Optional<String> statisticsPath; private Optional<List<String>> commandArguments; private Optional<String> processWrapperPath; @@ -65,6 +66,7 @@ public final class ProcessWrapperUtil { this.stderrPath = Optional.empty(); this.timeout = Optional.empty(); this.killDelay = Optional.empty(); + this.statisticsPath = Optional.empty(); this.commandArguments = Optional.empty(); this.processWrapperPath = Optional.empty(); } @@ -96,6 +98,12 @@ public final class ProcessWrapperUtil { return this; } + /** Sets the path for writing execution statistics (e.g. resource usage). */ + public CommandLineBuilder setStatisticsPath(String statisticsPath) { + this.statisticsPath = Optional.of(statisticsPath); + return this; + } + /** Sets the command (and its arguments) to run using the process wrapper tool. */ public CommandLineBuilder setCommandArguments(List<String> commandArguments) { this.commandArguments = Optional.of(commandArguments); @@ -129,6 +137,9 @@ public final class ProcessWrapperUtil { if (stderrPath.isPresent()) { fullCommandLine.add("--stderr=" + stderrPath.get()); } + if (statisticsPath.isPresent()) { + fullCommandLine.add("--stats=" + statisticsPath.get()); + } fullCommandLine.addAll(commandArguments.get()); diff --git a/src/main/java/com/google/devtools/build/lib/shell/BUILD b/src/main/java/com/google/devtools/build/lib/shell/BUILD index 2432831365..8b8d9301a5 100644 --- a/src/main/java/com/google/devtools/build/lib/shell/BUILD +++ b/src/main/java/com/google/devtools/build/lib/shell/BUILD @@ -14,6 +14,7 @@ java_library( name = "shell", srcs = glob(["*.java"]), deps = [ + "//src/main/protobuf:execution_statistics_java_proto", "//third_party:auto_value", "//third_party:guava", ], @@ -25,7 +26,10 @@ load("//tools/build_rules:java_rules_skylark.bzl", "bootstrap_java_library") bootstrap_java_library( name = "shell-skylark", - srcs = glob(["*.java"]), + srcs = glob( + ["*.java"], + exclude = ["ExecutionStatistics.java"], + ), jars = [ "//third_party:auto_value-jars", "//third_party:bootstrap_guava_and_error_prone-jars", diff --git a/src/main/java/com/google/devtools/build/lib/shell/ExecutionStatistics.java b/src/main/java/com/google/devtools/build/lib/shell/ExecutionStatistics.java new file mode 100644 index 0000000000..f1972ac847 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/shell/ExecutionStatistics.java @@ -0,0 +1,66 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.Optional; + +/** Provides execution statistics (e.g. resource usage) for external commands. */ +public final class ExecutionStatistics { + private final com.google.devtools.build.lib.shell.Protos.ExecutionStatistics + executionStatisticsProto; + + /** + * Provides execution statistics based on a {@code execution_statistics.proto} file. + * + * @param executionStatisticsProtoPath path to a materialized ExecutionStatistics proto + */ + public ExecutionStatistics(String executionStatisticsProtoPath) throws IOException { + try (InputStream protoInputStream = + new BufferedInputStream(new FileInputStream(executionStatisticsProtoPath))) { + executionStatisticsProto = + com.google.devtools.build.lib.shell.Protos.ExecutionStatistics.parseFrom( + protoInputStream); + } + } + + /** Returns the user time for command execution, if available. */ + public Optional<Duration> getUserExecutionTime() { + if (executionStatisticsProto.hasResourceUsage()) { + return Optional.of( + Duration.ofSeconds( + executionStatisticsProto.getResourceUsage().getUtimeSec(), + executionStatisticsProto.getResourceUsage().getUtimeUsec() * 1000)); + } else { + return Optional.empty(); + } + } + + /** Returns the system time for command execution, if available. */ + public Optional<Duration> getSystemExecutionTime() { + if (executionStatisticsProto.hasResourceUsage()) { + return Optional.of( + Duration.ofSeconds( + executionStatisticsProto.getResourceUsage().getStimeSec(), + executionStatisticsProto.getResourceUsage().getStimeUsec() * 1000)); + } else { + return Optional.empty(); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index 53a3c6b060..69bfcc5360 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD @@ -734,6 +734,8 @@ java_test( ]), data = [ ":shell/killmyself", + "//src/main/tools:process-wrapper", + "//src/test/shell/integration:spend_cpu_time", ], flaky = True, tags = [ @@ -746,8 +748,11 @@ java_test( ":test_runner", ":testutil", "//src/main/java/com/google/devtools/build/lib:bazel-main", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib/collect", "//src/main/java/com/google/devtools/build/lib/shell", + "//src/main/protobuf:execution_statistics_java_proto", "//third_party:guava", "//third_party:guava-testlib", "//third_party:jsr305", diff --git a/src/test/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtilTest.java b/src/test/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtilTest.java index 91735e3a22..6d182f29a8 100644 --- a/src/test/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtilTest.java +++ b/src/test/java/com/google/devtools/build/lib/runtime/ProcessWrapperUtilTest.java @@ -17,8 +17,8 @@ package com.google.devtools.build.lib.runtime; import static com.google.common.truth.Truth.assertThat; import static com.google.devtools.build.lib.testutil.MoreAsserts.expectThrows; +import com.google.common.collect.ImmutableList; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; @@ -30,9 +30,7 @@ public final class ProcessWrapperUtilTest { @Test public void testProcessWrapperCommandLineBuilder_ProcessWrapperPathIsRequired() { - List<String> commandArguments = new ArrayList<>(); - commandArguments.add("echo"); - commandArguments.add("hello, world"); + ImmutableList<String> commandArguments = ImmutableList.of("echo", "hello, world"); Exception e = expectThrows( @@ -62,13 +60,10 @@ public final class ProcessWrapperUtilTest { public void testProcessWrapperCommandLineBuilder_BuildsWithoutOptionalArguments() { String processWrapperPath = "process-wrapper"; - List<String> commandArguments = new ArrayList<>(); - commandArguments.add("echo"); - commandArguments.add("hello, world"); + ImmutableList<String> commandArguments = ImmutableList.of("echo", "hello, world"); - List<String> expectedCommandLine = new ArrayList<>(); - expectedCommandLine.add(processWrapperPath); - expectedCommandLine.addAll(commandArguments); + ImmutableList<String> expectedCommandLine = + ImmutableList.<String>builder().add(processWrapperPath).addAll(commandArguments).build(); List<String> commandLine = ProcessWrapperUtil.commandLineBuilder() @@ -83,22 +78,24 @@ public final class ProcessWrapperUtilTest { public void testProcessWrapperCommandLineBuilder_BuildsWithOptionalArguments() { String processWrapperPath = "process-wrapper"; - List<String> commandArguments = new ArrayList<>(); - commandArguments.add("echo"); - commandArguments.add("hello, world"); + ImmutableList<String> commandArguments = ImmutableList.of("echo", "hello, world"); Duration timeout = Duration.ofSeconds(10); Duration killDelay = Duration.ofSeconds(2); String stdoutPath = "stdout.txt"; String stderrPath = "stderr.txt"; - - List<String> expectedCommandLine = new ArrayList<>(); - expectedCommandLine.add(processWrapperPath); - expectedCommandLine.add("--timeout=" + timeout.getSeconds()); - expectedCommandLine.add("--kill_delay=" + killDelay.getSeconds()); - expectedCommandLine.add("--stdout=" + stdoutPath); - expectedCommandLine.add("--stderr=" + stderrPath); - expectedCommandLine.addAll(commandArguments); + String statisticsPath = "stats.out"; + + ImmutableList<String> expectedCommandLine = + ImmutableList.<String>builder() + .add(processWrapperPath) + .add("--timeout=" + timeout.getSeconds()) + .add("--kill_delay=" + killDelay.getSeconds()) + .add("--stdout=" + stdoutPath) + .add("--stderr=" + stderrPath) + .add("--stats=" + statisticsPath) + .addAll(commandArguments) + .build(); List<String> commandLine = ProcessWrapperUtil.commandLineBuilder() @@ -108,6 +105,7 @@ public final class ProcessWrapperUtilTest { .setKillDelay(killDelay) .setStdoutPath(stdoutPath) .setStderrPath(stderrPath) + .setStatisticsPath(statisticsPath) .build(); assertThat(commandLine).containsExactlyElementsIn(expectedCommandLine); diff --git a/src/test/java/com/google/devtools/build/lib/shell/CommandUsingProcessWrapperTest.java b/src/test/java/com/google/devtools/build/lib/shell/CommandUsingProcessWrapperTest.java new file mode 100644 index 0000000000..641afee610 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/shell/CommandUsingProcessWrapperTest.java @@ -0,0 +1,138 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.runtime.ProcessWrapperUtil; +import com.google.devtools.build.lib.testutil.BlazeTestUtils; +import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.testutil.TestUtils; +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link Command}s that are wrapped using the {@code process-wrapper}. */ +@RunWith(JUnit4.class) +public final class CommandUsingProcessWrapperTest { + private String getProcessWrapperPath() { + return BlazeTestUtils.runfilesDir() + "/" + TestConstants.PROCESS_WRAPPER_PATH; + } + + private String getCpuTimeSpenderPath() { + return BlazeTestUtils.runfilesDir() + "/" + TestConstants.CPU_TIME_SPENDER_PATH; + } + + @Test + public void testCommand_Echo() throws Exception { + ImmutableList<String> commandArguments = ImmutableList.of("echo", "worker bees can leave"); + + Command command = new Command(commandArguments.toArray(new String[0])); + CommandResult commandResult = command.execute(); + + assertThat(commandResult.getTerminationStatus().success()).isTrue(); + assertThat(commandResult.getStdoutStream().toString()).contains("worker bees can leave"); + } + + @Test + public void testProcessWrappedCommand_Echo() throws Exception { + ImmutableList<String> commandArguments = ImmutableList.of("echo", "even drones can fly away"); + + List<String> fullProcessWrapperCommandLine = + ProcessWrapperUtil.commandLineBuilder() + .setProcessWrapperPath(getProcessWrapperPath()) + .setCommandArguments(commandArguments) + .build(); + + Command command = new Command(fullProcessWrapperCommandLine.toArray(new String[0])); + CommandResult commandResult = command.execute(); + + assertThat(commandResult.getTerminationStatus().success()).isTrue(); + assertThat(commandResult.getStdoutStream().toString()).contains("even drones can fly away"); + } + + private void checkStatisticsAboutCpuTimeSpent( + Duration userTimeToSpend, Duration systemTimeToSpend) throws CommandException, IOException { + Duration userTimeLowerBound = userTimeToSpend; + Duration userTimeUpperBound = userTimeToSpend.plusSeconds(2); + Duration systemTimeLowerBound = systemTimeToSpend; + Duration systemTimeUpperBound = systemTimeToSpend.plusSeconds(2); + + File outputDir = TestUtils.makeTempDir(); + String statisticsFilePath = outputDir.getAbsolutePath() + "/" + "stats.out"; + + ImmutableList<String> commandArguments = + ImmutableList.of( + getCpuTimeSpenderPath(), + Long.toString(userTimeToSpend.getSeconds()), + Long.toString(systemTimeToSpend.getSeconds())); + + List<String> fullProcessWrapperCommandLine = + ProcessWrapperUtil.commandLineBuilder() + .setProcessWrapperPath(getProcessWrapperPath()) + .setCommandArguments(commandArguments) + .setStatisticsPath(statisticsFilePath) + .build(); + + Command command = new Command(fullProcessWrapperCommandLine.toArray(new String[0])); + CommandResult commandResult = command.execute(); + assertThat(commandResult.getTerminationStatus().success()).isTrue(); + + ExecutionStatistics executionStatistics = new ExecutionStatistics(statisticsFilePath); + + assertThat(executionStatistics.getUserExecutionTime()).isPresent(); + Duration userTime = executionStatistics.getUserExecutionTime().get(); + assertThat(userTime).isAtLeast(userTimeLowerBound); + assertThat(userTime).isAtMost(userTimeUpperBound); + + assertThat(executionStatistics.getSystemExecutionTime()).isPresent(); + Duration systemTime = executionStatistics.getSystemExecutionTime().get(); + assertThat(systemTime).isAtLeast(systemTimeLowerBound); + assertThat(systemTime).isAtMost(systemTimeUpperBound); + } + + @Test + public void testProcessWrappedCommand_WithStatistics_SpendUserTime() + throws CommandException, IOException { + Duration userTimeToSpend = Duration.ofSeconds(10); + Duration systemTimeToSpend = Duration.ZERO; + + checkStatisticsAboutCpuTimeSpent(userTimeToSpend, systemTimeToSpend); + } + + @Test + public void testProcessWrappedCommand_WithStatistics_SpendSystemTime() + throws CommandException, IOException { + Duration userTimeToSpend = Duration.ZERO; + Duration systemTimeToSpend = Duration.ofSeconds(10); + + checkStatisticsAboutCpuTimeSpent(userTimeToSpend, systemTimeToSpend); + } + + @Test + public void testProcessWrappedCommand_WithStatistics_SpendUserAndSystemTime() + throws CommandException, IOException { + Duration userTimeToSpend = Duration.ofSeconds(10); + Duration systemTimeToSpend = Duration.ofSeconds(10); + + checkStatisticsAboutCpuTimeSpent(userTimeToSpend, systemTimeToSpend); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/shell/ExecutionStatisticsTest.java b/src/test/java/com/google/devtools/build/lib/shell/ExecutionStatisticsTest.java new file mode 100644 index 0000000000..571b4ba25c --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/shell/ExecutionStatisticsTest.java @@ -0,0 +1,68 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.shell; + +import static com.google.common.truth.Truth8.assertThat; + +import com.google.devtools.build.lib.shell.Protos.ResourceUsage; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.time.Duration; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link ExecutionStatistics}. */ +@RunWith(JUnit4.class) +public final class ExecutionStatisticsTest { + + private com.google.devtools.build.lib.shell.Protos.ExecutionStatistics + makeExecutionStatisticsProto(Duration userExecutionTime, Duration systemExecutionTime) { + ResourceUsage resourceUsage = + ResourceUsage.newBuilder() + .setUtimeSec(userExecutionTime.getSeconds()) + .setUtimeUsec((long) (userExecutionTime.getNano() / 1000)) + .setStimeSec(systemExecutionTime.getSeconds()) + .setStimeUsec((long) (systemExecutionTime.getNano() / 1000)) + .build(); + + return com.google.devtools.build.lib.shell.Protos.ExecutionStatistics.newBuilder() + .setResourceUsage(resourceUsage) + .build(); + } + + @Test + public void testStatiticsProvided_fromProtoFilename() throws Exception { + Duration riggedUserExecutionTime = Duration.ofSeconds(42).plusNanos(19790000); + Duration riggedSystemExecutionTime = Duration.ofSeconds(33).plusNanos(290000); + + com.google.devtools.build.lib.shell.Protos.ExecutionStatistics executionStatisticsProto = + makeExecutionStatisticsProto(riggedUserExecutionTime, riggedSystemExecutionTime); + + byte[] protoBytes = executionStatisticsProto.toByteArray(); + File encodedProtoFile = File.createTempFile("encoded_action_execution_proto", ""); + String protoFilename = encodedProtoFile.getPath(); + try (BufferedOutputStream bufferedOutputStream = + new BufferedOutputStream(new FileOutputStream(encodedProtoFile))) { + bufferedOutputStream.write(protoBytes); + } + + ExecutionStatistics executionStatistics = new ExecutionStatistics(protoFilename); + + assertThat(executionStatistics.getUserExecutionTime()).hasValue(riggedUserExecutionTime); + assertThat(executionStatistics.getSystemExecutionTime()).hasValue(riggedSystemExecutionTime); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java b/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java index d2e2dcc458..6a3ca2b7ae 100644 --- a/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java +++ b/src/test/java/com/google/devtools/build/lib/testutil/TestConstants.java @@ -59,6 +59,14 @@ public class TestConstants { */ public static final String JAVATESTS_ROOT = "io_bazel/src/test/java/"; + /** Relative path to the process-wrapper tool. */ + public static final String PROCESS_WRAPPER_PATH + = "io_bazel/src/main/tools/process-wrapper"; + + /** Relative path to the spend_cpu_time testing tool. */ + public static final String CPU_TIME_SPENDER_PATH + = "io_bazel/src/test/shell/integration/spend_cpu_time"; + public static final String TEST_RULE_CLASS_PROVIDER = "com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider"; public static final String TEST_RULE_MODULE = diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD index 214d54799d..63ba90f072 100644 --- a/src/test/shell/integration/BUILD +++ b/src/test/shell/integration/BUILD @@ -296,10 +296,20 @@ sh_test( ], ) +package_group( + name = "spend_cpu_time_users", + packages = [ + "//src/test/java/com/google/devtools/build/lib/...", + ], +) + cc_binary( name = "spend_cpu_time", testonly = 1, srcs = ["spend_cpu_time.cc"], + visibility = [ + ":spend_cpu_time_users", + ], ) ######################################################################## |