From c9f87565db0d0b9d12b05d568e67527af06f7d54 Mon Sep 17 00:00:00 2001 From: Kristina Chodorow Date: Tue, 7 Jul 2015 15:44:07 +0000 Subject: Basic dashboard for build results -- MOS_MIGRATED_REVID=97675174 --- src/main/java/BUILD | 36 ++- .../google/devtools/build/lib/bazel/BazelMain.java | 1 + .../com/google/devtools/build/lib/bazel/dash/BUILD | 25 +++ .../devtools/build/lib/bazel/dash/DashModule.java | 249 +++++++++++++++++++++ .../devtools/build/lib/bazel/dash/DashOptions.java | 39 ++++ src/main/protobuf/BUILD | 1 + src/main/protobuf/dash.proto | 59 +++++ 7 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java create mode 100644 src/main/protobuf/dash.proto (limited to 'src/main') diff --git a/src/main/java/BUILD b/src/main/java/BUILD index 75e286a7d7..6c8e788296 100644 --- a/src/main/java/BUILD +++ b/src/main/java/BUILD @@ -322,6 +322,36 @@ java_library( ], ) +java_library( + name = "runtime", + srcs = glob([ + "com/google/devtools/build/lib/runtime/**/*.java", + "com/google/devtools/build/lib/buildtool/**/*.java", + "com/google/devtools/build/lib/server/**/*.java", + ]), + deps = [ + ":actions", + ":analysis-exec-rules-skyframe", + ":cmdline", + ":collect", + ":common", + ":concurrent", + ":docgen", + ":events", + ":options", + ":packages", + ":query2", + ":shell", + ":skyframe-base", + ":unix", + ":vfs", + "//src/main/protobuf:proto_build", + "//src/main/protobuf:proto_test_status", + "//third_party:guava", + "//third_party:jsr305", + ], +) + java_library( name = "server", srcs = glob([ @@ -358,6 +388,8 @@ java_library( srcs = glob( [ "com/google/devtools/build/lib/bazel/**/*.java", + "com/google/devtools/build/lib/standalone/*.java", + "com/google/devtools/build/lib/worker/**", ], exclude = [ "com/google/devtools/build/lib/bazel/repository/MavenConnector.java", @@ -401,12 +433,14 @@ java_library( ":options", ":packages", ":query2", + ":runtime", ":shell", ":skyframe-base", ":unix", ":vfs", ":webstatusserver", "//src/java_tools/singlejar:zip", + "//src/main/java/com/google/devtools/build/lib/bazel/dash", "//src/main/java/com/google/devtools/build/lib/sandbox", "//src/main/java/com/google/devtools/build/lib/standalone", "//src/main/java/com/google/devtools/build/lib/worker", @@ -417,7 +451,6 @@ java_library( "//src/main/protobuf:proto_worker_protocol", "//third_party:aether", "//third_party:apache_commons_pool2", - "//third_party:apache_velocity", "//third_party:auto_value", "//third_party:guava", "//third_party:joda_time", @@ -471,6 +504,7 @@ java_binary( filegroup( name = "srcs", srcs = glob(["**"]) + [ + "//src/main/java/com/google/devtools/build/lib/bazel/dash:srcs", "//src/main/java/com/google/devtools/build/lib/sandbox:srcs", "//src/main/java/com/google/devtools/build/lib/standalone:srcs", "//src/main/java/com/google/devtools/build/lib/worker:srcs", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java index 958c7ea3c3..f8e0c7d5ed 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java @@ -35,6 +35,7 @@ public final class BazelMain { com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class, com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class, com.google.devtools.build.lib.bazel.BazelRepositoryModule.class, + com.google.devtools.build.lib.bazel.dash.DashModule.class, com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class, com.google.devtools.build.lib.sandbox.SandboxModule.class, com.google.devtools.build.lib.standalone.StandaloneModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD new file mode 100644 index 0000000000..60b1da6aa1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD @@ -0,0 +1,25 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/main/java:__pkg__"], +) + +java_library( + name = "dash", + srcs = glob(["*.java"]), + visibility = [ + "//src/main/java:__pkg__", + ], + deps = [ + "//src/main/java:analysis-exec-rules-skyframe", + "//src/main/java:options", + "//src/main/java:packages", + "//src/main/java:runtime", + "//src/main/java:vfs", + "//src/main/protobuf:proto_dash", + "//third_party:apache_httpclient", + "//third_party:apache_httpcore", + "//third_party:guava", + "//third_party:protobuf", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java new file mode 100644 index 0000000000..c51c397d3d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java @@ -0,0 +1,249 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.dash; + +import com.google.common.collect.ImmutableList; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.CommandLine.Option; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.EnvironmentVar; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.TestData; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.CommandStartEvent; +import com.google.devtools.build.lib.runtime.GotOptionsEvent; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser.OptionValueDescription; +import com.google.devtools.common.options.OptionsProvider; +import com.google.protobuf.ByteString; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.DefaultHttpClient; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +/** + * Dashboard for a build. + */ +public class DashModule extends BlazeModule { + private static final int ONE_MB = 1024 * 1024; + + private Sendable sender; + private BlazeRuntime runtime; + private final ExecutorService executorService; + + public DashModule() { + // Make sure sender != null before we hop on the event bus. + sender = new NoOpSender(); + executorService = Executors.newFixedThreadPool(5, + new ThreadFactory() { + @Override + public Thread newThread(Runnable runnable) { + Thread thread = Executors.defaultThreadFactory().newThread(runnable); + thread.setDaemon(true); + return thread; + } + }); + } + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.runtime = runtime; + runtime.getEventBus().register(this); + } + + @Override + public Iterable> getCommandOptions(Command command) { + return (command.name().equals("build") || command.name().equals("test")) + ? ImmutableList.>of(DashOptions.class) + : ImmutableList.>of(); + } + + @Override + public void handleOptions(OptionsProvider optionsProvider) { + DashOptions options = optionsProvider.getOptions(DashOptions.class); + sender = (options == null || !options.useDash) + ? new NoOpSender() + : new Sender(options.url, runtime, executorService); + } + + @Subscribe + public void gotOptions(GotOptionsEvent event) { + BuildData.Builder builder = BuildData.newBuilder(); + BuildData.CommandLine.Builder cmdLineBuilder = BuildData.CommandLine.newBuilder(); + for (OptionValueDescription option : event.getStartupOptions().asListOfEffectiveOptions()) { + cmdLineBuilder.addStartupOptions(getOption(option)); + } + + for (OptionValueDescription option : event.getOptions().asListOfEffectiveOptions()) { + if (option.getName().equals("client_env")) { + String env[] = option.getValue().toString().split("="); + if (env.length == 1) { + builder.addClientEnv( + EnvironmentVar.newBuilder().setName(env[0]).setValue("true").build()); + } else if (env.length == 2) { + builder.addClientEnv( + EnvironmentVar.newBuilder().setName(env[0]).setValue(env[1]).build()); + } + } else { + cmdLineBuilder.addOptions(getOption(option)); + } + } + + for (String residue : event.getOptions().getResidue()) { + cmdLineBuilder.addResidue(residue); + } + builder.setCommandLine(cmdLineBuilder.build()); + sender.send("options", builder.build()); + } + + @Subscribe + public void commandStartEvent(CommandStartEvent event) { + BuildData.Builder builder = BuildData.newBuilder() + .setBuildId(event.getCommandId().toString()) + .setCommandName(event.getCommandName()) + .setWorkingDir(event.getWorkingDirectory().getPathString()); + sender.send("start", builder.build()); + } + + @Subscribe + public void parsingComplete(TargetParsingCompleteEvent event) { + BuildData.Builder builder = BuildData.newBuilder(); + for (Target target : event.getTargets()) { + builder.addTargetsBuilder() + .setLabel(target.getLabel().toString()) + .setRuleKind(target.getTargetKind()).build(); + } + sender.send("targets", builder.build()); + } + + @Subscribe + public void testFinished(TestResult result) { + BuildData.Builder builder = BuildData.newBuilder(); + TestData.Builder testDataBuilder = TestData.newBuilder(); + testDataBuilder.setLabel(result.getLabel()); + testDataBuilder.setPassed(result.getData().getTestPassed()); + if (!result.getData().getTestPassed()) { + Path logPath = result.getTestLogPath(); + try { + long fileSize = logPath.getFileSize(); + if (fileSize > ONE_MB) { + fileSize = ONE_MB; + testDataBuilder.setTruncated(true); + } + ByteString str = ByteString.copyFrom( + FileSystemUtils.readContent(logPath), 0, (int) fileSize); + testDataBuilder.setLog(str); + } catch (IOException e) { + runtime.getReporter().getOutErr().printOutLn( + "Error reading log file " + logPath + ": " + e.getMessage()); + // TODO(kchodorow): add this info to the proto and send. + return; + } + } + sender.send("test", builder.build()); + } + + @Override + public void blazeShutdown() { + executorService.shutdownNow(); + } + + private BuildData.CommandLine.Option getOption(OptionValueDescription option) { + Option.Builder optionBuilder = Option.newBuilder(); + optionBuilder.setName(option.getName()); + if (option.getSource() != null) { + optionBuilder.setSource(option.getSource()); + } + Object value = option.getValue(); + if (value != null) { + if (value instanceof Iterable) { + for (Object v : ((Iterable) value)) { + if (v != null) { + optionBuilder.addValue(v.toString()); + } + } + } else { + optionBuilder.addValue(value.toString()); + } + } + return optionBuilder.build(); + } + + private interface Sendable { + void send(final String suffix, final BuildData message); + } + + private static class Sender implements Sendable { + private final String url; + private final String buildId; + private final OutErr outErr; + private final ExecutorService executorService; + + public Sender(String url, BlazeRuntime runtime, ExecutorService executorService) { + this.url = url; + this.buildId = runtime.getCommandId().toString(); + this.outErr = runtime.getReporter().getOutErr(); + this.executorService = executorService; + outErr.printOutLn("Results are being streamed to " + url + "/result/" + buildId); + } + + @Override + public void send(final String suffix, final BuildData message) { + executorService.submit(new Runnable() { + @Override + public void run() { + HttpClient httpClient = new DefaultHttpClient(); + HttpPost httppost = new HttpPost(url + "/" + suffix + "/" + buildId); + httppost.setHeader(HttpHeaders.CONTENT_TYPE, "application/x-protobuf"); + httppost.setEntity(new ByteArrayEntity(message.toByteArray())); + + try { + httpClient.execute(httppost); + } catch (IOException | IllegalStateException e) { + // IllegalStateException is thrown if the URL was invalid (e.g., someone passed + // --dash_url=localhost:8080 instead of --dash_url=http://localhost:8080). + outErr.printErrLn("Error sending results to " + url + ": " + e.getMessage()); + } catch (Exception e) { + outErr.printErrLn("Unknown error sending results to " + url + ": " + e.getMessage()); + } + } + }); + } + } + + private static class NoOpSender implements Sendable { + public NoOpSender() { + } + + @Override + public void send(String suffix, BuildData message) { + } + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java new file mode 100644 index 0000000000..f01b0a3484 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java @@ -0,0 +1,39 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.dash; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +/** + * Options for sending build results to a dashboard. + */ +public class DashOptions extends OptionsBase { + + @Option( + name = "use_dash", + defaultValue = "false", + help = "If build/test results should be sent to a remote dashboard." + ) + public boolean useDash; + + @Option( + name = "dash_url", + defaultValue = "", + help = "The URL of the dashboard server." + ) + public String url; + +} diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD index f896347cfb..719bd7b6a0 100644 --- a/src/main/protobuf/BUILD +++ b/src/main/protobuf/BUILD @@ -4,6 +4,7 @@ load("/tools/build_rules/genproto", "proto_java_library") FILES = [ "build", + "dash", "deps", "java_compilation", "crosstool_config", diff --git a/src/main/protobuf/dash.proto b/src/main/protobuf/dash.proto new file mode 100644 index 0000000000..c4c394485f --- /dev/null +++ b/src/main/protobuf/dash.proto @@ -0,0 +1,59 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package dash; + +option java_package = "com.google.devtools.build.lib.bazel.dash"; +option java_outer_classname = "DashProtos"; + +message BuildData { + optional string build_id = 1; + optional string command_name = 2; + optional string working_dir = 3; + + message CommandLine { + message Option { + optional string name = 1; + repeated string value = 2; + optional string source = 3; + } + repeated Option startup_options = 1; + repeated Option options = 2; + repeated string residue = 3; + } + optional CommandLine command_line = 4; + + message EnvironmentVar { + optional string name = 1; + optional string value = 2; + } + repeated EnvironmentVar client_env = 5; + + message Target { + optional string label = 1; + optional string rule_kind = 2; + } + repeated Target targets = 6; + + message TestData { + optional string label = 2; + optional bool passed = 3; + optional bytes log = 4; + // Log is truncated after 1MB. + optional bool truncated = 5; + } + repeated TestData test_data = 7; +} -- cgit v1.2.3