Basic dashboard for build results

--
MOS_MIGRATED_REVID=97675174

7 files changed, 409 insertions, 1 deletions

diff --git a/src/main/java/BUILD b/src/main/java/BUILD
index 75e286a7d7..6c8e788296 100644
--- a/src/main/java/BUILD
+++ b/src/main/java/BUILD
@@ -323,6 +323,36 @@ java_library(
 )
 
 java_library(
+    name = "runtime",
+    srcs = glob([
+        "com/google/devtools/build/lib/runtime/**/*.java",
+        "com/google/devtools/build/lib/buildtool/**/*.java",
+        "com/google/devtools/build/lib/server/**/*.java",
+    ]),
+    deps = [
+        ":actions",
+        ":analysis-exec-rules-skyframe",
+        ":cmdline",
+        ":collect",
+        ":common",
+        ":concurrent",
+        ":docgen",
+        ":events",
+        ":options",
+        ":packages",
+        ":query2",
+        ":shell",
+        ":skyframe-base",
+        ":unix",
+        ":vfs",
+        "//src/main/protobuf:proto_build",
+        "//src/main/protobuf:proto_test_status",
+        "//third_party:guava",
+        "//third_party:jsr305",
+    ],
+)
+
+java_library(
     name = "server",
     srcs = glob([
         "com/google/devtools/build/lib/server/**/*.java",
@@ -358,6 +388,8 @@ java_library(
     srcs = glob(
         [
             "com/google/devtools/build/lib/bazel/**/*.java",
+            "com/google/devtools/build/lib/standalone/*.java",
+            "com/google/devtools/build/lib/worker/**",
         ],
         exclude = [
             "com/google/devtools/build/lib/bazel/repository/MavenConnector.java",
@@ -401,12 +433,14 @@ java_library(
         ":options",
         ":packages",
         ":query2",
+        ":runtime",
         ":shell",
         ":skyframe-base",
         ":unix",
         ":vfs",
         ":webstatusserver",
         "//src/java_tools/singlejar:zip",
+        "//src/main/java/com/google/devtools/build/lib/bazel/dash",
         "//src/main/java/com/google/devtools/build/lib/sandbox",
         "//src/main/java/com/google/devtools/build/lib/standalone",
         "//src/main/java/com/google/devtools/build/lib/worker",
@@ -417,7 +451,6 @@ java_library(
         "//src/main/protobuf:proto_worker_protocol",
         "//third_party:aether",
         "//third_party:apache_commons_pool2",
-        "//third_party:apache_velocity",
         "//third_party:auto_value",
         "//third_party:guava",
         "//third_party:joda_time",
@@ -471,6 +504,7 @@ java_binary(
 filegroup(
     name = "srcs",
     srcs = glob(["**"]) + [
+        "//src/main/java/com/google/devtools/build/lib/bazel/dash:srcs",
         "//src/main/java/com/google/devtools/build/lib/sandbox:srcs",
         "//src/main/java/com/google/devtools/build/lib/standalone:srcs",
         "//src/main/java/com/google/devtools/build/lib/worker:srcs",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java
index 958c7ea3c3..f8e0c7d5ed 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java
@@ -35,6 +35,7 @@ public final class BazelMain {
       com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class,
       com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class,
       com.google.devtools.build.lib.bazel.BazelRepositoryModule.class,
+      com.google.devtools.build.lib.bazel.dash.DashModule.class,
       com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class,
       com.google.devtools.build.lib.sandbox.SandboxModule.class,
       com.google.devtools.build.lib.standalone.StandaloneModule.class,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD
new file mode 100644
index 0000000000..60b1da6aa1
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD
@@ -0,0 +1,25 @@
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//src/main/java:__pkg__"],
+)
+
+java_library(
+    name = "dash",
+    srcs = glob(["*.java"]),
+    visibility = [
+        "//src/main/java:__pkg__",
+    ],
+    deps = [
+        "//src/main/java:analysis-exec-rules-skyframe",
+        "//src/main/java:options",
+        "//src/main/java:packages",
+        "//src/main/java:runtime",
+        "//src/main/java:vfs",
+        "//src/main/protobuf:proto_dash",
+        "//third_party:apache_httpclient",
+        "//third_party:apache_httpcore",
+        "//third_party:guava",
+        "//third_party:protobuf",
+    ],
+)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java
new file mode 100644
index 0000000000..c51c397d3d
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java
@@ -0,0 +1,249 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.dash;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.eventbus.Subscribe;
+import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData;
+import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.CommandLine.Option;
+import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.EnvironmentVar;
+import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.TestData;
+import com.google.devtools.build.lib.packages.Target;
+import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent;
+import com.google.devtools.build.lib.rules.test.TestResult;
+import com.google.devtools.build.lib.runtime.BlazeModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.runtime.Command;
+import com.google.devtools.build.lib.runtime.CommandStartEvent;
+import com.google.devtools.build.lib.runtime.GotOptionsEvent;
+import com.google.devtools.build.lib.util.io.OutErr;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsParser.OptionValueDescription;
+import com.google.devtools.common.options.OptionsProvider;
+import com.google.protobuf.ByteString;
+
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+
+/**
+ * Dashboard for a build.
+ */
+public class DashModule extends BlazeModule {
+  private static final int ONE_MB = 1024 * 1024;
+
+  private Sendable sender;
+  private BlazeRuntime runtime;
+  private final ExecutorService executorService;
+
+  public DashModule() {
+    // Make sure sender != null before we hop on the event bus.
+    sender = new NoOpSender();
+    executorService = Executors.newFixedThreadPool(5,
+        new ThreadFactory() {
+          @Override
+          public Thread newThread(Runnable runnable) {
+            Thread thread = Executors.defaultThreadFactory().newThread(runnable);
+            thread.setDaemon(true);
+            return thread;
+          }
+        });
+  }
+
+  @Override
+  public void beforeCommand(BlazeRuntime runtime, Command command) {
+    this.runtime = runtime;
+    runtime.getEventBus().register(this);
+  }
+
+  @Override
+  public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
+    return (command.name().equals("build") || command.name().equals("test"))
+        ? ImmutableList.<Class<? extends OptionsBase>>of(DashOptions.class)
+        : ImmutableList.<Class<? extends OptionsBase>>of();
+  }
+
+  @Override
+  public void handleOptions(OptionsProvider optionsProvider) {
+    DashOptions options = optionsProvider.getOptions(DashOptions.class);
+    sender = (options == null || !options.useDash)
+        ? new NoOpSender()
+        : new Sender(options.url, runtime, executorService);
+  }
+
+  @Subscribe
+  public void gotOptions(GotOptionsEvent event) {
+    BuildData.Builder builder = BuildData.newBuilder();
+    BuildData.CommandLine.Builder cmdLineBuilder = BuildData.CommandLine.newBuilder();
+    for (OptionValueDescription option : event.getStartupOptions().asListOfEffectiveOptions()) {
+      cmdLineBuilder.addStartupOptions(getOption(option));
+    }
+
+    for (OptionValueDescription option : event.getOptions().asListOfEffectiveOptions()) {
+      if (option.getName().equals("client_env")) {
+        String env[] = option.getValue().toString().split("=");
+        if (env.length == 1) {
+          builder.addClientEnv(
+              EnvironmentVar.newBuilder().setName(env[0]).setValue("true").build());
+        } else if (env.length == 2) {
+          builder.addClientEnv(
+              EnvironmentVar.newBuilder().setName(env[0]).setValue(env[1]).build());
+        }
+      } else {
+        cmdLineBuilder.addOptions(getOption(option));
+      }
+    }
+
+    for (String residue : event.getOptions().getResidue()) {
+      cmdLineBuilder.addResidue(residue);
+    }
+    builder.setCommandLine(cmdLineBuilder.build());
+    sender.send("options", builder.build());
+  }
+
+  @Subscribe
+  public void commandStartEvent(CommandStartEvent event) {
+    BuildData.Builder builder = BuildData.newBuilder()
+        .setBuildId(event.getCommandId().toString())
+        .setCommandName(event.getCommandName())
+        .setWorkingDir(event.getWorkingDirectory().getPathString());
+    sender.send("start", builder.build());
+  }
+
+  @Subscribe
+  public void parsingComplete(TargetParsingCompleteEvent event) {
+    BuildData.Builder builder = BuildData.newBuilder();
+    for (Target target : event.getTargets()) {
+      builder.addTargetsBuilder()
+          .setLabel(target.getLabel().toString())
+          .setRuleKind(target.getTargetKind()).build();
+    }
+    sender.send("targets", builder.build());
+  }
+
+  @Subscribe
+  public void testFinished(TestResult result) {
+    BuildData.Builder builder = BuildData.newBuilder();
+    TestData.Builder testDataBuilder = TestData.newBuilder();
+    testDataBuilder.setLabel(result.getLabel());
+    testDataBuilder.setPassed(result.getData().getTestPassed());
+    if (!result.getData().getTestPassed()) {
+      Path logPath = result.getTestLogPath();
+      try {
+        long fileSize = logPath.getFileSize();
+        if (fileSize > ONE_MB) {
+          fileSize = ONE_MB;
+          testDataBuilder.setTruncated(true);
+        }
+        ByteString str = ByteString.copyFrom(
+            FileSystemUtils.readContent(logPath), 0, (int) fileSize);
+        testDataBuilder.setLog(str);
+      } catch (IOException e) {
+        runtime.getReporter().getOutErr().printOutLn(
+            "Error reading log file " + logPath + ": " + e.getMessage());
+        // TODO(kchodorow): add this info to the proto and send.
+        return;
+      }
+    }
+    sender.send("test", builder.build());
+  }
+
+  @Override
+  public void blazeShutdown() {
+    executorService.shutdownNow();
+  }
+
+  private BuildData.CommandLine.Option getOption(OptionValueDescription option) {
+    Option.Builder optionBuilder = Option.newBuilder();
+    optionBuilder.setName(option.getName());
+    if (option.getSource() != null) {
+      optionBuilder.setSource(option.getSource());
+    }
+    Object value = option.getValue();
+    if (value != null) {
+      if (value instanceof Iterable<?>) {
+        for (Object v : ((Iterable<?>) value)) {
+          if (v != null) {
+            optionBuilder.addValue(v.toString());
+          }
+        }
+      } else {
+        optionBuilder.addValue(value.toString());
+      }
+    }
+    return optionBuilder.build();
+  }
+
+  private interface Sendable {
+    void send(final String suffix, final BuildData message);
+  }
+
+  private static class Sender implements Sendable {
+    private final String url;
+    private final String buildId;
+    private final OutErr outErr;
+    private final ExecutorService executorService;
+
+    public Sender(String url, BlazeRuntime runtime, ExecutorService executorService) {
+      this.url = url;
+      this.buildId = runtime.getCommandId().toString();
+      this.outErr = runtime.getReporter().getOutErr();
+      this.executorService = executorService;
+      outErr.printOutLn("Results are being streamed to " + url + "/result/" + buildId);
+    }
+
+    @Override
+    public void send(final String suffix, final BuildData message) {
+      executorService.submit(new Runnable() {
+        @Override
+        public void run() {
+          HttpClient httpClient = new DefaultHttpClient();
+          HttpPost httppost = new HttpPost(url + "/" + suffix + "/" + buildId);
+          httppost.setHeader(HttpHeaders.CONTENT_TYPE, "application/x-protobuf");
+          httppost.setEntity(new ByteArrayEntity(message.toByteArray()));
+
+          try {
+            httpClient.execute(httppost);
+          } catch (IOException | IllegalStateException e) {
+            // IllegalStateException is thrown if the URL was invalid (e.g., someone passed
+            // --dash_url=localhost:8080 instead of --dash_url=http://localhost:8080).
+            outErr.printErrLn("Error sending results to " + url + ": " + e.getMessage());
+          } catch (Exception e) {
+            outErr.printErrLn("Unknown error sending results to " + url + ": " + e.getMessage());
+          }
+        }
+      });
+    }
+  }
+
+  private static class NoOpSender implements Sendable {
+    public NoOpSender() {
+    }
+
+    @Override
+    public void send(String suffix, BuildData message) {
+    }
+  }
+
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java
new file mode 100644
index 0000000000..f01b0a3484
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java
@@ -0,0 +1,39 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.dash;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+/**
+ * Options for sending build results to a dashboard.
+ */
+public class DashOptions extends OptionsBase {
+
+  @Option(
+      name = "use_dash",
+      defaultValue = "false",
+      help = "If build/test results should be sent to a remote dashboard."
+  )
+  public boolean useDash;
+
+  @Option(
+      name = "dash_url",
+      defaultValue = "",
+      help = "The URL of the dashboard server."
+  )
+  public String url;
+
+}
diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD
index f896347cfb..719bd7b6a0 100644
--- a/src/main/protobuf/BUILD
+++ b/src/main/protobuf/BUILD
@@ -4,6 +4,7 @@ load("/tools/build_rules/genproto", "proto_java_library")
 
 FILES = [
     "build",
+    "dash",
     "deps",
     "java_compilation",
     "crosstool_config",
diff --git a/src/main/protobuf/dash.proto b/src/main/protobuf/dash.proto
new file mode 100644
index 0000000000..c4c394485f
--- /dev/null
+++ b/src/main/protobuf/dash.proto
@@ -0,0 +1,59 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package dash;
+
+option java_package = "com.google.devtools.build.lib.bazel.dash";
+option java_outer_classname = "DashProtos";
+
+message BuildData {
+  optional string build_id = 1;
+  optional string command_name = 2;
+  optional string working_dir = 3;
+
+  message CommandLine {
+    message Option {
+      optional string name = 1;
+      repeated string value = 2;
+      optional string source = 3;
+    }
+    repeated Option startup_options = 1;
+    repeated Option options = 2;
+    repeated string residue = 3;
+  }
+  optional CommandLine command_line = 4;
+
+  message EnvironmentVar {
+    optional string name = 1;
+    optional string value = 2;
+  }
+  repeated EnvironmentVar client_env = 5;
+
+  message Target {
+      optional string label = 1;
+      optional string rule_kind = 2;
+  }
+  repeated Target targets = 6;
+
+  message TestData {
+    optional string label = 2;
+    optional bool passed = 3;
+    optional bytes log = 4;
+    // Log is truncated after 1MB.
+    optional bool truncated = 5;
+  }
+  repeated TestData test_data = 7;
+}