diff options
23 files changed, 1142 insertions, 16 deletions
@@ -0,0 +1,38 @@ +# For src/tools/dash support. + +new_http_archive( + name = "appengine-java", + url = "http://central.maven.org/maven2/com/google/appengine/appengine-java-sdk/1.9.23/appengine-java-sdk-1.9.23.zip", + sha256 = "05e667036e9ef4f999b829fc08f8e5395b33a5a3c30afa9919213088db2b2e89", + build_file = "tools/build_rules/appengine/appengine.BUILD", +) + +bind( + name = "appengine/java/sdk", + actual = "@appengine-java//:sdk", +) + +bind( + name = "appengine/java/api", + actual = "@appengine-java//:api", +) + +bind( + name = "appengine/java/jars", + actual = "@appengine-java//:jars", +) + +maven_jar( + name = "javax-servlet-api", + artifact = "javax.servlet:servlet-api:2.5", +) + +maven_jar( + name = "commons-lang", + artifact = "commons-lang:commons-lang:2.6", +) + +bind( + name = "javax/servlet/api", + actual = "//tools/build_rules/appengine:javax.servlet.api", +) diff --git a/src/main/java/BUILD b/src/main/java/BUILD index 75e286a7d7..6c8e788296 100644 --- a/src/main/java/BUILD +++ b/src/main/java/BUILD @@ -323,6 +323,36 @@ java_library( ) java_library( + name = "runtime", + srcs = glob([ + "com/google/devtools/build/lib/runtime/**/*.java", + "com/google/devtools/build/lib/buildtool/**/*.java", + "com/google/devtools/build/lib/server/**/*.java", + ]), + deps = [ + ":actions", + ":analysis-exec-rules-skyframe", + ":cmdline", + ":collect", + ":common", + ":concurrent", + ":docgen", + ":events", + ":options", + ":packages", + ":query2", + ":shell", + ":skyframe-base", + ":unix", + ":vfs", + "//src/main/protobuf:proto_build", + "//src/main/protobuf:proto_test_status", + "//third_party:guava", + "//third_party:jsr305", + ], +) + +java_library( name = "server", srcs = glob([ "com/google/devtools/build/lib/server/**/*.java", @@ -358,6 +388,8 @@ java_library( srcs = glob( [ "com/google/devtools/build/lib/bazel/**/*.java", + "com/google/devtools/build/lib/standalone/*.java", + "com/google/devtools/build/lib/worker/**", ], exclude = [ "com/google/devtools/build/lib/bazel/repository/MavenConnector.java", @@ -401,12 +433,14 @@ java_library( ":options", ":packages", ":query2", + ":runtime", ":shell", ":skyframe-base", ":unix", ":vfs", ":webstatusserver", "//src/java_tools/singlejar:zip", + "//src/main/java/com/google/devtools/build/lib/bazel/dash", "//src/main/java/com/google/devtools/build/lib/sandbox", "//src/main/java/com/google/devtools/build/lib/standalone", "//src/main/java/com/google/devtools/build/lib/worker", @@ -417,7 +451,6 @@ java_library( "//src/main/protobuf:proto_worker_protocol", "//third_party:aether", "//third_party:apache_commons_pool2", - "//third_party:apache_velocity", "//third_party:auto_value", "//third_party:guava", "//third_party:joda_time", @@ -471,6 +504,7 @@ java_binary( filegroup( name = "srcs", srcs = glob(["**"]) + [ + "//src/main/java/com/google/devtools/build/lib/bazel/dash:srcs", "//src/main/java/com/google/devtools/build/lib/sandbox:srcs", "//src/main/java/com/google/devtools/build/lib/standalone:srcs", "//src/main/java/com/google/devtools/build/lib/worker:srcs", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java index 958c7ea3c3..f8e0c7d5ed 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelMain.java @@ -35,6 +35,7 @@ public final class BazelMain { com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class, com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class, com.google.devtools.build.lib.bazel.BazelRepositoryModule.class, + com.google.devtools.build.lib.bazel.dash.DashModule.class, com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class, com.google.devtools.build.lib.sandbox.SandboxModule.class, com.google.devtools.build.lib.standalone.StandaloneModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD new file mode 100644 index 0000000000..60b1da6aa1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/BUILD @@ -0,0 +1,25 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/main/java:__pkg__"], +) + +java_library( + name = "dash", + srcs = glob(["*.java"]), + visibility = [ + "//src/main/java:__pkg__", + ], + deps = [ + "//src/main/java:analysis-exec-rules-skyframe", + "//src/main/java:options", + "//src/main/java:packages", + "//src/main/java:runtime", + "//src/main/java:vfs", + "//src/main/protobuf:proto_dash", + "//third_party:apache_httpclient", + "//third_party:apache_httpcore", + "//third_party:guava", + "//third_party:protobuf", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java new file mode 100644 index 0000000000..c51c397d3d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashModule.java @@ -0,0 +1,249 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.dash; + +import com.google.common.collect.ImmutableList; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.CommandLine.Option; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.EnvironmentVar; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData.TestData; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.CommandStartEvent; +import com.google.devtools.build.lib.runtime.GotOptionsEvent; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser.OptionValueDescription; +import com.google.devtools.common.options.OptionsProvider; +import com.google.protobuf.ByteString; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.DefaultHttpClient; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; + +/** + * Dashboard for a build. + */ +public class DashModule extends BlazeModule { + private static final int ONE_MB = 1024 * 1024; + + private Sendable sender; + private BlazeRuntime runtime; + private final ExecutorService executorService; + + public DashModule() { + // Make sure sender != null before we hop on the event bus. + sender = new NoOpSender(); + executorService = Executors.newFixedThreadPool(5, + new ThreadFactory() { + @Override + public Thread newThread(Runnable runnable) { + Thread thread = Executors.defaultThreadFactory().newThread(runnable); + thread.setDaemon(true); + return thread; + } + }); + } + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.runtime = runtime; + runtime.getEventBus().register(this); + } + + @Override + public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) { + return (command.name().equals("build") || command.name().equals("test")) + ? ImmutableList.<Class<? extends OptionsBase>>of(DashOptions.class) + : ImmutableList.<Class<? extends OptionsBase>>of(); + } + + @Override + public void handleOptions(OptionsProvider optionsProvider) { + DashOptions options = optionsProvider.getOptions(DashOptions.class); + sender = (options == null || !options.useDash) + ? new NoOpSender() + : new Sender(options.url, runtime, executorService); + } + + @Subscribe + public void gotOptions(GotOptionsEvent event) { + BuildData.Builder builder = BuildData.newBuilder(); + BuildData.CommandLine.Builder cmdLineBuilder = BuildData.CommandLine.newBuilder(); + for (OptionValueDescription option : event.getStartupOptions().asListOfEffectiveOptions()) { + cmdLineBuilder.addStartupOptions(getOption(option)); + } + + for (OptionValueDescription option : event.getOptions().asListOfEffectiveOptions()) { + if (option.getName().equals("client_env")) { + String env[] = option.getValue().toString().split("="); + if (env.length == 1) { + builder.addClientEnv( + EnvironmentVar.newBuilder().setName(env[0]).setValue("true").build()); + } else if (env.length == 2) { + builder.addClientEnv( + EnvironmentVar.newBuilder().setName(env[0]).setValue(env[1]).build()); + } + } else { + cmdLineBuilder.addOptions(getOption(option)); + } + } + + for (String residue : event.getOptions().getResidue()) { + cmdLineBuilder.addResidue(residue); + } + builder.setCommandLine(cmdLineBuilder.build()); + sender.send("options", builder.build()); + } + + @Subscribe + public void commandStartEvent(CommandStartEvent event) { + BuildData.Builder builder = BuildData.newBuilder() + .setBuildId(event.getCommandId().toString()) + .setCommandName(event.getCommandName()) + .setWorkingDir(event.getWorkingDirectory().getPathString()); + sender.send("start", builder.build()); + } + + @Subscribe + public void parsingComplete(TargetParsingCompleteEvent event) { + BuildData.Builder builder = BuildData.newBuilder(); + for (Target target : event.getTargets()) { + builder.addTargetsBuilder() + .setLabel(target.getLabel().toString()) + .setRuleKind(target.getTargetKind()).build(); + } + sender.send("targets", builder.build()); + } + + @Subscribe + public void testFinished(TestResult result) { + BuildData.Builder builder = BuildData.newBuilder(); + TestData.Builder testDataBuilder = TestData.newBuilder(); + testDataBuilder.setLabel(result.getLabel()); + testDataBuilder.setPassed(result.getData().getTestPassed()); + if (!result.getData().getTestPassed()) { + Path logPath = result.getTestLogPath(); + try { + long fileSize = logPath.getFileSize(); + if (fileSize > ONE_MB) { + fileSize = ONE_MB; + testDataBuilder.setTruncated(true); + } + ByteString str = ByteString.copyFrom( + FileSystemUtils.readContent(logPath), 0, (int) fileSize); + testDataBuilder.setLog(str); + } catch (IOException e) { + runtime.getReporter().getOutErr().printOutLn( + "Error reading log file " + logPath + ": " + e.getMessage()); + // TODO(kchodorow): add this info to the proto and send. + return; + } + } + sender.send("test", builder.build()); + } + + @Override + public void blazeShutdown() { + executorService.shutdownNow(); + } + + private BuildData.CommandLine.Option getOption(OptionValueDescription option) { + Option.Builder optionBuilder = Option.newBuilder(); + optionBuilder.setName(option.getName()); + if (option.getSource() != null) { + optionBuilder.setSource(option.getSource()); + } + Object value = option.getValue(); + if (value != null) { + if (value instanceof Iterable<?>) { + for (Object v : ((Iterable<?>) value)) { + if (v != null) { + optionBuilder.addValue(v.toString()); + } + } + } else { + optionBuilder.addValue(value.toString()); + } + } + return optionBuilder.build(); + } + + private interface Sendable { + void send(final String suffix, final BuildData message); + } + + private static class Sender implements Sendable { + private final String url; + private final String buildId; + private final OutErr outErr; + private final ExecutorService executorService; + + public Sender(String url, BlazeRuntime runtime, ExecutorService executorService) { + this.url = url; + this.buildId = runtime.getCommandId().toString(); + this.outErr = runtime.getReporter().getOutErr(); + this.executorService = executorService; + outErr.printOutLn("Results are being streamed to " + url + "/result/" + buildId); + } + + @Override + public void send(final String suffix, final BuildData message) { + executorService.submit(new Runnable() { + @Override + public void run() { + HttpClient httpClient = new DefaultHttpClient(); + HttpPost httppost = new HttpPost(url + "/" + suffix + "/" + buildId); + httppost.setHeader(HttpHeaders.CONTENT_TYPE, "application/x-protobuf"); + httppost.setEntity(new ByteArrayEntity(message.toByteArray())); + + try { + httpClient.execute(httppost); + } catch (IOException | IllegalStateException e) { + // IllegalStateException is thrown if the URL was invalid (e.g., someone passed + // --dash_url=localhost:8080 instead of --dash_url=http://localhost:8080). + outErr.printErrLn("Error sending results to " + url + ": " + e.getMessage()); + } catch (Exception e) { + outErr.printErrLn("Unknown error sending results to " + url + ": " + e.getMessage()); + } + } + }); + } + } + + private static class NoOpSender implements Sendable { + public NoOpSender() { + } + + @Override + public void send(String suffix, BuildData message) { + } + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java new file mode 100644 index 0000000000..f01b0a3484 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/dash/DashOptions.java @@ -0,0 +1,39 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.dash; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +/** + * Options for sending build results to a dashboard. + */ +public class DashOptions extends OptionsBase { + + @Option( + name = "use_dash", + defaultValue = "false", + help = "If build/test results should be sent to a remote dashboard." + ) + public boolean useDash; + + @Option( + name = "dash_url", + defaultValue = "", + help = "The URL of the dashboard server." + ) + public String url; + +} diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD index f896347cfb..719bd7b6a0 100644 --- a/src/main/protobuf/BUILD +++ b/src/main/protobuf/BUILD @@ -4,6 +4,7 @@ load("/tools/build_rules/genproto", "proto_java_library") FILES = [ "build", + "dash", "deps", "java_compilation", "crosstool_config", diff --git a/src/main/protobuf/dash.proto b/src/main/protobuf/dash.proto new file mode 100644 index 0000000000..c4c394485f --- /dev/null +++ b/src/main/protobuf/dash.proto @@ -0,0 +1,59 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package dash; + +option java_package = "com.google.devtools.build.lib.bazel.dash"; +option java_outer_classname = "DashProtos"; + +message BuildData { + optional string build_id = 1; + optional string command_name = 2; + optional string working_dir = 3; + + message CommandLine { + message Option { + optional string name = 1; + repeated string value = 2; + optional string source = 3; + } + repeated Option startup_options = 1; + repeated Option options = 2; + repeated string residue = 3; + } + optional CommandLine command_line = 4; + + message EnvironmentVar { + optional string name = 1; + optional string value = 2; + } + repeated EnvironmentVar client_env = 5; + + message Target { + optional string label = 1; + optional string rule_kind = 2; + } + repeated Target targets = 6; + + message TestData { + optional string label = 2; + optional bool passed = 3; + optional bytes log = 4; + // Log is truncated after 1MB. + optional bool truncated = 5; + } + repeated TestData test_data = 7; +} diff --git a/src/test/java/BUILD b/src/test/java/BUILD index 8d37e27128..0e9e9221fc 100644 --- a/src/test/java/BUILD +++ b/src/test/java/BUILD @@ -247,6 +247,7 @@ java_library( "//src/main/java:events", "//src/main/java:options", "//src/main/java:packages", + "//src/main/java:runtime", "//src/main/java:skyframe-base", "//src/main/java:vfs", "//src/main/protobuf:proto_extra_actions_base", diff --git a/src/tools/dash/BUILD b/src/tools/dash/BUILD new file mode 100644 index 0000000000..a54a263ff0 --- /dev/null +++ b/src/tools/dash/BUILD @@ -0,0 +1,8 @@ +load("/tools/build_rules/appengine/appengine", "appengine_war") + +appengine_war( + name = "dash", + data = ["//src/tools/dash/src/main/webapp"], + data_path = "/src/tools/dash/src/main/webapp", + jars = ["//src/tools/dash/src/main/java:java-bin_deploy.jar"], +) diff --git a/src/tools/dash/README.md b/src/tools/dash/README.md new file mode 100644 index 0000000000..c68de4560b --- /dev/null +++ b/src/tools/dash/README.md @@ -0,0 +1,52 @@ +# A Dashboard for Bazel + +This is a self-hosted dashboard for Bazel. In particular, this runs a server +that turns build results and logs into webpages. + +## Running the server + +Build and run the server: + +```bash +$ bazel build //src/tools/dash:dash +$ bazel-bin/src/tools/dash +``` + +Once you see the log message `INFO: Dev App Server is now running`, you +can visit [http://localhost:8080] to see the main page (which should say "No +builds, yet!"). + +This builds a .war file that can be deployed to AppEngine (although this +doc assumes you'll run it locally). + +_Note: as of this writing, there is no authentication, rate limiting, or other +protection for the dashboard. Anyone who can access the URL can read and write +data to it. You may want to specify the `--address` or `--host` option +(depending on AppEngine SDK version) when you run `dash` to bind the server to +an internal network address._ + +## Configuring Bazel to write results to the dashboard + +You will need to tell Bazel where to send build results. Run `bazel` with the +`--use_dash` and `--dash_url=http://localhost:8080` flags, for +example: + +```bash +$ bazel build --use_dash --dash_url=http://localhost:8080 //foo:bar +``` + +If you don't want to have to specify the flags for every build and test, add +the following lines to your .bazelrc (either in your home directory, +_~/.bazelrc_, or on a per-project basis): + +``` +build --use_dash +build --dash_url=http://localhost:8080 +``` + +Then build results will be sent to the dashboard by default. You can specify +`--use_dash=false` for a particular build if you don't want it sent. + +Please email the +[mailing list](https://groups.google.com/forum/#!forum/bazel-discuss) +with any questions or concerns. diff --git a/src/tools/dash/src/main/java/BUILD b/src/tools/dash/src/main/java/BUILD new file mode 100644 index 0000000000..285b68658d --- /dev/null +++ b/src/tools/dash/src/main/java/BUILD @@ -0,0 +1,13 @@ +java_binary( + name = "java-bin", + srcs = glob(["**/*.java"]), + main_class = "does.not.exist", + visibility = ["//src/tools/dash:__pkg__"], + deps = [ + "@appengine-java//:api", + "//external:javax/servlet/api", + "//src/main/protobuf:proto_dash", + "//third_party:apache_velocity", + "//third_party:guava", + ], +) diff --git a/src/tools/dash/src/main/java/com/google/devtools/dash/BuildViewServlet.java b/src/tools/dash/src/main/java/com/google/devtools/dash/BuildViewServlet.java new file mode 100644 index 0000000000..e7faf2c513 --- /dev/null +++ b/src/tools/dash/src/main/java/com/google/devtools/dash/BuildViewServlet.java @@ -0,0 +1,88 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.dash; + +import com.google.appengine.api.datastore.Blob; +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.PreparedQuery; +import com.google.appengine.api.datastore.Query; +import com.google.appengine.api.datastore.Query.FilterOperator; +import com.google.appengine.api.datastore.Query.FilterPredicate; +import com.google.common.html.HtmlEscapers; +import com.google.devtools.build.lib.bazel.dash.DashProtos.BuildData; + +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; + +import java.io.IOException; +import java.io.StringWriter; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Handles HTTP gets of builds/tests. + */ +public class BuildViewServlet extends HttpServlet { + private static final String BUILD_ID = + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"; + private DatastoreService datastore; + + public BuildViewServlet() { + super(); + datastore = DatastoreServiceFactory.getDatastoreService(); + } + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse response) throws IOException { + DashRequest request; + try { + request = new DashRequest(req); + } catch (IllegalArgumentException e) { + // TODO(kchodorow): make an error page. + response.setContentType("text/html"); + response.getWriter().println("Error: " + HtmlEscapers.htmlEscaper().escape(e.getMessage())); + return; + } + Key buildKey = KeyFactory.createKey(DashRequest.KEY_KIND, request.getBuildId()); + + BuildData.Builder data = BuildData.newBuilder(); + Query query = new Query(DashRequest.KEY_KIND).setFilter(new FilterPredicate( + DashRequest.BUILD_ID, FilterOperator.EQUAL, request.getBuildId())); + PreparedQuery preparedQuery = datastore.prepare(query); + for (Entity result : preparedQuery.asIterable()) { + data.mergeFrom(BuildData.parseFrom( + ((Blob) result.getProperty(DashRequest.BUILD_DATA)).getBytes())); + } + + VelocityEngine velocityEngine = new VelocityEngine(); + velocityEngine.init(); + Template template = velocityEngine.getTemplate("result.html"); + VelocityContext context = new VelocityContext(); + + context.put("build_data", data); + + StringWriter writer = new StringWriter(); + template.merge(context, writer); + response.setContentType("text/html"); + response.getWriter().println(writer.toString()); + } +} diff --git a/src/tools/dash/src/main/java/com/google/devtools/dash/DashRequest.java b/src/tools/dash/src/main/java/com/google/devtools/dash/DashRequest.java new file mode 100644 index 0000000000..c4af2ff24f --- /dev/null +++ b/src/tools/dash/src/main/java/com/google/devtools/dash/DashRequest.java @@ -0,0 +1,73 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.dash; + +import com.google.appengine.api.datastore.Blob; +import com.google.appengine.api.datastore.Entity; +import com.google.common.io.ByteStreams; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.servlet.http.HttpServletRequest; + +/** + * Parent class for dash-related servlets. + */ +class DashRequest { + public static final String KEY_KIND = "build"; + + public static final String BUILD_ID = "build_id"; + public static final String PAGE_NAME = "page_name"; + public static final String BUILD_DATA = "build_data"; + + // URI is something like "/result/d2c64e09-df4e-461d-869e-33f014488655". + private static final Pattern URI_REGEX = Pattern.compile( + "/(\\w+)/([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})"); + + private final String pageName; + private final String buildId; + private final Blob blob; + + DashRequest(HttpServletRequest request) { + Matcher matcher = URI_REGEX.matcher(request.getRequestURI()); + if (matcher.find()) { + pageName = matcher.group(0); + buildId = matcher.group(1); + } else { + throw new IllegalArgumentException("Invalid URI pattern: " + request.getRequestURI()); + } + try { + // Requests are capped at 32MB (see + // https://cloud.google.com/appengine/docs/quotas?csw=1#Requests). + blob = new Blob(ByteStreams.toByteArray(request.getInputStream())); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read request body: " + e.getMessage()); + } + } + + public String getBuildId() { + return buildId; + } + + public Entity getEntity() { + Entity entity = new Entity(DashRequest.KEY_KIND); + entity.setProperty(BUILD_ID, buildId); + entity.setProperty(PAGE_NAME, pageName); + entity.setProperty(BUILD_DATA, blob); + return entity; + } +} diff --git a/src/tools/dash/src/main/java/com/google/devtools/dash/StoreServlet.java b/src/tools/dash/src/main/java/com/google/devtools/dash/StoreServlet.java new file mode 100644 index 0000000000..2cb9fce563 --- /dev/null +++ b/src/tools/dash/src/main/java/com/google/devtools/dash/StoreServlet.java @@ -0,0 +1,54 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.dash; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; + +import java.io.IOException; + +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Handles storing a test result. + */ +public class StoreServlet extends HttpServlet { + private DatastoreService datastore; + + public StoreServlet() { + super(); + datastore = DatastoreServiceFactory.getDatastoreService(); + } + + @Override + public void doPost(HttpServletRequest req, HttpServletResponse response) throws IOException { + DashRequest request; + try { + request = new DashRequest(req); + } catch (IllegalArgumentException e) { + response.setContentType("text/json"); + response.getWriter().println( + "{ \"error\": \"" + e.getMessage().replaceAll("\"", "") + "\" }"); + return; + } + + datastore.put(request.getEntity()); + + response.setContentType("text/json"); + response.getWriter().println("{ \"ok\": true }"); + } +} diff --git a/src/tools/dash/src/main/webapp/BUILD b/src/tools/dash/src/main/webapp/BUILD new file mode 100644 index 0000000000..a4f46de041 --- /dev/null +++ b/src/tools/dash/src/main/webapp/BUILD @@ -0,0 +1,8 @@ +filegroup( + name = "webapp", + srcs = glob( + ["**"], + exclude = ["BUILD"], + ), + visibility = ["//src/tools/dash:__pkg__"], +) diff --git a/src/tools/dash/src/main/webapp/WEB-INF/appengine-web.xml b/src/tools/dash/src/main/webapp/WEB-INF/appengine-web.xml new file mode 100644 index 0000000000..7118a0ed20 --- /dev/null +++ b/src/tools/dash/src/main/webapp/WEB-INF/appengine-web.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="utf-8"?> +<appengine-web-app xmlns="http://appengine.google.com/ns/1.0"> + <application>dash-of-bazel</application> + <version>1</version> + <threadsafe>true</threadsafe> +</appengine-web-app> diff --git a/src/tools/dash/src/main/webapp/WEB-INF/web.xml b/src/tools/dash/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000000..a4b6daffee --- /dev/null +++ b/src/tools/dash/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE web-app PUBLIC +"-//Oracle Corporation//DTD Web Application 2.3//EN" +"http://java.sun.com/dtd/web-app_2_3.dtd"> + +<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"> + <servlet> + <servlet-name>build</servlet-name> + <servlet-class>com.google.devtools.dash.BuildViewServlet</servlet-class> + </servlet> + <servlet> + <servlet-name>store</servlet-name> + <servlet-class>com.google.devtools.dash.StoreServlet</servlet-class> + </servlet> + + <servlet-mapping> + <servlet-name>build</servlet-name> + <url-pattern>/result/*</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>store</servlet-name> + <url-pattern>/start/*</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>store</servlet-name> + <url-pattern>/options/*</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>store</servlet-name> + <url-pattern>/targets/*</url-pattern> + </servlet-mapping> + <servlet-mapping> + <servlet-name>store</servlet-name> + <url-pattern>/test/*</url-pattern> + </servlet-mapping> + + <welcome-file-list> + <welcome-file>index.html</welcome-file> + </welcome-file-list> +</web-app> diff --git a/src/tools/dash/src/main/webapp/dashboard.css b/src/tools/dash/src/main/webapp/dashboard.css new file mode 100644 index 0000000000..d3d3904b11 --- /dev/null +++ b/src/tools/dash/src/main/webapp/dashboard.css @@ -0,0 +1,110 @@ +/* + * Base structure + */ + +/* Move down content because we have a fixed navbar that is 50px tall */ +body { + padding-top: 50px; +} + + +/* + * Global add-ons + */ + +.sub-header { + padding-bottom: 10px; + border-bottom: 1px solid #eee; +} + +/* + * Top navigation + * Hide default border to remove 1px line. + */ +.navbar-fixed-top { + border: 0; +} + +/* + * Sidebar + */ + +/* Hide for mobile, show later */ +.sidebar { + display: none; +} +@media (min-width: 768px) { + .sidebar { + position: fixed; + top: 51px; + bottom: 0; + left: 0; + z-index: 1000; + display: block; + padding: 20px; + overflow-x: hidden; + overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */ + background-color: #f5f5f5; + border-right: 1px solid #eee; + } +} + +/* Sidebar navigation */ +.nav-sidebar { + margin-right: -21px; /* 20px padding + 1px border */ + margin-bottom: 20px; + margin-left: -20px; +} +.nav-sidebar > li > a { + padding-right: 20px; + padding-left: 20px; +} +.nav-sidebar > .active > a, +.nav-sidebar > .active > a:hover, +.nav-sidebar > .active > a:focus { + color: #fff; + background-color: #428bca; +} + + +/* + * Main content + */ + +.main { + padding: 20px; +} +@media (min-width: 768px) { + .main { + padding-right: 40px; + padding-left: 40px; + } +} +.main .page-header { + margin-top: 0; +} + + +/* + * Placeholder dashboard ideas + */ + +.placeholders { + margin-bottom: 30px; + text-align: center; +} +.placeholders h4 { + margin-bottom: 0; +} +.placeholder { + margin-bottom: 20px; +} +.placeholder img { + display: inline-block; + border-radius: 50%; +} + +code { + color: #00A388; + background-color: transparent; +} diff --git a/src/tools/dash/src/main/webapp/favicon.ico b/src/tools/dash/src/main/webapp/favicon.ico Binary files differnew file mode 100644 index 0000000000..a37c760980 --- /dev/null +++ b/src/tools/dash/src/main/webapp/favicon.ico diff --git a/src/tools/dash/src/main/webapp/index.html b/src/tools/dash/src/main/webapp/index.html new file mode 100644 index 0000000000..db1b6af926 --- /dev/null +++ b/src/tools/dash/src/main/webapp/index.html @@ -0,0 +1,69 @@ +<!DOCTYPE html> +<html lang="en"> + <!-- TODO(kchodorow): make this and result.html inherit from a parent template. --> + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come + *after* these tags --> + <meta name="description" content=""> + <meta name="author" content=""> + <link rel="icon" href="/favicon.ico"> + + <title>Build $build_data.getBuildId()</title> + + <!-- Bootstrap core CSS --> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" + rel="stylesheet"> + + <!-- Custom styles for this template --> + <link href="/dashboard.css" rel="stylesheet"> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> + <![endif]--> + </head> + + <body> + + <nav class="navbar navbar-inverse navbar-fixed-top"> + <div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" + data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="#">Dash of Bazel</a> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav navbar-right"> + <li><a href="/">Dashboard</a></li> + <!-- TODO(kchodorow): link to a bazel.io documentation. --> + <li><a href="#todo">Help</a></li> + </ul> + <form class="navbar-form navbar-right"> + <!-- TODO(kchodorow): add fulltext search. --> + <input type="text" class="form-control" placeholder="Search..."> + </form> + </div> + </div> + </nav> + + <div class="container-fluid"> + <!-- TODO(kchodorow): add a list of existing builds. --> + <h2>No builds, yet!</h2> + </div> + + <!-- Bootstrap core JavaScript + ================================================== --> + <!-- Placed at the end of the document so the pages load faster --> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> + </body> +</html> diff --git a/src/tools/dash/src/main/webapp/result.html b/src/tools/dash/src/main/webapp/result.html new file mode 100644 index 0000000000..106fec6e6f --- /dev/null +++ b/src/tools/dash/src/main/webapp/result.html @@ -0,0 +1,156 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <!-- The above 3 meta tags *must* come first in the head; any other head content must come + *after* these tags --> + <meta name="description" content=""> + <meta name="author" content=""> + <link rel="icon" href="/favicon.ico"> + + <title>Build $build_data.getBuildId()</title> + + <!-- Bootstrap core CSS --> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" + rel="stylesheet"> + + <!-- Custom styles for this template --> + <link href="/dashboard.css" rel="stylesheet"> + + <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> + <!--[if lt IE 9]> + <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script> + <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script> + <![endif]--> +</head> + +<body> + +<nav class="navbar navbar-inverse navbar-fixed-top"> + <div class="container-fluid"> + <div class="navbar-header"> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" + data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + <a class="navbar-brand" href="#">$build_data.getBuildId()</a> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav navbar-right"> + <li><a href="/">Dashboard</a></li> + <!-- TODO(kchodorow): link to a bazel.io documentation. --> + <li><a href="#">Help</a></li> + </ul> + <form class="navbar-form navbar-right"> + <!-- TODO(kchodorow): add fulltext search. --> + <input type="text" class="form-control" placeholder="Search..."> + </form> + </div> + </div> +</nav> + +<div class="container-fluid"> + <div class="row"> + <div class="col-sm-3 col-md-2 sidebar"> + <ul class="nav nav-sidebar"> + <li class="active"> + <a href="#results"> + Results <span class="sr-only">(current)</span> + </a> + </li> + <li><a href="#command-line">Command line</a></li> + <li><a href="#env">Environment</a></li> + </ul> + </div> + + <div id="results" class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> + <h1 class="page-header">bazel $build_data.getCommandName() results</h1> + <div class="table-responsive"> + <table class="table table-striped"> + <thead> + <tr> + <th>Target</th> + <th>Status</th> + </tr> + </thead> + <tbody> + #if ($build_data.getCommandName() == "test") + #foreach( $result in $build_data.getTestDataList() ) + <tr> + #if ( $result.getPassed() ) + <td> + <div> + <code style="color:#79BD8F; background-color: transparent;">$result.getLabel()</code> + </div> + </td> + <td style="color:#79BD8F;">Passed</td> + #else + <td> + <div onclick="$('#log-$velocityCount').toggle();"> + <code style="color:#FF6138;">$result.getLabel()</code> + </div> + <pre id="log-$velocityCount" style="display: none;"> +$result.getLog().toStringUtf8() + </pre> + #if ( $result.getTruncated() ) + <div>Truncated after 1MB, see local log for full output.</div> + #end + </td> + <td style="color:#FF6138;">Failed</td> + #end + </tr> + #end + #else + #foreach( $result in $build_data.getTargetsList() ) + <tr> + <td><code>$result.getLabel()</code></td> + <td>Built</td> + </tr> + #end + #end + </tbody> + </table> + </div> + </div> + + <div id="command-line" class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> + <h1>bazel command line</h1> + <div> + <pre> +bazel #foreach( $option in $build_data.getCommandLine().getStartupOptionsList())--$option #end $build_data.getCommandName() #foreach( $option in $build_data.getCommandLine().getOptionsList())--$option #end #foreach( $residue in $build_data.getCommandLine().getResidueList())$residue #end + </pre> + </div> + </div> + + <div id="env" class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> + <h1>bazel environment</h1> + <div> + <table> + <tr> + <th>Name</th> + <th>Value</th> + </tr> + #foreach( $env in $build_data.getClientEnvList()) + <tr> + <td><code>$env.getName()</code></td> + <td><code>$env.getValue()</code></td> + </tr> + #end + </pre> + </div> + </div> + </div> +</div> + +<!-- Bootstrap core JavaScript +================================================== --> +<!-- Placed at the end of the document so the pages load faster --> +<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script> +<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script> +</body> +</html> diff --git a/tools/build_rules/appengine/appengine.bzl b/tools/build_rules/appengine/appengine.bzl index 5a1bae00bc..337a74982f 100644 --- a/tools/build_rules/appengine/appengine.bzl +++ b/tools/build_rules/appengine/appengine.bzl @@ -62,14 +62,6 @@ appengine_war( jar_filetype = FileType([".jar"]) -def _extract_jar(zipper, jar, output): - return [ - "mkdir -p %s" % output, - "(root=$(pwd);" + - ("cd %s && " % output) + - ("${root}/%s x ${root}/%s)\n" % (zipper.path, jar.path)) - ] - def _add_file(in_file, output, path = None): output_path = output input_path = in_file.path @@ -77,7 +69,8 @@ def _add_file(in_file, output, path = None): output_path += input_path[len(path):] return [ "mkdir -p $(dirname %s)" % output_path, - "ln -s $(pwd)/%s %s\n" % (input_path, output_path) + "test -l %s || ln -s $(pwd)/%s %s\n" % (output_path, input_path, + output_path) ] def _make_war(zipper, input_dir, output): @@ -120,14 +113,17 @@ def _war_impl(ctxt): ] inputs = ctxt.files.jars + [zipper] + cmd += ["mkdir -p %s" % build_output + "/WEB-INF/lib"] for jar in ctxt.files.jars: - # Add the jar content to WEB-INF/classes - cmd += _extract_jar(zipper, jar, build_output + "/WEB-INF/classes") + # Add the jar to WEB-INF/lib. + cmd += _add_file(jar, build_output + "/WEB-INF/lib") # Add its runtime classpath to WEB-INF/lib if hasattr(jar, "java"): inputs += jar.java.transitive_runtime_deps for run_jar in jar.java.transitive_runtime_deps: cmd += _add_file(run_jar, build_output + "/WEB-INF/lib") + for jar in ctxt.files._appengine_deps: + cmd += _add_file(jar, build_output + "/WEB-INF/lib") inputs += ctxt.files.data for res in ctxt.files.data: @@ -148,7 +144,7 @@ def _war_impl(ctxt): for f in ctxt.files._appengine_sdk: if not appengine_sdk: appengine_sdk = f.path - elif not file.path.startswith(appengine_sdk): + elif not f.path.startswith(appengine_sdk): appengine_sdk = _common_substring(appengine_sdk, f.path) classpath = [ @@ -174,13 +170,12 @@ def _war_impl(ctxt): " fi", "fi", "", - "tmp_dir=$(mktemp -d ${TMPDIR:-/tmp}/war.XXXXXXXX)", - "trap \"rm -rf ${tmp_dir}\" EXIT", "root_path=$(pwd)", + "tmp_dir=$(mktemp -d ${TMPDIR:-/tmp}/war.XXXXXXXX)", + "trap \"cd ${root_path}; rm -rf ${tmp_dir}\" EXIT", "cd ${tmp_dir}", "${JAVA_RUNFILES}/%s x ${JAVA_RUNFILES}/%s" % ( ctxt.file._zipper.short_path, ctxt.outputs.war.short_path), - "cd ${root_path}", "jvm_bin=${JAVA_RUNFILES}/%s" % (ctxt.file._java.short_path), "if [[ ! -x ${jvm_bin} ]]; then", " jvm_bin=$(which java)", @@ -214,6 +209,13 @@ appengine_war = rule( default=Label("//external:appengine/java/sdk")), "_appengine_jars": attr.label( default=Label("//external:appengine/java/jars")), + "_appengine_deps": attr.label_list( + default=[ + Label("@appengine-java//:api"), + Label("@commons-lang//jar"), + Label("//third_party:apache_commons_collections"), + ] + ), "jars": attr.label_list(allow_files=jar_filetype, mandatory=True), "data": attr.label_list(allow_files=True), "data_path": attr.string(), |