From 68aa410229cb36ceedc910c803a0aff2db6d027f Mon Sep 17 00:00:00 2001 From: ulfjack Date: Fri, 15 Jun 2018 01:40:02 -0700 Subject: Add a mechanism for build event protocol events to upload files This should be a no-op, mostly replacing PathConverter with BuildEventArtifactUploader, since none of the implementations perform any upload yet. PiperOrigin-RevId: 200685325 --- .../build/lib/buildeventstream/BuildEvent.java | 23 ++++- .../BuildEventArtifactUploader.java | 38 ++++++++ .../BuildEventArtifactUploaderMap.java | 59 ++++++++++++ .../BuildEventProtocolOptions.java | 11 ++- .../lib/buildeventstream/BuildEventTransport.java | 9 +- .../build/lib/buildeventstream/BuildToolLogs.java | 10 ++ .../build/lib/buildeventstream/PathConverter.java | 64 ++++++++++++- .../build/lib/buildeventstream/transports/BUILD | 2 + .../transports/BinaryFormatFileTransport.java | 46 +++------ .../transports/BuildEventTransportFactory.java | 105 +++++++++------------ .../buildeventstream/transports/FileTransport.java | 91 +++++++++++++++++- .../transports/JsonFormatFileTransport.java | 46 +++------ .../transports/TextFormatFileTransport.java | 43 +++------ 13 files changed, 382 insertions(+), 165 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java create mode 100644 src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploaderMap.java (limited to 'src/main/java/com/google/devtools/build/lib/buildeventstream') diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java index de4cfb8c88..6d6900687c 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java @@ -14,7 +14,10 @@ package com.google.devtools.build.lib.buildeventstream; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.vfs.Path; +import java.util.Set; /** * Interface for objects that can be posted on the public event stream. @@ -23,11 +26,29 @@ import com.google.devtools.build.lib.events.ExtendedEventHandler; * pass-through of events, as well as proper chaining of events. */ public interface BuildEvent extends ChainableEvent, ExtendedEventHandler.Postable { + + /** + * Returns a list of files that are referenced in the protobuf representation returned by {@link + * #asStreamProto(BuildEventContext)}. + * + *

This method is different from {@code EventReportingArtifacts#reportedArtifacts()} in that it + * only returns files directly referenced in the protobuf returned by {@link + * #asStreamProto(BuildEventContext)}. + * + *

Note the consistency requirement - you must not attempt to pass Path objects to the + * {@link PathConverter} unless you have returned the Path object here. + */ + // TODO(ulfjack): Consider moving the upload call to the BuildEventContext and returning a map + // from Path to URI, rather than a callback. + default Set referencedLocalFiles() { + return ImmutableSet.of(); + } + /** * Provide a binary representation of the event. * *

Provide a presentation of the event according to the specified binary format, as appropriate * protocol buffer. */ - BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext converters); + BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext context); } diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java new file mode 100644 index 0000000000..7b7ffb15ab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java @@ -0,0 +1,38 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildeventstream; + +import com.google.devtools.build.lib.buildeventstream.PathConverter.FileUriPathConverter; +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import java.util.Set; + +/** Uploads artifacts referenced by the Build Event Protocol (BEP). */ +public interface BuildEventArtifactUploader { + public static final BuildEventArtifactUploader LOCAL_FILES_UPLOADER = + new BuildEventArtifactUploader() { + @Override + public PathConverter upload(Set files) { + return new FileUriPathConverter(); + } + }; + + /** + * Uploads a set of files referenced by the protobuf representation of a {@link BuildEvent}. + * + *

Returns a {@link PathConverter} that must provide a name for each uploaded file as it should + * appear in the BEP. + */ + PathConverter upload(Set files) throws IOException, InterruptedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploaderMap.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploaderMap.java new file mode 100644 index 0000000000..8987def882 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploaderMap.java @@ -0,0 +1,59 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildeventstream; + +import com.google.common.collect.ImmutableMap; +import java.util.SortedMap; +import java.util.TreeMap; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** Selects between multiple available upload strategies. */ +@ThreadSafe +public class BuildEventArtifactUploaderMap { + private final ImmutableMap uploaders; + + private BuildEventArtifactUploaderMap( + ImmutableMap uploaders) { + this.uploaders = uploaders; + } + + public BuildEventArtifactUploader select(@Nullable String name) { + if (name == null) { + // TODO(b/110235226): We currently choose the strategy with alphabetically first strategy, + // which happens to be backwards-compatible; we need to set + // experimental_build_event_upload_strategy to appropriate default values instead, and then + // make it an error to pass null. + return uploaders.values().iterator().next(); + } + return uploaders.getOrDefault(name, BuildEventArtifactUploader.LOCAL_FILES_UPLOADER); + } + + /** Builder class for {@link BuildEventArtifactUploaderMap}. */ + public static class Builder { + private final SortedMap uploaders = new TreeMap<>(); + + public Builder() { + } + + public Builder add(String name, BuildEventArtifactUploader uploader) { + uploaders.put(name, uploader); + return this; + } + + public BuildEventArtifactUploaderMap build() { + return new BuildEventArtifactUploaderMap(ImmutableMap.copyOf(uploaders)); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventProtocolOptions.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventProtocolOptions.java index e22d1669ef..4f31f75eaa 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventProtocolOptions.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventProtocolOptions.java @@ -28,7 +28,16 @@ public class BuildEventProtocolOptions extends OptionsBase { documentationCategory = OptionDocumentationCategory.LOGGING, effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, help = "Use this to suppress generation of the legacy important_outputs field in the " - + "TargetComplete event" + + "TargetComplete event." ) public boolean legacyImportantOutputs; + + @Option( + name = "experimental_build_event_upload_strategy", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = "Selects how to upload artifacts referenced in the build event protocol." + ) + public String buildEventUploadStrategy; } diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventTransport.java index 9b930b9ecd..9d61c1c4e3 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventTransport.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventTransport.java @@ -26,7 +26,6 @@ import javax.annotation.concurrent.ThreadSafe; */ @ThreadSafe public interface BuildEventTransport { - /** * The name of this transport as can be displayed to a user. */ @@ -39,16 +38,14 @@ public interface BuildEventTransport { *

In case the transport is in error, this method still needs to be able to accept build * events. It may choose to ignore them, though. * - *

This method should not throw any exceptions. - * * @param event the event to sendBuildEvent. */ void sendBuildEvent(BuildEvent event, ArtifactGroupNamer namer); /** - * Initiates a close. Callers may listen to the returned future to be notified when the close - * is complete i.e. wait for all build events to be sent. The future does not contain any - * information about possible transport errors. + * Initiates a close. Callers may listen to the returned future to be notified when the close is + * complete i.e. wait for all build events to be sent. The future may contain any information + * about possible transport errors. * *

This method might be called multiple times without any effect after the first call. * diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java index fe68043eb0..c4eee48b28 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.buildeventstream; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.Path; import com.google.protobuf.ByteString; @@ -40,6 +41,15 @@ public class BuildToolLogs implements BuildEventWithOrderConstraint { return ImmutableList.of(); } + @Override + public ImmutableSet referencedLocalFiles() { + ImmutableSet.Builder artifacts = ImmutableSet.builder(); + for (Pair logFile : logFiles) { + artifacts.add(logFile.getSecond()); + } + return artifacts.build(); + } + @Override public BuildEventStreamProtos.BuildEvent asStreamProto(BuildEventContext converters) { BuildEventStreamProtos.BuildToolLogs.Builder toolLogs = diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/PathConverter.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/PathConverter.java index e230d36953..a1aaf31d40 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/PathConverter.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/PathConverter.java @@ -13,15 +13,75 @@ // limitations under the License. package com.google.devtools.build.lib.buildeventstream; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.devtools.build.lib.vfs.Path; +import java.net.URI; +import java.net.URISyntaxException; /** * Interface for conversion of paths to URIs. */ public interface PathConverter { + /** An implementation that throws on every call to {@link #apply(Path)}. */ + public static final PathConverter NO_CONVERSION = new PathConverter() { + @Override + public String apply(Path path) { + throw new IllegalStateException( + String.format( + "Can't convert '%s', as it has not been" + + "declared as a referenced artifact of a build event", + path.getPathString())); + } + }; + + /** A {@link PathConverter} that returns a path formatted as a URI with a {@code file} scheme. */ + // TODO(ulfjack): Make this a static final field. + final class FileUriPathConverter implements PathConverter { + @Override + public String apply(Path path) { + Preconditions.checkNotNull(path); + return pathToUriString(path.getPathString()); + } + + /** + * Returns the path encoded as an {@link URI}. + * + *

This concrete implementation returns URIs with "file" as the scheme. For Example: - On + * Unix the path "/tmp/foo bar.txt" will be encoded as "file:///tmp/foo%20bar.txt". - On Windows + * the path "C:\Temp\Foo Bar.txt" will be encoded as "file:///C:/Temp/Foo%20Bar.txt" + * + *

Implementors extending this class for special filesystems will likely need to override + * this method. + */ + @VisibleForTesting + static String pathToUriString(String path) { + if (!path.startsWith("/")) { + // On Windows URI's need to start with a '/'. i.e. C:\Foo\Bar would be file:///C:/Foo/Bar + path = "/" + path; + } + try { + return new URI( + "file", + // Needs to be "" instead of null, so that toString() will append "//" after the + // scheme. + // We need this for backwards compatibility reasons as some consumers of the BEP are + // broken. + "", + path, + null, + null) + .toString(); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + } + /** - * Return the URI corresponding to the given path, if the path can be converted to a URI by this - * path converter; return {@link null} otherwise. + * Return the URI corresponding to the given path. + * + *

This method must not return {@code null}. */ String apply(Path path); } diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BUILD b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BUILD index 7d8fe5709a..117d18cf19 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BUILD +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BUILD @@ -10,7 +10,9 @@ java_library( name = "transports", srcs = glob(["*.java"]), deps = [ + "//src/main/java/com/google/devtools/build/lib:exitcode-external", "//src/main/java/com/google/devtools/build/lib:io", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib/buildeventstream", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/vfs", diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BinaryFormatFileTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BinaryFormatFileTransport.java index 71e4fd878a..2f1d15c25a 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BinaryFormatFileTransport.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BinaryFormatFileTransport.java @@ -14,30 +14,28 @@ package com.google.devtools.build.lib.buildeventstream.transports; -import static com.google.common.base.Preconditions.checkNotNull; - import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer; import com.google.devtools.build.lib.buildeventstream.BuildEvent; -import com.google.devtools.build.lib.buildeventstream.BuildEventContext; +import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; import com.google.devtools.build.lib.buildeventstream.BuildEventTransport; -import com.google.devtools.build.lib.buildeventstream.PathConverter; +import com.google.devtools.build.lib.util.AbruptExitException; import java.io.IOException; +import java.util.function.Consumer; /** * A simple {@link BuildEventTransport} that writes a varint delimited binary representation of * {@link BuildEvent} protocol buffers to a file. */ public final class BinaryFormatFileTransport extends FileTransport { - private final BuildEventProtocolOptions options; - private final PathConverter pathConverter; - BinaryFormatFileTransport( - String path, BuildEventProtocolOptions options, PathConverter pathConverter) - throws IOException { - super(path); - this.options = options; - this.pathConverter = pathConverter; + String path, + BuildEventProtocolOptions options, + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { + super(path, options, uploader, exitFunc); } @Override @@ -47,24 +45,10 @@ public final class BinaryFormatFileTransport extends FileTransport { @Override public synchronized void sendBuildEvent(BuildEvent event, final ArtifactGroupNamer namer) { - checkNotNull(event); - BuildEventContext converters = - new BuildEventContext() { - @Override - public PathConverter pathConverter() { - return pathConverter; - } - - @Override - public ArtifactGroupNamer artifactGroupNamer() { - return namer; - } - - @Override - public BuildEventProtocolOptions getOptions() { - return options; - } - }; - write(event.asStreamProto(converters)); + BuildEventStreamProtos.BuildEvent protoEvent = asStreamProto(event, namer); + if (protoEvent == null) { + return; + } + write(protoEvent); } } diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventTransportFactory.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventTransportFactory.java index 0cc84b0f74..eef7583451 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventTransportFactory.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/BuildEventTransportFactory.java @@ -17,13 +17,13 @@ package com.google.devtools.build.lib.buildeventstream.transports; import static com.google.common.base.Strings.isNullOrEmpty; import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; +import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploaderMap; import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions; import com.google.devtools.build.lib.buildeventstream.BuildEventTransport; -import com.google.devtools.build.lib.buildeventstream.PathConverter; -import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.util.AbruptExitException; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; +import java.util.function.Consumer; /** Factory used to create a Set of BuildEventTransports from BuildEventStreamOptions. */ public enum BuildEventTransportFactory { @@ -37,11 +37,16 @@ public enum BuildEventTransportFactory { protected BuildEventTransport create( BuildEventStreamOptions options, BuildEventProtocolOptions protocolOptions, - PathConverter pathConverter) throws IOException { + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { return new TextFormatFileTransport( - options.getBuildEventTextFile(), - protocolOptions, - options.getBuildEventTextFilePathConversion() ? pathConverter : new NullPathConverter()); + options.getBuildEventTextFile(), protocolOptions, uploader, exitFunc); + } + + @Override + protected boolean usePathConverter(BuildEventStreamOptions options) { + return options.getBuildEventTextFilePathConversion(); } }, @@ -55,13 +60,16 @@ public enum BuildEventTransportFactory { protected BuildEventTransport create( BuildEventStreamOptions options, BuildEventProtocolOptions protocolOptions, - PathConverter pathConverter) throws IOException { + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { return new BinaryFormatFileTransport( - options.getBuildEventBinaryFile(), - protocolOptions, - options.getBuildEventBinaryFilePathConversion() - ? pathConverter - : new NullPathConverter()); + options.getBuildEventBinaryFile(), protocolOptions, uploader, exitFunc); + } + + @Override + protected boolean usePathConverter(BuildEventStreamOptions options) { + return options.getBuildEventBinaryFilePathConversion(); } }, @@ -75,11 +83,16 @@ public enum BuildEventTransportFactory { protected BuildEventTransport create( BuildEventStreamOptions options, BuildEventProtocolOptions protocolOptions, - PathConverter pathConverter) throws IOException { + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { return new JsonFormatFileTransport( - options.getBuildEventJsonFile(), - protocolOptions, - options.getBuildEventJsonFilePathConversion() ? pathConverter : new NullPathConverter()); + options.getBuildEventJsonFile(), protocolOptions, uploader, exitFunc); + } + + @Override + protected boolean usePathConverter(BuildEventStreamOptions options) { + return options.getBuildEventJsonFilePathConversion(); } }; @@ -94,13 +107,17 @@ public enum BuildEventTransportFactory { public static ImmutableSet createFromOptions( BuildEventStreamOptions options, BuildEventProtocolOptions protocolOptions, - PathConverter pathConverter) - throws IOException { + BuildEventArtifactUploaderMap artifactUploaders, + Consumer exitFunc) + throws IOException { ImmutableSet.Builder buildEventTransportsBuilder = ImmutableSet.builder(); for (BuildEventTransportFactory transportFactory : BuildEventTransportFactory.values()) { if (transportFactory.enabled(options)) { + BuildEventArtifactUploader uploader = transportFactory.usePathConverter(options) + ? artifactUploaders.select(protocolOptions.buildEventUploadStrategy) + : BuildEventArtifactUploader.LOCAL_FILES_UPLOADER; buildEventTransportsBuilder.add( - transportFactory.create(options, protocolOptions, pathConverter)); + transportFactory.create(options, protocolOptions, uploader, exitFunc)); } } return buildEventTransportsBuilder.build(); @@ -113,47 +130,9 @@ public enum BuildEventTransportFactory { protected abstract BuildEventTransport create( BuildEventStreamOptions options, BuildEventProtocolOptions protocolOptions, - PathConverter pathConverter) - throws IOException; + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException; - private static class NullPathConverter implements PathConverter { - @Override - public String apply(Path path) { - return pathToUriString(path.getPathString()); - } - } - - /** - * Returns the path encoded as an {@link URI}. - * - *

This concrete implementation returns URIs with "file" as the scheme. For Example: - On Unix - * the path "/tmp/foo bar.txt" will be encoded as "file:///tmp/foo%20bar.txt". - On Windows the - * path "C:\Temp\Foo Bar.txt" will be encoded as "file:///C:/Temp/Foo%20Bar.txt" - * - *

Implementors extending this class for special filesystems will likely need to override this - * method. - * - * @throws URISyntaxException if the URI cannot be constructed. - */ - static String pathToUriString(String path) { - if (!path.startsWith("/")) { - // On Windows URI's need to start with a '/'. i.e. C:\Foo\Bar would be file:///C:/Foo/Bar - path = "/" + path; - } - try { - return new URI( - "file", - // Needs to be "" instead of null, so that toString() will append "//" after the - // scheme. - // We need this for backwards compatibility reasons as some consumers of the BEP are - // broken. - "", - path, - null, - null) - .toString(); - } catch (URISyntaxException e) { - throw new IllegalStateException(e); - } - } + protected abstract boolean usePathConverter(BuildEventStreamOptions options); } diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java index a2d1164bb2..540a7e84cd 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java @@ -14,16 +14,32 @@ package com.google.devtools.build.lib.buildeventstream.transports; +import static com.google.common.base.Preconditions.checkNotNull; + import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Throwables; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer; +import com.google.devtools.build.lib.buildeventstream.BuildEvent; +import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; +import com.google.devtools.build.lib.buildeventstream.BuildEventContext; +import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; import com.google.devtools.build.lib.buildeventstream.BuildEventTransport; +import com.google.devtools.build.lib.buildeventstream.PathConverter; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream; +import com.google.devtools.build.lib.vfs.Path; import com.google.protobuf.Message; import java.io.IOException; +import java.util.Set; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; +import javax.annotation.Nullable; /** * Non-blocking file transport. @@ -35,10 +51,23 @@ import java.util.logging.Logger; abstract class FileTransport implements BuildEventTransport { private static final Logger logger = Logger.getLogger(FileTransport.class.getName()); + private final BuildEventProtocolOptions options; + private final BuildEventArtifactUploader uploader; + private final Consumer exitFunc; + private boolean errored; + @VisibleForTesting final AsynchronousFileOutputStream out; - FileTransport(String path) throws IOException { + FileTransport( + String path, + BuildEventProtocolOptions options, + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { + this.uploader = uploader; + this.options = options; + this.exitFunc = exitFunc; out = new AsynchronousFileOutputStream(path); } @@ -60,7 +89,6 @@ abstract class FileTransport implements BuildEventTransport { } } - @Override public synchronized ListenableFuture close() { return Futures.catching( @@ -77,4 +105,63 @@ abstract class FileTransport implements BuildEventTransport { public void closeNow() { out.closeNow(); } + + /** + * Converts the given event into a proto object; this may trigger uploading of referenced files as + * a side effect. May return {@code null} if there was an interrupt. This method is not + * thread-safe. + */ + @Nullable + protected BuildEventStreamProtos.BuildEvent asStreamProto( + BuildEvent event, ArtifactGroupNamer namer) { + checkNotNull(event); + PathConverter pathConverter = uploadReferencedFiles(event.referencedLocalFiles()); + if (pathConverter == null) { + return null; + } + + BuildEventContext context = + new BuildEventContext() { + @Override + public PathConverter pathConverter() { + return pathConverter; + } + + @Override + public ArtifactGroupNamer artifactGroupNamer() { + return namer; + } + + @Override + public BuildEventProtocolOptions getOptions() { + return options; + } + }; + return event.asStreamProto(context); + } + + /** + * Returns a {@link PathConverter} for the uploaded files, or {@code null} when the uploaded + * failed. + */ + private @Nullable PathConverter uploadReferencedFiles(Set artifacts) { + checkNotNull(artifacts); + + if (errored) { + return null; + } + try { + return uploader.upload(artifacts); + } catch (IOException e) { + errored = true; + exitFunc.accept( + new AbruptExitException( + Throwables.getStackTraceAsString(e), ExitCode.PUBLISH_ERROR, e)); + } catch (InterruptedException e) { + errored = true; + exitFunc.accept(new AbruptExitException(ExitCode.INTERRUPTED, e)); + Thread.currentThread().interrupt(); + } + return null; + } } diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/JsonFormatFileTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/JsonFormatFileTransport.java index 3d51c4f632..33b296143e 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/JsonFormatFileTransport.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/JsonFormatFileTransport.java @@ -16,28 +16,28 @@ package com.google.devtools.build.lib.buildeventstream.transports; import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer; import com.google.devtools.build.lib.buildeventstream.BuildEvent; -import com.google.devtools.build.lib.buildeventstream.BuildEventContext; +import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; import com.google.devtools.build.lib.buildeventstream.BuildEventTransport; -import com.google.devtools.build.lib.buildeventstream.PathConverter; +import com.google.devtools.build.lib.util.AbruptExitException; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.io.IOException; +import java.util.function.Consumer; /** * A simple {@link BuildEventTransport} that writes the JSON representation of the protocol-buffer * representation of the events to a file. */ public final class JsonFormatFileTransport extends FileTransport { - private final BuildEventProtocolOptions options; - private final PathConverter pathConverter; - JsonFormatFileTransport( - String path, BuildEventProtocolOptions options, PathConverter pathConverter) - throws IOException { - super(path); - this.options = options; - this.pathConverter = pathConverter; + String path, + BuildEventProtocolOptions options, + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { + super(path, options, uploader, exitFunc); } @Override @@ -47,30 +47,14 @@ public final class JsonFormatFileTransport extends FileTransport { @Override public synchronized void sendBuildEvent(BuildEvent event, final ArtifactGroupNamer namer) { - BuildEventContext converters = - new BuildEventContext() { - @Override - public PathConverter pathConverter() { - return pathConverter; - } - - @Override - public ArtifactGroupNamer artifactGroupNamer() { - return namer; - } - - @Override - public BuildEventProtocolOptions getOptions() { - return options; - } - }; + BuildEventStreamProtos.BuildEvent protoEvent = asStreamProto(event, namer); + if (protoEvent == null) { + return; + } String protoJsonRepresentation; try { protoJsonRepresentation = - JsonFormat.printer() - .omittingInsignificantWhitespace() - .print(event.asStreamProto(converters)) - + "\n"; + JsonFormat.printer().omittingInsignificantWhitespace().print(protoEvent) + "\n"; } catch (InvalidProtocolBufferException e) { // We don't expect any unknown Any fields in our protocol buffer. Nevertheless, handle // the exception gracefully and, at least, return valid JSON with an id field. diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/TextFormatFileTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/TextFormatFileTransport.java index 2735712bfb..af1364f3bf 100644 --- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/TextFormatFileTransport.java +++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/TextFormatFileTransport.java @@ -16,12 +16,14 @@ package com.google.devtools.build.lib.buildeventstream.transports; import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer; import com.google.devtools.build.lib.buildeventstream.BuildEvent; -import com.google.devtools.build.lib.buildeventstream.BuildEventContext; +import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader; import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions; +import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos; import com.google.devtools.build.lib.buildeventstream.BuildEventTransport; -import com.google.devtools.build.lib.buildeventstream.PathConverter; +import com.google.devtools.build.lib.util.AbruptExitException; import com.google.protobuf.TextFormat; import java.io.IOException; +import java.util.function.Consumer; /** * A simple {@link BuildEventTransport} that writes the text representation of the protocol-buffer @@ -30,15 +32,13 @@ import java.io.IOException; *

This class is used for debugging. */ public final class TextFormatFileTransport extends FileTransport { - private final BuildEventProtocolOptions options; - private final PathConverter pathConverter; - TextFormatFileTransport( - String path, BuildEventProtocolOptions options, PathConverter pathConverter) - throws IOException { - super(path); - this.options = options; - this.pathConverter = pathConverter; + String path, + BuildEventProtocolOptions options, + BuildEventArtifactUploader uploader, + Consumer exitFunc) + throws IOException { + super(path, options, uploader, exitFunc); } @Override @@ -48,24 +48,11 @@ public final class TextFormatFileTransport extends FileTransport { @Override public synchronized void sendBuildEvent(BuildEvent event, final ArtifactGroupNamer namer) { - BuildEventContext converters = - new BuildEventContext() { - @Override - public PathConverter pathConverter() { - return pathConverter; - } - - @Override - public ArtifactGroupNamer artifactGroupNamer() { - return namer; - } - - @Override - public BuildEventProtocolOptions getOptions() { - return options; - } - }; - String protoTextRepresentation = TextFormat.printToString(event.asStreamProto(converters)); + BuildEventStreamProtos.BuildEvent protoEvent = asStreamProto(event, namer); + if (protoEvent == null) { + return; + } + String protoTextRepresentation = TextFormat.printToString(protoEvent); write("event {\n" + protoTextRepresentation + "}\n\n"); } } -- cgit v1.2.3