From b86f8b06f5f4832b51e3be0de9d32170a79371ee Mon Sep 17 00:00:00 2001 From: janakr Date: Thu, 17 Aug 2017 17:49:50 +0200 Subject: Open-source Skyframe serialization, and make AppleConfiguration serializable as a pilot. Currently not hooked up to anything, but will be shortly. PiperOrigin-RevId: 165583517 --- src/main/java/com/google/devtools/build/lib/BUILD | 2 + .../lib/rules/apple/AppleCommandLineOptions.java | 70 +++++++++++ .../build/lib/rules/apple/AppleConfiguration.java | 98 ++++++++++++++- .../build/lib/rules/apple/ApplePlatform.java | 3 + .../google/devtools/build/lib/rules/apple/BUILD | 5 +- .../build/lib/rules/apple/DottedVersion.java | 50 +++++++- .../skyframe/NotSerializableRuntimeException.java | 24 ---- .../lib/skyframe/TargetPatternPhaseValue.java | 1 + .../lib/skyframe/TestSuiteExpansionValue.java | 1 + .../build/lib/skyframe/TestsInSuiteValue.java | 1 + .../build/lib/skyframe/serialization/BUILD | 20 +++ .../lib/skyframe/serialization/BaseCodec.java | 31 +++++ .../lib/skyframe/serialization/EnumCodec.java | 59 +++++++++ .../skyframe/serialization/FastStringCodec.java | 140 +++++++++++++++++++++ .../skyframe/serialization/ImmutableListCodec.java | 62 +++++++++ .../lib/skyframe/serialization/LabelCodec.java | 49 ++++++++ .../NotSerializableRuntimeException.java | 25 ++++ .../lib/skyframe/serialization/ObjectCodec.java | 47 +++++++ .../serialization/PackageIdentifierCodec.java | 49 ++++++++ .../skyframe/serialization/PathFragmentCodec.java | 55 ++++++++ .../serialization/RepositoryNameCodec.java | 56 +++++++++ .../serialization/SerializationCommonUtils.java | 41 ++++++ .../serialization/SerializationException.java | 50 ++++++++ .../lib/skyframe/serialization/StringCodec.java | 38 ++++++ 24 files changed, 942 insertions(+), 35 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/NotSerializableRuntimeException.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/BaseCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/FastStringCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/ImmutableListCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/LabelCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/NotSerializableRuntimeException.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/PackageIdentifierCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/PathFragmentCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/RepositoryNameCodec.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationCommonUtils.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationException.java create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/serialization/StringCodec.java (limited to 'src/main') diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index eb3b78d53f..18b40a4636 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -37,6 +37,7 @@ filegroup( "//src/main/java/com/google/devtools/build/lib/rules/genquery:srcs", "//src/main/java/com/google/devtools/build/lib/rules/genrule:srcs", "//src/main/java/com/google/devtools/build/lib/rules/objc:srcs", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization:srcs", "//src/main/java/com/google/devtools/build/lib/analysis/featurecontrol:srcs", "//src/main/java/com/google/devtools/build/lib/analysis/platform:srcs", "//src/main/java/com/google/devtools/build/lib/analysis/whitelisting:srcs", @@ -641,6 +642,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/causes", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java index 36c9449462..268c47d4e7 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.rules.apple; +import static com.google.devtools.build.lib.skyframe.serialization.SerializationCommonUtils.STRING_LIST_CODEC; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -24,6 +26,10 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.rules.apple.AppleConfiguration.ConfigurationDistinguisher; import com.google.devtools.build.lib.rules.apple.ApplePlatform.PlatformType; +import com.google.devtools.build.lib.skyframe.serialization.EnumCodec; +import com.google.devtools.build.lib.skyframe.serialization.FastStringCodec; +import com.google.devtools.build.lib.skyframe.serialization.LabelCodec; +import com.google.devtools.build.lib.skyframe.serialization.SerializationException; import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; import com.google.devtools.build.lib.skylarkinterface.SkylarkModuleCategory; import com.google.devtools.build.lib.skylarkinterface.SkylarkPrinter; @@ -34,6 +40,9 @@ import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionDocumentationCategory; import com.google.devtools.common.options.OptionEffectTag; import com.google.devtools.common.options.OptionMetadataTag; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; import java.util.List; /** @@ -470,6 +479,8 @@ public class AppleCommandLineOptions extends FragmentOptions { super(AppleBitcodeMode.class, "apple bitcode mode"); } } + + static final EnumCodec CODEC = new EnumCodec<>(AppleBitcodeMode.class); } @Override @@ -492,6 +503,65 @@ public class AppleCommandLineOptions extends FragmentOptions { return host; } + void serialize(CodedOutputStream out) throws IOException, SerializationException { + out.writeBoolNoTag(mandatoryMinimumVersion); + xcodeVersion.serialize(out); + iosSdkVersion.serialize(out); + watchOsSdkVersion.serialize(out); + tvOsSdkVersion.serialize(out); + macOsSdkVersion.serialize(out); + iosMinimumOs.serialize(out); + watchosMinimumOs.serialize(out); + tvosMinimumOs.serialize(out); + macosMinimumOs.serialize(out); + FastStringCodec.INSTANCE.serialize(iosCpu, out); + LabelCodec.INSTANCE.serialize(appleCrosstoolTop, out); + PlatformType.CODEC.serialize(applePlatformType, out); + FastStringCodec.INSTANCE.serialize(appleSplitCpu, out); + ConfigurationDistinguisher.CODEC.serialize(configurationDistinguisher, out); + STRING_LIST_CODEC.serialize((ImmutableList) iosMultiCpus, out); + STRING_LIST_CODEC.serialize((ImmutableList) watchosCpus, out); + STRING_LIST_CODEC.serialize((ImmutableList) tvosCpus, out); + STRING_LIST_CODEC.serialize((ImmutableList) macosCpus, out); + LabelCodec.INSTANCE.serialize(defaultProvisioningProfile, out); + LabelCodec.INSTANCE.serialize(xcodeVersionConfig, out); + FastStringCodec.INSTANCE.serialize(xcodeToolchain, out); + AppleBitcodeMode.CODEC.serialize(appleBitcodeMode, out); + out.writeBoolNoTag(enableAppleCrosstoolTransition); + out.writeBoolNoTag(targetUsesAppleCrosstool); + } + + static AppleCommandLineOptions deserialize(CodedInputStream in) + throws IOException, SerializationException { + AppleCommandLineOptions result = new AppleCommandLineOptions(); + result.mandatoryMinimumVersion = in.readBool(); + result.xcodeVersion = DottedVersion.deserialize(in); + result.iosSdkVersion = DottedVersion.deserialize(in); + result.watchOsSdkVersion = DottedVersion.deserialize(in); + result.tvOsSdkVersion = DottedVersion.deserialize(in); + result.macOsSdkVersion = DottedVersion.deserialize(in); + result.iosMinimumOs = DottedVersion.deserialize(in); + result.watchosMinimumOs = DottedVersion.deserialize(in); + result.tvosMinimumOs = DottedVersion.deserialize(in); + result.macosMinimumOs = DottedVersion.deserialize(in); + result.iosCpu = FastStringCodec.INSTANCE.deserialize(in); + result.appleCrosstoolTop = LabelCodec.INSTANCE.deserialize(in); + result.applePlatformType = PlatformType.CODEC.deserialize(in); + result.appleSplitCpu = FastStringCodec.INSTANCE.deserialize(in); + result.configurationDistinguisher = ConfigurationDistinguisher.CODEC.deserialize(in); + result.iosMultiCpus = STRING_LIST_CODEC.deserialize(in); + result.watchosCpus = STRING_LIST_CODEC.deserialize(in); + result.tvosCpus = STRING_LIST_CODEC.deserialize(in); + result.macosCpus = STRING_LIST_CODEC.deserialize(in); + result.defaultProvisioningProfile = LabelCodec.INSTANCE.deserialize(in); + result.xcodeVersionConfig = LabelCodec.INSTANCE.deserialize(in); + result.xcodeToolchain = FastStringCodec.INSTANCE.deserialize(in); + result.appleBitcodeMode = AppleBitcodeMode.CODEC.deserialize(in); + result.enableAppleCrosstoolTransition = in.readBool(); + result.targetUsesAppleCrosstool = in.readBool(); + return result; + } + /** Converter for the Apple configuration distinguisher. */ public static final class ConfigurationDistinguisherConverter extends EnumConverter { diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java index 3ab182ad59..b2a1a98604 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java @@ -31,13 +31,20 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.rules.apple.AppleCommandLineOptions.AppleBitcodeMode; import com.google.devtools.build.lib.rules.apple.ApplePlatform.PlatformType; +import com.google.devtools.build.lib.skyframe.serialization.EnumCodec; +import com.google.devtools.build.lib.skyframe.serialization.FastStringCodec; +import com.google.devtools.build.lib.skyframe.serialization.SerializationException; import com.google.devtools.build.lib.skylarkinterface.SkylarkCallable; import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; import com.google.devtools.build.lib.skylarkinterface.SkylarkModuleCategory; import com.google.devtools.build.lib.util.Preconditions; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import javax.annotation.Nullable; /** A configuration containing flags required for Apple platforms and tools. */ @@ -99,10 +106,11 @@ public class AppleConfiguration extends BuildConfiguration.Fragment { @Nullable private final Label defaultProvisioningProfileLabel; private final boolean mandatoryMinimumVersion; + @VisibleForTesting AppleConfiguration( AppleCommandLineOptions options, - String cpu, - XcodeVersionProperties xcodeVersionProperties, + String iosCpu, + @Nullable DottedVersion xcodeVersion, DottedVersion iosSdkVersion, DottedVersion iosMinimumOs, DottedVersion watchosSdkVersion, @@ -126,8 +134,8 @@ public class AppleConfiguration extends BuildConfiguration.Fragment { Preconditions.checkNotNull(macosSdkVersion, "macOsSdkVersion"); this.macosMinimumOs = Preconditions.checkNotNull(macosMinimumOs, "macOsMinimumOs"); - this.xcodeVersion = xcodeVersionProperties.getXcodeVersion().orNull(); - this.iosCpu = iosCpuFromCpu(cpu); + this.xcodeVersion = xcodeVersion; + this.iosCpu = iosCpu; this.appleSplitCpu = Preconditions.checkNotNull(options.appleSplitCpu, "appleSplitCpu"); this.applePlatformType = Preconditions.checkNotNull(options.applePlatformType, "applePlatformType"); @@ -670,6 +678,81 @@ public class AppleConfiguration extends BuildConfiguration.Fragment { .build(); } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof AppleConfiguration)) { + return false; + } + AppleConfiguration that = (AppleConfiguration) obj; + return this.options.equals(that.options) + && Objects.equals(this.xcodeVersion, that.xcodeVersion) + && this.iosSdkVersion.equals(that.iosSdkVersion) + && this.iosMinimumOs.equals(that.iosMinimumOs) + && this.watchosSdkVersion.equals(that.watchosSdkVersion) + && this.watchosMinimumOs.equals(that.watchosMinimumOs) + && this.tvosSdkVersion.equals(that.tvosSdkVersion) + && this.tvosMinimumOs.equals(that.tvosMinimumOs) + && this.macosSdkVersion.equals(that.macosSdkVersion) + && this.macosMinimumOs.equals(that.macosMinimumOs); + } + + @Override + public int hashCode() { + return Objects.hash( + options, + xcodeVersion, + iosSdkVersion, + iosMinimumOs, + watchosSdkVersion, + watchosMinimumOs, + tvosSdkVersion, + tvosMinimumOs, + macosSdkVersion, + macosMinimumOs); + } + + void serialize(CodedOutputStream out) throws IOException, SerializationException { + options.serialize(out); + out.writeStringNoTag(iosCpu); + if (xcodeVersion == null) { + out.writeBoolNoTag(false); + } else { + out.writeBoolNoTag(true); + xcodeVersion.serialize(out); + } + iosSdkVersion.serialize(out); + iosMinimumOs.serialize(out); + watchosSdkVersion.serialize(out); + watchosMinimumOs.serialize(out); + tvosSdkVersion.serialize(out); + tvosMinimumOs.serialize(out); + macosSdkVersion.serialize(out); + macosMinimumOs.serialize(out); + } + + static AppleConfiguration deserialize(CodedInputStream in) + throws IOException, SerializationException { + AppleCommandLineOptions options = AppleCommandLineOptions.deserialize(in); + String iosCpu = FastStringCodec.INSTANCE.deserialize(in); + boolean hasXcodeVersion = in.readBool(); + DottedVersion xcodeVersion = hasXcodeVersion ? DottedVersion.deserialize(in) : null; + return new AppleConfiguration( + options, + iosCpu, + xcodeVersion, + DottedVersion.deserialize(in), + DottedVersion.deserialize(in), + DottedVersion.deserialize(in), + DottedVersion.deserialize(in), + DottedVersion.deserialize(in), + DottedVersion.deserialize(in), + DottedVersion.deserialize(in), + DottedVersion.deserialize(in)); + } + /** * Loads {@link AppleConfiguration} from build options. */ @@ -701,8 +784,8 @@ public class AppleConfiguration extends BuildConfiguration.Fragment { AppleConfiguration configuration = new AppleConfiguration( appleOptions, - cpu, - xcodeVersionProperties, + iosCpuFromCpu(cpu), + xcodeVersionProperties.getXcodeVersion().orNull(), iosSdkVersion, iosMinimumOsVersion, watchosSdkVersion, @@ -781,5 +864,8 @@ public class AppleConfiguration extends BuildConfiguration.Fragment { public String getFileSystemName() { return fileSystemName; } + + static final EnumCodec CODEC = + new EnumCodec<>(ConfigurationDistinguisher.class); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java b/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java index 6be2024464..bccee16485 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java @@ -19,6 +19,7 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.packages.Info; import com.google.devtools.build.lib.packages.NativeProvider; import com.google.devtools.build.lib.packages.Provider; +import com.google.devtools.build.lib.skyframe.serialization.EnumCodec; import com.google.devtools.build.lib.skylarkinterface.SkylarkCallable; import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; import com.google.devtools.build.lib.skylarkinterface.SkylarkModuleCategory; @@ -299,5 +300,7 @@ public enum ApplePlatform implements SkylarkValue { public void repr(SkylarkPrinter printer) { printer.append(toString()); } + + static final EnumCodec CODEC = new EnumCodec<>(PlatformType.class); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/BUILD b/src/main/java/com/google/devtools/build/lib/rules/apple/BUILD index 15ae78dbf2..b6fb30189c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/BUILD @@ -14,15 +14,14 @@ java_library( "//src/main/java/com/google/devtools/build/lib:events", "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:preconditions", - "//src/main/java/com/google/devtools/build/lib:shell", "//src/main/java/com/google/devtools/build/lib:skylarkinterface", "//src/main/java/com/google/devtools/build/lib:syntax", - "//src/main/java/com/google/devtools/build/lib:vfs", - "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization", "//src/main/java/com/google/devtools/common/options", "//third_party:guava", "//third_party:jsr305", + "//third_party/protobuf:protobuf_java", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java b/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java index bdbc910905..5fbbef561c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java @@ -26,10 +26,14 @@ import com.google.devtools.build.lib.skylarkinterface.SkylarkModule; import com.google.devtools.build.lib.skylarkinterface.SkylarkModuleCategory; import com.google.devtools.build.lib.skylarkinterface.SkylarkPrinter; import com.google.devtools.build.lib.skylarkinterface.SkylarkValue; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nullable; /** * Represents a value with multiple components, separated by periods, for example {@code 4.5.6} or @@ -249,13 +253,35 @@ public final class DottedVersion implements Comparable, SkylarkVa printer.append(stringRepresentation); } + void serialize(CodedOutputStream out) throws IOException { + out.writeInt32NoTag(components.size()); + for (Component component : components) { + component.serialize(out); + } + out.writeStringNoTag(stringRepresentation); + out.writeInt32NoTag(numOriginalComponents); + } + + static DottedVersion deserialize(CodedInputStream in) throws IOException { + int numComponents = in.readInt32(); + // TODO(janakr: Presize this if/when https://github.com/google/guava/issues/196 is resolved. + ImmutableList.Builder components = ImmutableList.builder(); + for (int i = 0; i < numComponents; i++) { + components.add(Component.deserialize(in)); + } + return new DottedVersion(components.build(), in.readString(), in.readInt32()); + } + private static final class Component implements Comparable { private final int firstNumber; - private final String alphaSequence; + @Nullable private final String alphaSequence; private final int secondNumber; private final String stringRepresentation; - public Component(int firstNumber, String alphaSequence, int secondNumber, + public Component( + int firstNumber, + @Nullable String alphaSequence, + int secondNumber, String stringRepresentation) { this.firstNumber = firstNumber; this.alphaSequence = alphaSequence; @@ -293,5 +319,25 @@ public final class DottedVersion implements Comparable, SkylarkVa public String toString() { return stringRepresentation; } + + void serialize(CodedOutputStream out) throws IOException { + if (alphaSequence == null) { + out.writeBoolNoTag(false); + } else { + out.writeBoolNoTag(true); + out.writeStringNoTag(alphaSequence); + } + out.writeInt32NoTag(firstNumber); + out.writeInt32NoTag(secondNumber); + out.writeStringNoTag(stringRepresentation); + } + + static Component deserialize(CodedInputStream in) throws IOException { + String alphaSequence = null; + if (in.readBool()) { + alphaSequence = in.readString(); + } + return new Component(in.readInt32(), alphaSequence, in.readInt32(), in.readString()); + } } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/NotSerializableRuntimeException.java b/src/main/java/com/google/devtools/build/lib/skyframe/NotSerializableRuntimeException.java deleted file mode 100644 index c54bd45dd7..0000000000 --- a/src/main/java/com/google/devtools/build/lib/skyframe/NotSerializableRuntimeException.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.skyframe; - -import java.io.Serializable; - -/** - * An exception that can be thrown by {@code writeObject} and {@code readObject} implementations of - * {@link Serializable} objects to indicate that they cannot actually be serialized. Should be used - * only as a shim: in the long run, all {@link Serializable} objects should actually be - * serializable. - */ -public class NotSerializableRuntimeException extends RuntimeException {} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java index 5d237076a1..49cd581f52 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TargetPatternPhaseValue.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.pkgcache.LoadingResult; import com.google.devtools.build.lib.pkgcache.TestFilter; +import com.google.devtools.build.lib.skyframe.serialization.NotSerializableRuntimeException; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestSuiteExpansionValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestSuiteExpansionValue.java index 1da1feeffb..8bc2f2bfb4 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/TestSuiteExpansionValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestSuiteExpansionValue.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.cmdline.ResolvedTargets; import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.skyframe.serialization.NotSerializableRuntimeException; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/TestsInSuiteValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/TestsInSuiteValue.java index bed1646217..63a55fbd86 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/TestsInSuiteValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/TestsInSuiteValue.java @@ -19,6 +19,7 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.skyframe.serialization.NotSerializableRuntimeException; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD new file mode 100644 index 0000000000..6de7a8e5a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD @@ -0,0 +1,20 @@ +# TODO(janakr): find out how to avoid this default visibility and still have +# automatic BUILD-file generation. +package(default_visibility = ["//src:__subpackages__"]) + +java_library( + name = "serialization", + srcs = glob(["*.java"]), + deps = [ + "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:guava", + "//third_party/protobuf:protobuf_java", + ], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), +) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BaseCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BaseCodec.java new file mode 100644 index 0000000000..dd286fc7eb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BaseCodec.java @@ -0,0 +1,31 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +/** An opaque interface for codecs that just reveals the {@link Class} of its objects. */ +interface BaseCodec { + /** + * Returns the class of the objects serialized/deserialized by this codec. + * + *

This is useful for automatically dispatching to the correct codec, e.g. in {@link + * ObjectCodecs} and {@link BaseCodecMap}. It may also be useful for automatically registering + * codecs for {@link SkyKey}s and {@link SkyValue}s instead of using the current manual mapping + * (b/26186886). + */ + Class getEncodedClass(); +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumCodec.java new file mode 100644 index 0000000000..523620e1cb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/EnumCodec.java @@ -0,0 +1,59 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; + +/** Codec for an enum. */ +public class EnumCodec> implements ObjectCodec { + + private final Class enumClass; + + /** + * A cached copy of T.values(), to avoid allocating an array upon every deserialization operation. + */ + private final ImmutableList values; + + public EnumCodec(Class enumClass) { + this.enumClass = enumClass; + this.values = ImmutableList.copyOf(enumClass.getEnumConstants()); + } + + @Override + public Class getEncodedClass() { + return enumClass; + } + + @Override + public void serialize(T value, CodedOutputStream codedOut) throws IOException { + Preconditions.checkNotNull(value, "Enum value for %s is null", enumClass); + codedOut.writeEnumNoTag(value.ordinal()); + } + + @Override + public T deserialize(CodedInputStream codedIn) throws SerializationException, IOException { + int ordinal = codedIn.readEnum(); + try { + return values.get(ordinal); + } catch (ArrayIndexOutOfBoundsException e) { + throw new SerializationException( + "Invalid ordinal for " + enumClass.getName() + " enum: " + ordinal, e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/FastStringCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/FastStringCodec.java new file mode 100644 index 0000000000..23bf38baae --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/FastStringCodec.java @@ -0,0 +1,140 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import sun.misc.Unsafe; + +/** + * Similar to {@link StringCodec}, except with deserialization optimized for ascii data. It can + * still handle UTF-8, though less efficiently than {@link StringCodec}. Should be used when the + * majority of the data passing through will be ascii. + */ +public class FastStringCodec implements ObjectCodec { + public static final FastStringCodec INSTANCE = new FastStringCodec(); + + private static final Unsafe theUnsafe; + private static final long STRING_VALUE_OFFSET; + + private static final String EMPTY_STRING = ""; + + static { + theUnsafe = getUnsafe(); + try { + // String's 'value' field stores its char[]. If this field changes name or type then the + // reflective check below will fail. We can reasonably expect our approach to be stable for + // now, but things are likely to change in java 9, hopefully in a way which obsoletes this + // optimization. + Field valueField = String.class.getDeclaredField("value"); + Class valueFieldType = valueField.getType(); + if (!valueFieldType.equals(char[].class)) { + throw new AssertionError( + "Expected String's value field to be char[], but was " + valueFieldType); + } + STRING_VALUE_OFFSET = theUnsafe.objectFieldOffset(valueField); + } catch (NoSuchFieldException | SecurityException e) { + throw new AssertionError("Failed to find String's 'value' offset", e); + } + } + + @Override + public Class getEncodedClass() { + return String.class; + } + + @Override + public void serialize(String string, CodedOutputStream codedOut) throws IOException { + codedOut.writeStringNoTag(string); + } + + @Override + public String deserialize(CodedInputStream codedIn) throws IOException { + int length = codedIn.readInt32(); + if (length == 0) { + return EMPTY_STRING; + } + + char[] maybeDecoded = new char[length]; + for (int i = 0; i < length; i++) { + // Read one byte at a time to avoid creating a new ByteString/copy of the underlying array. + byte b = codedIn.readRawByte(); + // Check highest order bit, if it's set we've crossed into extended ascii/utf8. + if ((b & 0x80) == 0) { + maybeDecoded[i] = (char) b; + } else { + // Fail, we encountered a non-ascii byte. Copy what we have so far plus and then the rest + // of the data into a buffer and let String's constructor do the UTF-8 decoding work. + byte[] decodeFrom = new byte[length]; + for (int j = 0; j < i; j++) { + decodeFrom[j] = (byte) maybeDecoded[j]; + } + decodeFrom[i] = b; + for (int j = i + 1; j < length; j++) { + decodeFrom[j] = codedIn.readRawByte(); + } + return new String(decodeFrom, StandardCharsets.UTF_8); + } + } + + try { + String result = (String) theUnsafe.allocateInstance(String.class); + theUnsafe.putObject(result, STRING_VALUE_OFFSET, maybeDecoded); + return result; + } catch (Exception e) { + // This should only catch InstantiationException, but that makes IntelliJ unhappy for + // some reason; it insists that that exception cannot be thrown from here, even though it + // is set to JDK 8 + throw new IllegalStateException("Could not create string", e); + } + } + + /** + * Get a reference to {@link sun.misc.Unsafe} or throw an {@link AssertionError} if failing to do + * so. Failure is highly unlikely, but possible if the underlying VM stores unsafe in an + * unexpected location. + */ + private static Unsafe getUnsafe() { + try { + // sun.misc.Unsafe is intentionally difficult to get a hold of - it gives us the power to + // do things like access raw memory and segfault the JVM. + return AccessController.doPrivileged( + new PrivilegedExceptionAction() { + @Override + public Unsafe run() throws Exception { + Class unsafeClass = Unsafe.class; + // Unsafe usually exists in the field 'theUnsafe', however check all fields + // in case it's somewhere else in this VM's version of Unsafe. + for (Field f : unsafeClass.getDeclaredFields()) { + f.setAccessible(true); + Object fieldValue = f.get(null); + if (unsafeClass.isInstance(fieldValue)) { + return unsafeClass.cast(fieldValue); + } + } + throw new AssertionError("Failed to find sun.misc.Unsafe instance"); + } + }); + } catch (PrivilegedActionException pae) { + throw new AssertionError("Unable to get sun.misc.Unsafe", pae); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ImmutableListCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ImmutableListCodec.java new file mode 100644 index 0000000000..1b54d1fb17 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ImmutableListCodec.java @@ -0,0 +1,62 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; + +/** Encodes a list of elements using a specified {@link ObjectCodec}. */ +public class ImmutableListCodec implements ObjectCodec> { + + private final ObjectCodec codec; + + public ImmutableListCodec(ObjectCodec codec) { + this.codec = codec; + } + + @SuppressWarnings("unchecked") + @Override + public Class> getEncodedClass() { + // Compiler doesn't like cast from Class -> Class>, but it + // does allow what we see below. Type is lost at runtime anyway, so while gross this works. + return (Class>) ((Class) ImmutableList.class); + } + + @Override + public void serialize(ImmutableList list, CodedOutputStream codedOut) + throws SerializationException, IOException { + codedOut.writeInt32NoTag(list.size()); + for (T item : list) { + codec.serialize(item, codedOut); + } + } + + @Override + public ImmutableList deserialize(CodedInputStream codedIn) + throws SerializationException, IOException { + int length = codedIn.readInt32(); + if (length < 0) { + throw new SerializationException("Expected non-negative length: " + length); + } + + ImmutableList.Builder builder = ImmutableList.builder(); + for (int i = 0; i < length; i++) { + builder.add(codec.deserialize(codedIn)); + } + return builder.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/LabelCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/LabelCodec.java new file mode 100644 index 0000000000..6f8881dfef --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/LabelCodec.java @@ -0,0 +1,49 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; + +/** Custom serialization logic for {@link Label}s. */ +public class LabelCodec implements ObjectCodec