diff options
author | 2018-02-20 13:20:30 -0800 | |
---|---|---|
committer | 2018-02-20 13:22:33 -0800 | |
commit | 80a0633bbeef3f84ab6eb6513a5d36936cd1bdbb (patch) | |
tree | a4ce5b4d8422a6103d0c93fb6ed8db3ccd1f1c22 /src/main/java/com/google/devtools/build/lib | |
parent | dd090a6e8825093e7e5364ed49f16ba68f0fe54c (diff) |
Adds ObjectCodecRegistry to {Des|S}erializationContext.
* AutoCodec now delegates to the registry.
* Adds getSuperclass logic for resolving a codec.
* Small cleanups for classes that break the registry.
TODO after this change:
* Explicit CODEC definitions are no longer needed and existing ones should be cleaned up.
* POLYMORPHIC is no longer be needed and should be cleaned up.
PiperOrigin-RevId: 186351845
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib')
16 files changed, 429 insertions, 192 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java index 02904d45bc..27b97a4521 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java +++ b/src/main/java/com/google/devtools/build/lib/actions/ArtifactOwner.java @@ -16,7 +16,6 @@ package com.google.devtools.build.lib.actions; import com.google.common.annotations.VisibleForTesting; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec; -import com.google.devtools.build.lib.skyframe.serialization.SingletonCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; /** @@ -34,11 +33,10 @@ public interface ArtifactOwner { * An {@link ArtifactOwner} that just returns null for its label. Only for use with resolved * source artifacts and tests. */ + @AutoCodec(strategy = AutoCodec.Strategy.SINGLETON) class NullArtifactOwner implements ArtifactOwner { @VisibleForTesting public static final NullArtifactOwner INSTANCE = new NullArtifactOwner(); - static final ObjectCodec<NullArtifactOwner> CODEC = SingletonCodec.of(INSTANCE, "NULL_OWNER"); - private NullArtifactOwner() {} @Override diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetCodec.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetCodec.java index ec61883b2a..255456139a 100644 --- a/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetCodec.java +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/NestedSetCodec.java @@ -42,11 +42,6 @@ import java.util.Map; public class NestedSetCodec<T> implements ObjectCodec<NestedSet<T>> { private static final EnumCodec<Order> orderCodec = new EnumCodec<>(Order.class); - private final ObjectCodec<T> objectCodec; - - public NestedSetCodec(ObjectCodec<T> objectCodec) { - this.objectCodec = objectCodec; - } @SuppressWarnings("unchecked") @Override @@ -76,7 +71,7 @@ public class NestedSetCodec<T> implements ObjectCodec<NestedSet<T>> { int nestedSetCount = codedIn.readInt32(); Preconditions.checkState( nestedSetCount >= 1, - "Should have at least serialized one nested set, got: %d", + "Should have at least serialized one nested set, got: %s", nestedSetCount); Order order = orderCodec.deserialize(context, codedIn); Object children = null; @@ -131,7 +126,7 @@ public class NestedSetCodec<T> implements ObjectCodec<NestedSet<T>> { childCodedOut.writeByteArrayNoTag(digest); } else { childCodedOut.writeBoolNoTag(false); - objectCodec.serialize(context, cast(child), childCodedOut); + context.serialize(cast(child), childCodedOut); } } } @@ -141,7 +136,7 @@ public class NestedSetCodec<T> implements ObjectCodec<NestedSet<T>> { throws IOException, SerializationException { childCodedOut.writeInt32NoTag(1); T singleChild = cast(children); - objectCodec.serialize(context, singleChild, childCodedOut); + context.serialize(singleChild, childCodedOut); } private Object deserializeOneNestedSet( @@ -157,7 +152,7 @@ public class NestedSetCodec<T> implements ObjectCodec<NestedSet<T>> { if (childCount > 1) { result = deserializeMultipleItemChildArray(context, digestToChild, childCodedIn, childCount); } else if (childCount == 1) { - result = objectCodec.deserialize(context, childCodedIn); + result = context.deserialize(childCodedIn); } else { result = NestedSet.EMPTY_CHILDREN; } @@ -179,7 +174,7 @@ public class NestedSetCodec<T> implements ObjectCodec<NestedSet<T>> { children[i] = Preconditions.checkNotNull(digestToChild.get(digest), "Transitive nested set missing"); } else { - children[i] = objectCodec.deserialize(context, childCodedIn); + children[i] = context.deserialize(childCodedIn); } } return children; diff --git a/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java b/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java index 46460b0b02..d455d01fbb 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java +++ b/src/main/java/com/google/devtools/build/lib/rules/android/DexArchiveAspect.java @@ -65,7 +65,6 @@ import com.google.devtools.build.lib.rules.java.proto.JavaProtoLibraryAspectProv import com.google.devtools.build.lib.rules.proto.ProtoLangToolchainProvider; import com.google.devtools.build.lib.rules.proto.ProtoSourcesProvider; import com.google.devtools.build.lib.skyframe.ConfiguredTargetAndTarget; -import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -73,7 +72,6 @@ import java.util.Map; import java.util.Set; /** Aspect to {@link DexArchiveProvider build .dex Archives} from Jars. */ -@AutoCodec public final class DexArchiveAspect extends NativeAspectClass implements ConfiguredAspectFactory { public static final String NAME = "DexArchiveAspect"; diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java b/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java index 12e236c2fa..a3085525a0 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/DottedVersion.java @@ -262,38 +262,36 @@ public final class DottedVersion implements Comparable<DottedVersion>, SkylarkVa printer.append(stringRepresentation); } - public static final ObjectCodec<DottedVersion> CODEC = - new ObjectCodec<DottedVersion>() { - @Override - public void serialize( - SerializationContext context, DottedVersion obj, CodedOutputStream codedOut) - throws IOException { - codedOut.writeInt32NoTag(obj.components.size()); - for (Component component : obj.components) { - component.serialize(codedOut); - } - codedOut.writeStringNoTag(obj.stringRepresentation); - codedOut.writeInt32NoTag(obj.numOriginalComponents); - } - - @Override - public DottedVersion deserialize(DeserializationContext context, CodedInputStream codedIn) - throws IOException { - int numComponents = codedIn.readInt32(); - // TODO(janakr: Presize this if/when https://github.com/google/guava/issues/196 is - // resolved. - ImmutableList.Builder<Component> components = ImmutableList.builder(); - for (int i = 0; i < numComponents; i++) { - components.add(Component.deserialize(codedIn)); - } - return new DottedVersion(components.build(), codedIn.readString(), codedIn.readInt32()); - } - - @Override - public Class<DottedVersion> getEncodedClass() { - return DottedVersion.class; - } - }; + private static class DottedVersionCodec implements ObjectCodec<DottedVersion> { + @Override + public Class<DottedVersion> getEncodedClass() { + return DottedVersion.class; + } + + @Override + public void serialize( + SerializationContext context, DottedVersion obj, CodedOutputStream codedOut) + throws IOException { + codedOut.writeInt32NoTag(obj.components.size()); + for (Component component : obj.components) { + component.serialize(codedOut); + } + codedOut.writeStringNoTag(obj.stringRepresentation); + codedOut.writeInt32NoTag(obj.numOriginalComponents); + } + + @Override + public DottedVersion deserialize(DeserializationContext context, CodedInputStream codedIn) + throws IOException { + int numComponents = codedIn.readInt32(); + // TODO(janakr: Presize this if/when https://github.com/google/guava/issues/196 is resolved. + ImmutableList.Builder<Component> components = ImmutableList.builder(); + for (int i = 0; i < numComponents; i++) { + components.add(Component.deserialize(codedIn)); + } + return new DottedVersion(components.build(), codedIn.readString(), codedIn.readInt32()); + } + } private static final class Component implements Comparable<Component> { private final int firstNumber; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java new file mode 100644 index 0000000000..cd8c3c4464 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/AutoRegistry.java @@ -0,0 +1,45 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import java.io.IOException; + +/** + * A lazy, automatically populated registry. + * + * <p>Must not be accessed by any {@link CodecRegisterer} or {@link ObjectCodec} constructors or + * static initializers. + */ +public class AutoRegistry { + + private static final Supplier<ObjectCodecRegistry> SUPPLIER = + Suppliers.memoize(AutoRegistry::create); + + public static ObjectCodecRegistry get() { + return SUPPLIER.get(); + } + + private static ObjectCodecRegistry create() { + try { + return CodecScanner.initializeCodecRegistry("com.google.devtools.build.lib") + .setAllowDefaultCodec(false) + .build(); + } catch (IOException | ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecRegisterer.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecRegisterer.java index ec9ba202f5..72d99191e3 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecRegisterer.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/CodecRegisterer.java @@ -40,5 +40,5 @@ package com.google.devtools.build.lib.skyframe.serialization; */ public interface CodecRegisterer<T extends ObjectCodec<?>> { - void register(ObjectCodecRegistry.Builder builder); + default void register(ObjectCodecRegistry.Builder builder) {} } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java index 3bf44adde1..5377d4ffc6 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/DeserializationContext.java @@ -16,17 +16,36 @@ package com.google.devtools.build.lib.skyframe.serialization; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.CodedInputStream; +import java.io.IOException; /** Stateful class for providing additional context to a single deserialization "session". */ // TODO(bazel-team): This class is just a shell, fill in. public class DeserializationContext { + private final ObjectCodecRegistry registry; private final ImmutableMap<Class<?>, Object> dependencies; - public DeserializationContext(ImmutableMap<Class<?>, Object> dependencies) { + public DeserializationContext( + ObjectCodecRegistry registry, ImmutableMap<Class<?>, Object> dependencies) { + this.registry = registry; this.dependencies = dependencies; } + public DeserializationContext(ImmutableMap<Class<?>, Object> dependencies) { + this(AutoRegistry.get(), dependencies); + } + + // TODO(shahan): consider making codedIn a member of this class. + @SuppressWarnings({"TypeParameterUnusedInFormals", "unchecked"}) + public <T> T deserialize(CodedInputStream codedIn) throws IOException, SerializationException { + int tag = codedIn.readSInt32(); + if (tag == 0) { + return null; + } + return (T) registry.getCodecDescriptorByTag(tag).deserialize(this, codedIn); + } + @SuppressWarnings("unchecked") public <T> T getDependency(Class<T> type) { Preconditions.checkNotNull(type); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/JavaSerializableCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/JavaSerializableCodec.java index e6e4bd6f31..d0b6a7368f 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/JavaSerializableCodec.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/JavaSerializableCodec.java @@ -63,4 +63,8 @@ class JavaSerializableCodec implements ObjectCodec<Object> { throw new SerializationException("Java deserialization failed", e); } } + + /** Disables auto-registration. */ + private static class JavaSerializableCodecRegisterer + implements CodecRegisterer<JavaSerializableCodec> {} } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java index 20f8708e8b..e404ab0145 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/ObjectCodecRegistry.java @@ -17,6 +17,9 @@ package com.google.devtools.build.lib.skyframe.serialization; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; import java.util.Map; import java.util.Map.Entry; import javax.annotation.Nullable; @@ -38,20 +41,19 @@ public class ObjectCodecRegistry { @Nullable private final CodecDescriptor defaultCodecDescriptor; - private ObjectCodecRegistry(Map<String, ObjectCodec<?>> codecs, boolean allowDefaultCodec) { + private ObjectCodecRegistry(Map<String, CodecHolder> codecs, boolean allowDefaultCodec) { ImmutableMap.Builder<String, CodecDescriptor> codecMappingsBuilder = ImmutableMap.builder(); - int nextTag = 0; + int nextTag = 1; // 0 is reserved for null. for (String classifier : ImmutableList.sortedCopyOf(codecs.keySet())) { - codecMappingsBuilder.put(classifier, new CodecDescriptor(nextTag, codecs.get(classifier))); + codecMappingsBuilder.put(classifier, codecs.get(classifier).createDescriptor(nextTag)); nextTag++; } this.stringMappedCodecs = codecMappingsBuilder.build(); this.byteStringMappedCodecs = makeByteStringMappedCodecs(stringMappedCodecs); - this.defaultCodecDescriptor = allowDefaultCodec - ? new CodecDescriptor(nextTag, new JavaSerializableCodec()) - : null; + this.defaultCodecDescriptor = + allowDefaultCodec ? new TypedCodecDescriptor<>(nextTag, new JavaSerializableCodec()) : null; this.tagMappedCodecs = makeTagMappedCodecs(stringMappedCodecs, defaultCodecDescriptor); } @@ -83,26 +85,37 @@ public class ObjectCodecRegistry { } } + /** + * Returns a {@link CodecDescriptor} for the given type. + * + * <p>Falls back to a codec for the nearest super type of type. Failing that, may fall back to the + * registry's default codec. + */ public CodecDescriptor getCodecDescriptor(Class<?> type) throws SerializationException.NoCodecException { - CodecDescriptor result = - stringMappedCodecs.getOrDefault(type.getName(), defaultCodecDescriptor); - if (result != null) { - return result; - } else { + // TODO(blaze-team): consider caching this traversal. + for (Class<?> nextType = type; nextType != null; nextType = nextType.getSuperclass()) { + CodecDescriptor result = stringMappedCodecs.get(nextType.getName()); + if (result != null) { + return result; + } + } + if (defaultCodecDescriptor == null) { throw new SerializationException.NoCodecException( "No codec available for " + type + " and default fallback disabled"); } + return defaultCodecDescriptor; } /** Returns the {@link CodecDescriptor} associated with the supplied tag. */ public CodecDescriptor getCodecDescriptorByTag(int tag) throws SerializationException.NoCodecException { - if (tag < 0 || tag > tagMappedCodecs.size()) { + int tagOffset = tag - 1; + if (tagOffset < 0 || tagOffset > tagMappedCodecs.size()) { throw new SerializationException.NoCodecException("No codec available for tag " + tag); } - CodecDescriptor result = tagMappedCodecs.get(tag); + CodecDescriptor result = tagMappedCodecs.get(tagOffset); if (result != null) { return result; } else { @@ -111,32 +124,86 @@ public class ObjectCodecRegistry { } /** Describes encoding logic. */ - static class CodecDescriptor { + static interface CodecDescriptor { + void serialize(SerializationContext context, Object obj, CodedOutputStream codedOut) + throws IOException, SerializationException; + + Object deserialize(DeserializationContext context, CodedInputStream codedIn) + throws IOException, SerializationException; + + /** + * Unique identifier identifying the associated codec. + * + * <p>Intended to be used as a compact on-the-wire representation of an encoded object's type. + * + * <p>Returns a value ≥ 1. + * + * <p>0 is a special tag representing null while negative numbers are reserved for + * backreferences. + */ + int getTag(); + + /** + * Returns the underlying codec. + * + * <p>For backwards compatibility. New callers should prefer the methods above. + */ + ObjectCodec<?> getCodec(); + } + + private static class TypedCodecDescriptor<T> implements CodecDescriptor { private final int tag; - private final ObjectCodec<?> codec; + private final ObjectCodec<T> codec; - private CodecDescriptor(int tag, ObjectCodec<?> codec) { + private TypedCodecDescriptor(int tag, ObjectCodec<T> codec) { this.tag = tag; this.codec = codec; } - /** - * Unique identifier identifying the associated codec. Intended to be used as a compact - * on-the-wire representation of an encoded object's type. - */ - int getTag() { + @Override + @SuppressWarnings("unchecked") + public void serialize(SerializationContext context, Object obj, CodedOutputStream codedOut) + throws IOException, SerializationException { + codec.serialize(context, (T) obj, codedOut); + } + + @Override + public T deserialize(DeserializationContext context, CodedInputStream codedIn) + throws IOException, SerializationException { + return codec.deserialize(context, codedIn); + } + + @Override + public int getTag() { return tag; } - ObjectCodec<?> getCodec() { + @Override + public ObjectCodec<T> getCodec() { return codec; } } + private interface CodecHolder { + CodecDescriptor createDescriptor(int tag); + } + + private static class TypedCodecHolder<T> implements CodecHolder { + private final ObjectCodec<T> codec; + + private TypedCodecHolder(ObjectCodec<T> codec) { + this.codec = codec; + } + + @Override + public CodecDescriptor createDescriptor(int tag) { + return new TypedCodecDescriptor<T>(tag, codec); + } + } + /** Builder for {@link ObjectCodecRegistry}. */ public static class Builder { - private final ImmutableMap.Builder<String, ObjectCodec<?>> codecsBuilder = - ImmutableMap.builder(); + private final ImmutableMap.Builder<String, CodecHolder> codecsBuilder = ImmutableMap.builder(); private boolean allowDefaultCodec = true; private Builder() {} @@ -147,8 +214,8 @@ public class ObjectCodecRegistry { * <p>Intended for package-internal usage only. Consider using the specialized build types * returned by {@link #asClassKeyedBuilder()} before using this method. */ - Builder add(String classifier, ObjectCodec<?> codec) { - codecsBuilder.put(classifier, codec); + <T> Builder add(String classifier, ObjectCodec<T> codec) { + codecsBuilder.put(classifier, new TypedCodecHolder<>(codec)); return this; } @@ -208,11 +275,11 @@ public class ObjectCodecRegistry { CodecDescriptor[] codecTable = new CodecDescriptor[codecs.size() + (defaultCodecDescriptor != null ? 1 : 0)]; for (Entry<String, CodecDescriptor> entry : codecs.entrySet()) { - codecTable[entry.getValue().getTag()] = entry.getValue(); + codecTable[entry.getValue().getTag() - 1] = entry.getValue(); } if (defaultCodecDescriptor != null) { - codecTable[defaultCodecDescriptor.getTag()] = defaultCodecDescriptor; + codecTable[defaultCodecDescriptor.getTag() - 1] = defaultCodecDescriptor; } return ImmutableList.copyOf(codecTable); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java index 7470fe495e..bb5a50492c 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializationContext.java @@ -16,17 +16,38 @@ package com.google.devtools.build.lib.skyframe.serialization; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.CodedOutputStream; +import java.io.IOException; /** Stateful class for providing additional context to a single serialization "session". */ // TODO(bazel-team): This class is just a shell, fill in. public class SerializationContext { + private final ObjectCodecRegistry registry; private final ImmutableMap<Class<?>, Object> dependencies; - public SerializationContext(ImmutableMap<Class<?>, Object> dependencies) { + public SerializationContext( + ObjectCodecRegistry registry, ImmutableMap<Class<?>, Object> dependencies) { + this.registry = registry; this.dependencies = dependencies; } + public SerializationContext(ImmutableMap<Class<?>, Object> dependencies) { + this(AutoRegistry.get(), dependencies); + } + + // TODO(shahan): consider making codedOut a member of this class. + public void serialize(Object object, CodedOutputStream codedOut) + throws IOException, SerializationException { + if (object == null) { + codedOut.writeSInt32NoTag(0); + return; + } + ObjectCodecRegistry.CodecDescriptor descriptor = registry.getCodecDescriptor(object.getClass()); + codedOut.writeSInt32NoTag(descriptor.getTag()); + descriptor.serialize(this, object, codedOut); + } + @SuppressWarnings("unchecked") public <T> T getDependency(Class<T> type) { Preconditions.checkNotNull(type); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java index 8476298243..0f597dbee1 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/AutoCodec.java @@ -25,11 +25,11 @@ import java.lang.annotation.Target; * <pre>{@code * @AutoCodec * class Target { - * public static final ObjectCodec<Target> CODEC = new Target_AutoCodec(); - * } * }</pre> * * The {@code _AutoCodec} suffix is added to the {@code Target} to obtain the generated class name. + * In the example, that results in a class named {@code Target_AutoCodec} but applications should + * not need to directly access the generated class. */ @Target(ElementType.TYPE) public @interface AutoCodec { @@ -46,7 +46,8 @@ public @interface AutoCodec { * * <ul> * <li>a designated constructor or factory method to inspect to generate the codec - * <li>the parameters must match member fields on name and type. + * <li>each parameter must match a member field on name and the field will be interpreted as + * an instance of the parameter type. * </ul> * * <p>If there is a unique constructor, @AutoCodec may select that as the default instantiator, @@ -65,13 +66,14 @@ public @interface AutoCodec { * * <p>Uses reflection to determine the concrete subclass, stores the name of the subclass and * uses its codec to serialize the data. + * + * <p>This is no longer needed and only adds useless overhead. */ + // TODO(shahan): delete this and all references to it POLYMORPHIC, /** * For use with classes that are singleton. * - * <p>Commonly used with the POLYMORPHIC strategy. - * * <p>The serialized class must have a codec accessible, static INSTANCE field. */ SINGLETON diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD index 91d46d58de..9c7bc1945e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/BUILD @@ -16,6 +16,8 @@ java_library( # Generated classes have the following dependencies. ":unsafe-provider", "//third_party/protobuf:protobuf_java", + "//src/main/java/com/google/devtools/build/lib/collect/nestedset", + "//src/main/java/com/google/devtools/build/lib/collect/nestedset:serialization", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/Marshallers.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/Marshallers.java index d1a04da8c3..bf844741da 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/Marshallers.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec/Marshallers.java @@ -28,7 +28,6 @@ import com.google.common.collect.Maps; import com.google.common.hash.HashCode; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetCodec; -import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationCodeGenerator.Context; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationCodeGenerator.Marshaller; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationCodeGenerator.PrimitiveValueSerializationCodeGenerator; @@ -45,9 +44,7 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.regex.Pattern; import javax.annotation.processing.ProcessingEnvironment; -import javax.lang.model.element.Element; import javax.lang.model.element.ElementKind; -import javax.lang.model.element.Modifier; import javax.lang.model.element.TypeElement; import javax.lang.model.type.ArrayType; import javax.lang.model.type.DeclaredType; @@ -64,12 +61,14 @@ class Marshallers { } void writeSerializationCode(Context context) { - if (context.canBeNull()) { + SerializationCodeGenerator generator = getMatchingCodeGenerator(context.type); + boolean needsNullHandling = context.canBeNull() && generator != contextMarshaller; + if (needsNullHandling) { context.builder.beginControlFlow("if ($L != null)", context.name); context.builder.addStatement("codedOut.writeBoolNoTag(true)"); } - getMatchingCodeGenerator(context.type).addSerializationCode(context); - if (context.canBeNull()) { + generator.addSerializationCode(context); + if (needsNullHandling) { context.builder.nextControlFlow("else"); context.builder.addStatement("codedOut.writeBoolNoTag(false)"); context.builder.endControlFlow(); @@ -77,14 +76,16 @@ class Marshallers { } void writeDeserializationCode(Context context) { - if (context.canBeNull()) { + SerializationCodeGenerator generator = getMatchingCodeGenerator(context.type); + boolean needsNullHandling = context.canBeNull() && generator != contextMarshaller; + if (needsNullHandling) { context.builder.addStatement("$T $L = null", context.getTypeName(), context.name); context.builder.beginControlFlow("if (codedIn.readBool())"); } else { context.builder.addStatement("$T $L", context.getTypeName(), context.name); } - getMatchingCodeGenerator(context.type).addDeserializationCode(context); - if (requiresNullityCheck(context)) { + generator.addDeserializationCode(context); + if (needsNullHandling) { context.builder.endControlFlow(); } } @@ -108,10 +109,6 @@ class Marshallers { context.builder.addStatement("$L = $L.build()", context.name, builderName); } - private static boolean requiresNullityCheck(Context context) { - return !(context.type instanceof PrimitiveType); - } - private SerializationCodeGenerator getMatchingCodeGenerator(TypeMirror type) { if (type.getKind() == TypeKind.ARRAY) { return arrayCodeGenerator; @@ -814,27 +811,13 @@ class Marshallers { private void addSerializationCodeForNestedSet(Context context) { TypeMirror typeParameter = context.getDeclaredType().getTypeArguments().get(0); - String nestedSetCodec = context.makeName("nestedSetCodec"); - Optional<? extends Element> typeParameterCodec = - Optional.fromJavaUtil(getCodec((DeclaredType) typeParameter)); - if (!typeParameterCodec.isPresent()) { - // AutoCodec can only serialize NestedSets of declared types. However, this code must - // be generated for Iterables of non-declared types (e.g. String), since Iterable - // serialization involves a runtime check for NestedSet. In this case, throw on the unused - // NestedSet branch. - context.builder.addStatement( - "throw new $T(\"NestedSet<$T> is not supported in AutoCodec\")", - AssertionError.class, - typeParameter); - return; - } + String nestedSetCodec = context.makeName("nestedSetCodec"); context.builder.addStatement( - "$T<$T> $L = new $T<>($T.CODEC)", + "$T<$T> $L = new $T<>()", NestedSetCodec.class, typeParameter, nestedSetCodec, - NestedSetCodec.class, - typeParameter); + NestedSetCodec.class); context.builder.addStatement( "$L.serialize(context, ($T<$T>) $L, codedOut)", nestedSetCodec, @@ -846,26 +829,12 @@ class Marshallers { private void addDeserializationCodeForNestedSet(Context context) { TypeMirror typeParameter = context.getDeclaredType().getTypeArguments().get(0); String nestedSetCodec = context.makeName("nestedSetCodec"); - Optional<? extends Element> typeParameterCodec = - Optional.fromJavaUtil(getCodec((DeclaredType) typeParameter)); - if (!typeParameterCodec.isPresent()) { - // AutoCodec can only serialize NestedSets of declared types. However, this code must - // be generated for Iterables of non-declared types (e.g. String), since Iterable - // serialization involves a runtime check for NestedSet. In this case, we throw on the unused - // NestedSet branch. - context.builder.addStatement( - "throw new $T(\"NestedSet<$T> is not supported in AutoCodec\")", - AssertionError.class, - typeParameter); - return; - } context.builder.addStatement( - "$T<$T> $L = new $T<>($T.CODEC)", + "$T<$T> $L = new $T<>()", NestedSetCodec.class, typeParameter, nestedSetCodec, - NestedSetCodec.class, - typeParameter); + NestedSetCodec.class); context.builder.addStatement( "$L = $L.deserialize(context, codedIn)", context.name, nestedSetCodec); } @@ -893,39 +862,22 @@ class Marshallers { } }; - private final Marshaller codecMarshaller = + /** Delegates marshalling back to the context. */ + private final Marshaller contextMarshaller = new Marshaller() { @Override - public boolean matches(DeclaredType type) { - return getCodec(type).isPresent(); + public boolean matches(DeclaredType unusedType) { + return true; } @Override public void addSerializationCode(Context context) { - TypeMirror codecType = getCodec(context.getDeclaredType()).get().asType(); - if (isSubtypeErased(codecType, ObjectCodec.class)) { - context.builder.addStatement( - "$T.CODEC.serialize(context, $L, codedOut)", context.getTypeName(), context.name); - } else { - throw new IllegalArgumentException( - "CODEC field of " - + ((TypeElement) context.getDeclaredType().asElement()).getQualifiedName() - + " is not ObjectCodec"); - } + context.builder.addStatement("context.serialize($L, codedOut)", context.name); } @Override public void addDeserializationCode(Context context) { - TypeMirror codecType = getCodec(context.getDeclaredType()).get().asType(); - if (isSubtypeErased(codecType, ObjectCodec.class)) { - context.builder.addStatement( - "$L = $T.CODEC.deserialize(context, codedIn)", context.name, context.getTypeName()); - } else { - throw new IllegalArgumentException( - "CODEC field of " - + ((TypeElement) context.getDeclaredType().asElement()).getQualifiedName() - + " is neither ObjectCodec"); - } + context.builder.addStatement("$L = context.deserialize(codedIn)", context.name); } }; @@ -952,8 +904,8 @@ class Marshallers { patternMarshaller, hashCodeMarshaller, protoMarshaller, - codecMarshaller, - iterableMarshaller); + iterableMarshaller, + contextMarshaller); /** True when {@code type} has the same type as {@code clazz}. */ private boolean matchesType(TypeMirror type, Class<?> clazz) { @@ -971,24 +923,8 @@ class Marshallers { .isSameType(env.getTypeUtils().erasure(type), env.getTypeUtils().erasure(getType(clazz))); } - /** True when erasure of {@code type} is a subtype of the erasure of {@code clazz}. */ - private boolean isSubtypeErased(TypeMirror type, Class<?> clazz) { - return env.getTypeUtils() - .isSubtype(env.getTypeUtils().erasure(type), env.getTypeUtils().erasure(getType(clazz))); - } - /** Returns the TypeMirror corresponding to {@code clazz}. */ private TypeMirror getType(Class<?> clazz) { return env.getElementUtils().getTypeElement((clazz.getCanonicalName())).asType(); } - - private static java.util.Optional<? extends Element> getCodec(DeclaredType type) { - return type.asElement() - .getEnclosedElements() - .stream() - .filter(t -> t.getModifiers().contains(Modifier.STATIC)) - .filter(t -> t.getSimpleName().contentEquals("CODEC")) - .filter(t -> t.getKind() == ElementKind.FIELD) - .findAny(); - } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/SerializationTester.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/SerializationTester.java new file mode 100644 index 0000000000..5d561ad7c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/SerializationTester.java @@ -0,0 +1,155 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization.testutils; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assert_; + +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.skyframe.serialization.DeserializationContext; +import com.google.devtools.build.lib.skyframe.serialization.SerializationContext; +import com.google.devtools.build.lib.skyframe.serialization.SerializationException; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Random; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility for testing serialization of given subjects. + * + * <p>Differs from {@link ObjectCodecTester} in that this uses the context to perform serialization + * deserialization instead of a specific codec. + */ +public class SerializationTester { + public static final int DEFAULT_JUNK_INPUTS = 20; + public static final int JUNK_LENGTH_UPPER_BOUND = 20; + + private static final Logger logger = Logger.getLogger(SerializationTester.class.getName()); + + /** Interface for testing successful deserialization of an object. */ + @FunctionalInterface + public interface VerificationFunction<T> { + /** + * Verifies that the original object was sufficiently serialized/deserialized. + * + * <p><i>Must</i> throw an exception on failure. + */ + void verifyDeserialized(T original, T deserialized) throws Exception; + } + + private final ImmutableList<Object> subjects; + private Supplier<SerializationContext> writeContextFactory = + () -> new SerializationContext(ImmutableMap.of()); + private Supplier<DeserializationContext> readContextFactory = + () -> new DeserializationContext(ImmutableMap.of()); + + @SuppressWarnings("rawtypes") + private VerificationFunction verificationFunction = + (original, deserialized) -> assertThat(deserialized).isEqualTo(original); + + public SerializationTester(Object... subjects) { + Preconditions.checkArgument(subjects.length > 0); + this.subjects = ImmutableList.copyOf(subjects); + } + + public SerializationTester setWriteContextFactory( + Supplier<SerializationContext> writeContextFactory) { + this.writeContextFactory = writeContextFactory; + return this; + } + + public SerializationTester setReadContextFactory( + Supplier<DeserializationContext> readContextFactory) { + this.readContextFactory = readContextFactory; + return this; + } + + @SuppressWarnings("rawtypes") + public SerializationTester setVerificationFunction(VerificationFunction verificationFunction) { + this.verificationFunction = verificationFunction; + return this; + } + + public void runTests() throws Exception { + testSerializeDeserialize(); + testStableSerialization(); + testDeserializeJunkData(); + } + + /** Runs serialization/deserialization tests. */ + private void testSerializeDeserialize() throws Exception { + Stopwatch timer = Stopwatch.createStarted(); + int totalBytes = 0; + for (Object subject : subjects) { + byte[] serialized = toBytes(subject); + totalBytes += serialized.length; + Object deserialized = fromBytes(serialized); + verificationFunction.verifyDeserialized(subject, deserialized); + } + logger.log( + Level.INFO, + subjects.get(0).getClass().getSimpleName() + + " total serialized bytes = " + + totalBytes + + ", " + + timer); + } + + /** Runs serialized bytes stability tests. */ + private void testStableSerialization() throws IOException, SerializationException { + for (Object subject : subjects) { + byte[] serialized = toBytes(subject); + Object deserialized = fromBytes(serialized); + byte[] reserialized = toBytes(deserialized); + assertThat(reserialized).isEqualTo(serialized); + } + } + + /** Runs junk-data recognition tests. */ + private void testDeserializeJunkData() { + Random rng = new Random(0); + for (int i = 0; i < DEFAULT_JUNK_INPUTS; ++i) { + byte[] junkData = new byte[rng.nextInt(JUNK_LENGTH_UPPER_BOUND)]; + rng.nextBytes(junkData); + try { + readContextFactory.get().deserialize(CodedInputStream.newInstance(junkData)); + // OK. Junk string was coincidentally parsed. + } catch (IOException | SerializationException e) { + // OK. Deserialization of junk failed. + return; + } + } + assert_().fail("all junk was parsed successfully"); + } + + private Object fromBytes(byte[] bytes) throws IOException, SerializationException { + return readContextFactory.get().deserialize(CodedInputStream.newInstance(bytes)); + } + + private byte[] toBytes(Object subject) throws IOException, SerializationException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + CodedOutputStream codedOut = CodedOutputStream.newInstance(bytes); + writeContextFactory.get().serialize(subject, codedOut); + codedOut.flush(); + return bytes.toByteArray(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/TestUtils.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/TestUtils.java index 2c729d4671..538d35f084 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/TestUtils.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/TestUtils.java @@ -19,7 +19,6 @@ import static com.google.common.truth.Truth.assertThat; import com.google.devtools.build.lib.skyframe.serialization.CodecRegisterer; import com.google.devtools.build.lib.skyframe.serialization.DeserializationContext; import com.google.devtools.build.lib.skyframe.serialization.ObjectCodec; -import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecRegistry; import com.google.devtools.build.lib.skyframe.serialization.SerializationContext; import com.google.devtools.build.lib.skyframe.serialization.SerializationException; import com.google.devtools.build.lib.skyframe.serialization.strings.StringCodecs; @@ -94,9 +93,7 @@ public class TestUtils { } /** Disables auto-registration of ConstantStringCodec. */ - static class ConstantStringCodecRegisterer implements CodecRegisterer<ConstantStringCodec> { - @Override - public void register(ObjectCodecRegistry.Builder unusedBuilder) {} - } + private static class ConstantStringCodecRegisterer + implements CodecRegisterer<ConstantStringCodec> {} } } diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Root.java b/src/main/java/com/google/devtools/build/lib/vfs/Root.java index 6f090e41a7..c23dbeb7d0 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Root.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/Root.java @@ -33,37 +33,37 @@ import javax.annotation.Nullable; * <p>A typical root could be the exec path, a package root, or an output root specific to some * configuration. We also support absolute roots for non-hermetic paths outside the user workspace. */ -public interface Root extends Comparable<Root>, Serializable { +public abstract class Root implements Comparable<Root>, Serializable { - ObjectCodec<Root> CODEC = new RootCodec(); + public static final ObjectCodec<Root> CODEC = new RootCodec(); /** Constructs a root from a path. */ - static Root fromPath(Path path) { + public static Root fromPath(Path path) { return new PathRoot(path); } /** Returns an absolute root. Can only be used with absolute path fragments. */ - static Root absoluteRoot(FileSystem fileSystem) { + public static Root absoluteRoot(FileSystem fileSystem) { return fileSystem.getAbsoluteRoot(); } /** Returns a path by concatenating the root and the root-relative path. */ - Path getRelative(PathFragment rootRelativePath); + public abstract Path getRelative(PathFragment rootRelativePath); /** Returns a path by concatenating the root and the root-relative path. */ - Path getRelative(String rootRelativePath); + public abstract Path getRelative(String rootRelativePath); /** Returns the relative path between the root and the given path. */ - PathFragment relativize(Path path); + public abstract PathFragment relativize(Path path); /** Returns the relative path between the root and the given absolute path fragment. */ - PathFragment relativize(PathFragment absolutePathFragment); + public abstract PathFragment relativize(PathFragment absolutePathFragment); /** Returns whether the given path is under this root. */ - boolean contains(Path path); + public abstract boolean contains(Path path); /** Returns whether the given absolute path fragment is under this root. */ - boolean contains(PathFragment absolutePathFragment); + public abstract boolean contains(PathFragment absolutePathFragment); /** * Returns the underlying path. Please avoid using this method. @@ -71,12 +71,12 @@ public interface Root extends Comparable<Root>, Serializable { * <p>Not all roots are backed by paths, so this may return null. */ @Nullable - Path asPath(); + public abstract Path asPath(); - boolean isAbsolute(); + public abstract boolean isAbsolute(); /** Implementation of Root that is backed by a {@link Path}. */ - final class PathRoot implements Root { + public static final class PathRoot extends Root { private final Path path; PathRoot(Path path) { @@ -160,7 +160,7 @@ public interface Root extends Comparable<Root>, Serializable { } /** An absolute root of a file system. Can only resolve absolute path fragments. */ - final class AbsoluteRoot implements Root { + public static final class AbsoluteRoot extends Root { private FileSystem fileSystem; // Non-final for serialization AbsoluteRoot(FileSystem fileSystem) { @@ -258,7 +258,7 @@ public interface Root extends Comparable<Root>, Serializable { } /** Codec to serialize {@link Root}s. */ - class RootCodec implements ObjectCodec<Root> { + private static class RootCodec implements ObjectCodec<Root> { @Override public Class<Root> getEncodedClass() { return Root.class; @@ -272,7 +272,7 @@ public interface Root extends Comparable<Root>, Serializable { throws SerializationException, IOException { if (obj instanceof PathRoot) { codedOut.writeBoolNoTag(false); - Path.CODEC.serialize(context, ((PathRoot) obj).path, codedOut); + context.serialize(((PathRoot) obj).path, codedOut); } else if (obj instanceof AbsoluteRoot) { Preconditions.checkArgument( ((AbsoluteRoot) obj).fileSystem == context.getDependency(FileSystem.class)); @@ -289,7 +289,7 @@ public interface Root extends Comparable<Root>, Serializable { if (isAbsolute) { return context.getDependency(FileSystem.class).getAbsoluteRoot(); } else { - Path path = Path.CODEC.deserialize(context, codedIn); + Path path = context.deserialize(codedIn); return new PathRoot(path); } } |