diff options
author | shahan <shahan@google.com> | 2018-01-24 15:12:39 -0800 |
---|---|---|
committer | Copybara-Service <copybara-piper@google.com> | 2018-01-24 15:14:16 -0800 |
commit | 9bfe87f29cd7808dc3086a5f933936704ecdc199 (patch) | |
tree | 27aeeccb45fb5f8cc6ace0d7ef5c6ec503807257 | |
parent | 146c2457808318191b7afd3424170fea13eff28b (diff) |
SerializerAdapter
Allows ObjectCodec to be registered as a Kryo Serializer.
PiperOrigin-RevId: 183149177
4 files changed, 276 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD index 5fd16ef4cc..6fc533c0a8 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD @@ -12,9 +12,23 @@ java_library( name = "serialization", srcs = glob(["**/*.java"]), deps = [ + ":kryo", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:guava", "//third_party:jsr305", "//third_party/protobuf:protobuf_java", ], ) + +# A convenience library that bundles together kryo and its runtime dependencies. +java_library( + name = "kryo", + exports = [ + "//third_party:kryo", + "//third_party:objenesis", + ], + runtime_deps = [ + "//third_party:minlog", + "//third_party:reflectasm", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializerAdapter.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializerAdapter.java new file mode 100644 index 0000000000..00004075f3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/SerializerAdapter.java @@ -0,0 +1,59 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoException; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** Converts an {@link ObjectCodec} into a Kryo {@link Serializer}. */ +public class SerializerAdapter<T> extends Serializer<T> { + private final ObjectCodec<T> codec; + + public SerializerAdapter(ObjectCodec<T> codec) { + this.codec = codec; + } + + @Override + public void write(Kryo kryo, Output output, T object) { + try { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + CodedOutputStream codedOut = CodedOutputStream.newInstance(byteOutput); + codec.serialize(object, codedOut); + codedOut.flush(); + byte[] byteData = byteOutput.toByteArray(); + output.writeInt(byteData.length, true); + output.writeBytes(byteData); + } catch (SerializationException | IOException e) { + throw new KryoException(e); + } + } + + @Override + public T read(Kryo kryo, Input input, Class<T> unusedClass) { + try { + byte[] byteData = input.readBytes(input.readInt(true)); + return codec.deserialize(CodedInputStream.newInstance(byteData)); + } catch (SerializationException | IOException e) { + throw new KryoException(e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/BUILD index d193666032..ea7cf9b9bb 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/BUILD @@ -12,6 +12,7 @@ java_library( deps = [ "//src/main/java/com/google/devtools/build/lib:syntax", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization:kryo", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/SerializerTester.java b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/SerializerTester.java new file mode 100644 index 0000000000..e4fd2d1748 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils/SerializerTester.java @@ -0,0 +1,202 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe.serialization.testutils; + +import static com.google.common.truth.Truth.assertThat; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.KryoException; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.UnsafeInput; +import com.esotericsoftware.kryo.io.UnsafeOutput; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.io.ByteArrayOutputStream; +import java.util.Random; +import org.objenesis.instantiator.ObjectInstantiator; + +/** Utility for testing {@link Serializer} instances. */ +public class SerializerTester<T> { + public static final int DEFAULT_JUNK_INPUTS = 20; + public static final int JUNK_LENGTH_UPPER_BOUND = 20; + + /** Interface for testing successful deserialization of an object. */ + @FunctionalInterface + public interface VerificationFunction<T> { + /** + * Verifies whether or not the original object was sufficiently serialized/deserialized. + * + * @throws Exception on verification failure + */ + void verifyDeserialized(T original, T deserialized) throws Exception; + } + + /** + * Creates a {@link SerializerTester.Builder} for the given class. + * + * <p>See {@link SerializerTester.Builder} for details. + */ + public static <T> SerializerTester.Builder<T> newBuilder(Class<T> type) { + return new SerializerTester.Builder<>(type); + } + + private final Class<T> type; + private final Kryo kryo; + private final ImmutableList<T> subjects; + private final VerificationFunction<T> verificationFunction; + + private SerializerTester( + Class<T> type, + Kryo kryo, + ImmutableList<T> subjects, + VerificationFunction<T> verificationFunction) { + this.type = type; + this.kryo = kryo; + Preconditions.checkState(!subjects.isEmpty(), "No subjects provided"); + this.subjects = subjects; + this.verificationFunction = verificationFunction; + } + + private void runTests() throws Exception { + testSerializeDeserialize(); + testStableSerialization(); + testDeserializeJunkData(); + } + + /** Runs serialization/deserialization tests. */ + void testSerializeDeserialize() throws Exception { + for (T subject : subjects) { + byte[] serialized = toBytes(subject); + T deserialized = fromBytes(serialized); + verificationFunction.verifyDeserialized(subject, deserialized); + } + } + + /** Runs serialized bytes stability tests. */ + void testStableSerialization() throws Exception { + for (T subject : subjects) { + byte[] serialized = toBytes(subject); + T deserialized = fromBytes(serialized); + byte[] reserialized = toBytes(deserialized); + assertThat(reserialized).isEqualTo(serialized); + } + } + + /** + * Junk tolerance test. + * + * <p>Verifies that the Serializer only throws KryoException or IndexOutOfBoundsException. + * + * <p>TODO(shahan): Allowing IndexOutOfBoundsException here is not ideal, but Kryo itself encodes + * lengths in the stream and seeking to random lengths triggers this. + */ + void testDeserializeJunkData() { + Random rng = new Random(0); + int numFailures = 0; + for (int i = 0; i < DEFAULT_JUNK_INPUTS; ++i) { + byte[] junkData = new byte[rng.nextInt(JUNK_LENGTH_UPPER_BOUND)]; + rng.nextBytes(junkData); + try { + UnsafeInput input = new UnsafeInput(junkData); + kryo.readObject(input, type); + // OK. Junk string was coincidentally parsed. + } catch (IndexOutOfBoundsException | KryoException e) { + // OK. Deserialization of junk failed. + ++numFailures; + } + } + assertThat(numFailures).isAtLeast(1); + } + + private T fromBytes(byte[] bytes) { + return kryo.readObject(new UnsafeInput(bytes), type); + } + + private byte[] toBytes(T subject) { + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + UnsafeOutput out = new UnsafeOutput(byteOut); + kryo.writeObject(out, subject); + out.flush(); + return byteOut.toByteArray(); + } + + /** Builder for {@link SerializerTester}. */ + public static class Builder<T> { + private final Class<T> type; + private final Kryo kryo; + private final ImmutableList.Builder<T> subjectsBuilder = ImmutableList.builder(); + private VerificationFunction<T> verificationFunction = + (original, deserialized) -> assertThat(deserialized).isEqualTo(original); + + private Builder(Class<T> type) { + this.type = type; + this.kryo = new Kryo(); + kryo.setRegistrationRequired(true); + } + + public <X> Builder<T> register(Class<X> type, Serializer<X> serializer) { + kryo.register(type, serializer); + return this; + } + + public <X> Builder<T> register(Class<X> type) { + kryo.register(type); + return this; + } + + public <X> Builder<T> register(Class<X> type, ObjectInstantiator instantiator) { + kryo.register(type).setInstantiator(instantiator); + return this; + } + + /** Adds subjects to be tested for serialization/deserialization. */ + @SafeVarargs + public final Builder<T> addSubjects(@SuppressWarnings("unchecked") T... subjects) { + return addSubjects(ImmutableList.copyOf(subjects)); + } + + /** Adds subjects to be tested for serialization/deserialization. */ + public Builder<T> addSubjects(ImmutableList<T> subjects) { + subjectsBuilder.addAll(subjects); + return this; + } + + /** + * Sets {@link SerializerTester.VerificationFunction} for verifying deserialization. + * + * <p>Default is simple equality assertion, a custom version may be provided for more, or less, + * detailed checks. + */ + public Builder<T> setVerificationFunction(VerificationFunction<T> verificationFunction) { + this.verificationFunction = Preconditions.checkNotNull(verificationFunction); + return this; + } + + public Builder<T> setRegistrationRequired(boolean isRequired) { + kryo.setRegistrationRequired(isRequired); + return this; + } + + /** Captures the state of this builder and runs all associated tests. */ + public void buildAndRunTests() throws Exception { + build().runTests(); + } + + /** Creates a new {@link SerializerTester} from this builder. */ + private SerializerTester<T> build() { + return new SerializerTester<>(type, kryo, subjectsBuilder.build(), verificationFunction); + } + } +} |