From d36c0c538a545fac5d9db6ba65c525246d4efa95 Mon Sep 17 00:00:00 2001 From: Feng Xiao Date: Wed, 29 Mar 2017 14:32:48 -0700 Subject: Down-integrate from google3. --- .../java/com/google/protobuf/AbstractMessage.java | 4 +- .../com/google/protobuf/AbstractMessageLite.java | 21 +- .../java/com/google/protobuf/AbstractParser.java | 26 +- .../java/com/google/protobuf/BooleanArrayList.java | 12 +- .../java/com/google/protobuf/CodedInputStream.java | 4 +- .../com/google/protobuf/CodedOutputStream.java | 4 +- .../main/java/com/google/protobuf/Descriptors.java | 7 +- .../java/com/google/protobuf/DoubleArrayList.java | 12 +- .../java/com/google/protobuf/DynamicMessage.java | 7 +- .../main/java/com/google/protobuf/FieldSet.java | 7 +- .../java/com/google/protobuf/FloatArrayList.java | 12 +- .../com/google/protobuf/GeneratedMessageLite.java | 22 +- .../com/google/protobuf/GeneratedMessageV3.java | 2 +- .../java/com/google/protobuf/IntArrayList.java | 12 +- .../main/java/com/google/protobuf/Internal.java | 15 + .../java/com/google/protobuf/LazyFieldLite.java | 1 + .../java/com/google/protobuf/LongArrayList.java | 12 +- .../main/java/com/google/protobuf/MapEntry.java | 24 +- .../java/com/google/protobuf/MapEntryLite.java | 5 + .../main/java/com/google/protobuf/MapField.java | 8 + .../java/com/google/protobuf/MapFieldLite.java | 14 +- .../src/main/java/com/google/protobuf/Parser.java | 13 + .../protobuf/PrimitiveNonBoxingCollection.java | 34 ++ .../google/protobuf/RepeatedFieldBuilderV3.java | 18 +- .../com/google/protobuf/SingleFieldBuilderV3.java | 12 +- .../main/java/com/google/protobuf/TextFormat.java | 4 +- .../java/com/google/protobuf/UnknownFieldSet.java | 4 +- .../main/java/com/google/protobuf/UnsafeUtil.java | 401 ++++++++++++++++----- .../src/main/java/com/google/protobuf/Utf8.java | 2 +- .../java/com/google/protobuf/LazyFieldTest.java | 2 +- .../test/java/com/google/protobuf/LiteTest.java | 108 ++++++ .../java/com/google/protobuf/MapForProto2Test.java | 1 + .../src/test/java/com/google/protobuf/MapTest.java | 39 +- .../test/java/com/google/protobuf/ParserTest.java | 27 +- .../test/java/com/google/protobuf/TestUtil.java | 2 + .../java/com/google/protobuf/TextFormatTest.java | 92 +++++ .../com/google/protobuf/lite_equals_and_hash.proto | 8 + java/lite/pom.xml | 1 + .../java/com/google/protobuf/util/JsonFormat.java | 175 +++++---- .../java/com/google/protobuf/util/Timestamps.java | 2 +- .../com/google/protobuf/util/JsonFormatTest.java | 137 ++++++- 41 files changed, 1027 insertions(+), 286 deletions(-) create mode 100644 java/core/src/main/java/com/google/protobuf/PrimitiveNonBoxingCollection.java (limited to 'java') diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java index 37180da8..b5043eb5 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java @@ -34,7 +34,6 @@ import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.Internal.EnumLite; - import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -328,7 +327,8 @@ public abstract class AbstractMessage extends AbstractMessageLite.Builder implements Message.Builder { // The compiler produces an error if this is not declared explicitly. - // Method isn't abstract to bypass Java 1.6 compiler issue http://bugs.java.com/view_bug.do?bug_id=6908259 + // Method isn't abstract to bypass Java 1.6 compiler issue: + // http://bugs.java.com/view_bug.do?bug_id=6908259 @Override public BuilderType clone() { throw new UnsupportedOperationException("clone() should be implemented in subclasses."); diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java index 4f691dfd..99787fcc 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; @@ -351,22 +353,23 @@ public abstract class AbstractMessageLite< */ protected static void addAll(final Iterable values, final Collection list) { - if (values == null) { - throw new NullPointerException(); - } + checkNotNull(values); if (values instanceof LazyStringList) { // For StringOrByteStringLists, check the underlying elements to avoid // forcing conversions of ByteStrings to Strings. + // TODO(dweis): Could we just prohibit nulls in all protobuf lists and get rid of this? Is + // if even possible to hit this condition as all protobuf methods check for null first, + // right? checkForNullValues(((LazyStringList) values).getUnderlyingElements()); list.addAll((Collection) values); } else if (values instanceof Collection) { - checkForNullValues(values); + if (!(values instanceof PrimitiveNonBoxingCollection)) { + checkForNullValues(values); + } list.addAll((Collection) values); } else { for (final T value : values) { - if (value == null) { - throw new NullPointerException(); - } + checkNotNull(value); list.add(value); } } @@ -374,9 +377,7 @@ public abstract class AbstractMessageLite< private static void checkForNullValues(final Iterable values) { for (final Object value : values) { - if (value == null) { - throw new NullPointerException(); - } + checkNotNull(value); } } } diff --git a/java/core/src/main/java/com/google/protobuf/AbstractParser.java b/java/core/src/main/java/com/google/protobuf/AbstractParser.java index 7ff73ba4..ba570e3d 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractParser.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractParser.java @@ -31,9 +31,9 @@ package com.google.protobuf; import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream; - import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; /** * A partial implementation of the {@link Parser} interface which implements @@ -130,6 +130,30 @@ public abstract class AbstractParser return parseFrom(data, EMPTY_REGISTRY); } + @Override + public MessageType parseFrom(ByteBuffer data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + MessageType message; + try { + CodedInputStream input = CodedInputStream.newInstance(data); + message = parsePartialFrom(input, extensionRegistry); + try { + input.checkLastTagWas(0); + } catch (InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(message); + } + } catch (InvalidProtocolBufferException e) { + throw e; + } + + return checkMessageInitialized(message); + } + + @Override + public MessageType parseFrom(ByteBuffer data) throws InvalidProtocolBufferException { + return parseFrom(data, EMPTY_REGISTRY); + } + @Override public MessageType parsePartialFrom( byte[] data, int off, int len, ExtensionRegistryLite extensionRegistry) diff --git a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java index 0d9f87ba..fd4c142b 100644 --- a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import com.google.protobuf.Internal.BooleanList; +import static com.google.protobuf.Internal.checkNotNull; +import com.google.protobuf.Internal.BooleanList; import java.util.Arrays; import java.util.Collection; import java.util.RandomAccess; @@ -41,9 +42,8 @@ import java.util.RandomAccess; * * @author dweis@google.com (Daniel Weis) */ -final class BooleanArrayList - extends AbstractProtobufList - implements BooleanList, RandomAccess { +final class BooleanArrayList extends AbstractProtobufList + implements BooleanList, RandomAccess, PrimitiveNonBoxingCollection { private static final BooleanArrayList EMPTY_LIST = new BooleanArrayList(); static { @@ -198,9 +198,7 @@ final class BooleanArrayList public boolean addAll(Collection collection) { ensureIsMutable(); - if (collection == null) { - throw new NullPointerException(); - } + checkNotNull(collection); // We specialize when adding another BooleanArrayList to avoid boxing elements. if (!(collection instanceof BooleanArrayList)) { diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index 239798e4..3dfbcb0a 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -355,8 +355,8 @@ public abstract class CodedInputStream { *

Set the maximum message size. In order to prevent malicious messages from exhausting memory * or causing integer overflows, {@code CodedInputStream} limits how large a message may be. The * default limit is {@code Integer.MAX_INT}. You should set this limit as small as you can without - * harming your app's functionality. Note that size limits only apply when reading from an - * {@code InputStream}, not when constructed around a raw byte array. + * harming your app's functionality. Note that size limits only apply when reading from an {@code + * InputStream}, not when constructed around a raw byte array. * *

If you want to read several messages from a single CodedInputStream, you could call {@link * #resetSizeCounter()} after each one to avoid hitting the size limit. diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index 3e32c2c5..da0e9b12 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -184,7 +184,7 @@ public abstract class CodedOutputStream extends ByteOutput { * maps are sorted on the lexicographical order of the UTF8 encoded keys. * */ - void useDeterministicSerialization() { + public void useDeterministicSerialization() { serializationDeterministic = true; } @@ -1854,7 +1854,7 @@ public abstract class CodedOutputStream extends ByteOutput { } static boolean isSupported() { - return UnsafeUtil.hasUnsafeByteBufferOperations(); + return UnsafeUtil.hasUnsafeByteBufferOperations() && UnsafeUtil.hasUnsafeCopyMemory(); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/Descriptors.java b/java/core/src/main/java/com/google/protobuf/Descriptors.java index 38346f15..75b16fe3 100644 --- a/java/core/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/core/src/main/java/com/google/protobuf/Descriptors.java @@ -30,9 +30,10 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import com.google.protobuf.DescriptorProtos.*; import com.google.protobuf.Descriptors.FileDescriptor.Syntax; - import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Arrays; @@ -682,9 +683,7 @@ public final class Descriptors { /** Determines if the given field name is reserved. */ public boolean isReservedName(final String name) { - if (name == null) { - throw new NullPointerException(); - } + checkNotNull(name); for (final String reservedName : proto.getReservedNameList()) { if (reservedName.equals(name)) { return true; diff --git a/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java b/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java index 6177f3ca..867b85ce 100644 --- a/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import com.google.protobuf.Internal.DoubleList; +import static com.google.protobuf.Internal.checkNotNull; +import com.google.protobuf.Internal.DoubleList; import java.util.Arrays; import java.util.Collection; import java.util.RandomAccess; @@ -41,9 +42,8 @@ import java.util.RandomAccess; * * @author dweis@google.com (Daniel Weis) */ -final class DoubleArrayList - extends AbstractProtobufList - implements DoubleList, RandomAccess { +final class DoubleArrayList extends AbstractProtobufList + implements DoubleList, RandomAccess, PrimitiveNonBoxingCollection { private static final DoubleArrayList EMPTY_LIST = new DoubleArrayList(); static { @@ -199,9 +199,7 @@ final class DoubleArrayList public boolean addAll(Collection collection) { ensureIsMutable(); - if (collection == null) { - throw new NullPointerException(); - } + checkNotNull(collection); // We specialize when adding another DoubleArrayList to avoid boxing elements. if (!(collection instanceof DoubleArrayList)) { diff --git a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java index e6358c3b..2631db74 100644 --- a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java +++ b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java @@ -30,11 +30,12 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; - import java.io.IOException; import java.io.InputStream; import java.util.Collections; @@ -631,9 +632,7 @@ public final class DynamicMessage extends AbstractMessage { /** Verifies that the value is EnumValueDescriptor and matches Enum Type. */ private void ensureSingularEnumValueDescriptor( FieldDescriptor field, Object value) { - if (value == null) { - throw new NullPointerException(); - } + checkNotNull(value); if (!(value instanceof EnumValueDescriptor)) { throw new IllegalArgumentException( "DynamicMessage should use EnumValueDescriptor to set Enum Value."); diff --git a/java/core/src/main/java/com/google/protobuf/FieldSet.java b/java/core/src/main/java/com/google/protobuf/FieldSet.java index a828f30e..8a9239ed 100644 --- a/java/core/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/FieldSet.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import com.google.protobuf.LazyField.LazyIterator; +import static com.google.protobuf.Internal.checkNotNull; +import com.google.protobuf.LazyField.LazyIterator; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -385,9 +386,7 @@ final class FieldSet - implements FloatList, RandomAccess { +final class FloatArrayList extends AbstractProtobufList + implements FloatList, RandomAccess, PrimitiveNonBoxingCollection { private static final FloatArrayList EMPTY_LIST = new FloatArrayList(); static { @@ -198,9 +198,7 @@ final class FloatArrayList public boolean addAll(Collection collection) { ensureIsMutable(); - if (collection == null) { - throw new NullPointerException(); - } + checkNotNull(collection); // We specialize when adding another FloatArrayList to avoid boxing elements. if (!(collection instanceof FloatArrayList)) { diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 2d7fd334..cf3486e1 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -34,6 +34,7 @@ import com.google.protobuf.AbstractMessageLite.Builder.LimitedInputStream; import com.google.protobuf.GeneratedMessageLite.EqualsVisitor.NotEqualsException; import com.google.protobuf.Internal.BooleanList; import com.google.protobuf.Internal.DoubleList; +import com.google.protobuf.Internal.EnumLiteMap; import com.google.protobuf.Internal.FloatList; import com.google.protobuf.Internal.IntList; import com.google.protobuf.Internal.LongList; @@ -45,6 +46,7 @@ import java.io.ObjectStreamException; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -136,6 +138,7 @@ public abstract class GeneratedMessageLite< return false; } + try { visit(EqualsVisitor.INSTANCE, (MessageType) other); } catch (NotEqualsException e) { @@ -1154,6 +1157,7 @@ public abstract class GeneratedMessageLite< } } + /** * Lite equivalent to {@link GeneratedMessage.GeneratedExtension}. * @@ -1527,6 +1531,20 @@ public abstract class GeneratedMessageLite< return message; } + // Validates last tag. + protected static > T parseFrom( + T defaultInstance, ByteBuffer data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + return checkMessageInitialized( + parseFrom(defaultInstance, CodedInputStream.newInstance(data), extensionRegistry)); + } + + // Validates last tag. + protected static > T parseFrom( + T defaultInstance, ByteBuffer data) throws InvalidProtocolBufferException { + return parseFrom(defaultInstance, data, ExtensionRegistryLite.getEmptyRegistry()); + } + // Validates last tag. protected static > T parseFrom( T defaultInstance, ByteString data) @@ -1979,13 +1997,13 @@ public abstract class GeneratedMessageLite< /** * Implements hashCode by accumulating state. */ - private static class HashCodeVisitor implements Visitor { + static class HashCodeVisitor implements Visitor { // The caller must ensure that the visitor is invoked parameterized with this and this such that // other is this. This is required due to how oneof cases are handled. See the class comment // on Visitor for more information. - private int hashCode = 0; + int hashCode = 0; @Override public boolean visitBoolean( diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index fd051e75..b949cd17 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -41,7 +41,7 @@ import com.google.protobuf.Descriptors.OneofDescriptor; // class without breaking binary compatibility with old generated code that still subclasses // the old GeneratedMessageV3 class. To allow these different GeneratedMessageV3V? classes to // interoperate (e.g., a GeneratedMessageV3V3 object has a message extension field whose class -// type is GeneratedMessageV3V4), these classes still share a common parent class AbstarctMessage +// type is GeneratedMessageV3V4), these classes still share a common parent class AbstractMessage // and are using the same GeneratedMessage.GeneratedExtension class for extension definitions. // Since this class becomes GeneratedMessageV3V? in opensource, we have to add an import here // to be able to use GeneratedMessage.GeneratedExtension. The GeneratedExtension definition in diff --git a/java/core/src/main/java/com/google/protobuf/IntArrayList.java b/java/core/src/main/java/com/google/protobuf/IntArrayList.java index 2f526e3f..aff5c21b 100644 --- a/java/core/src/main/java/com/google/protobuf/IntArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/IntArrayList.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import com.google.protobuf.Internal.IntList; +import static com.google.protobuf.Internal.checkNotNull; +import com.google.protobuf.Internal.IntList; import java.util.Arrays; import java.util.Collection; import java.util.RandomAccess; @@ -41,9 +42,8 @@ import java.util.RandomAccess; * * @author dweis@google.com (Daniel Weis) */ -final class IntArrayList - extends AbstractProtobufList - implements IntList, RandomAccess { +final class IntArrayList extends AbstractProtobufList + implements IntList, RandomAccess, PrimitiveNonBoxingCollection { private static final IntArrayList EMPTY_LIST = new IntArrayList(); static { @@ -198,9 +198,7 @@ final class IntArrayList public boolean addAll(Collection collection) { ensureIsMutable(); - if (collection == null) { - throw new NullPointerException(); - } + checkNotNull(collection); // We specialize when adding another IntArrayList to avoid boxing elements. if (!(collection instanceof IntArrayList)) { diff --git a/java/core/src/main/java/com/google/protobuf/Internal.java b/java/core/src/main/java/com/google/protobuf/Internal.java index c234559c..36bdece3 100644 --- a/java/core/src/main/java/com/google/protobuf/Internal.java +++ b/java/core/src/main/java/com/google/protobuf/Internal.java @@ -59,6 +59,16 @@ public final class Internal { static final Charset UTF_8 = Charset.forName("UTF-8"); static final Charset ISO_8859_1 = Charset.forName("ISO-8859-1"); + /** + * Throws an appropriate {@link NullPointerException} if the given objects is {@code null}. + */ + static T checkNotNull(T obj) { + if (obj == null) { + throw new NullPointerException(); + } + return obj; + } + /** * Throws an appropriate {@link NullPointerException} if the given objects is {@code null}. */ @@ -420,6 +430,11 @@ public final class Internal { CodedInputStream.newInstance(EMPTY_BYTE_ARRAY); + /** Helper method to merge two MessageLite instances. */ + static Object mergeMessage(Object destination, Object source) { + return ((MessageLite) destination).toBuilder().mergeFrom((MessageLite) source).buildPartial(); + } + /** * Provides an immutable view of {@code List} around a {@code List}. * diff --git a/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java b/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java index 4b0ba0fd..49ecfc0b 100644 --- a/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java +++ b/java/core/src/main/java/com/google/protobuf/LazyFieldLite.java @@ -394,6 +394,7 @@ public class LazyFieldLite { } } + /** * Might lazily parse the bytes that were previously passed in. Is thread-safe. */ diff --git a/java/core/src/main/java/com/google/protobuf/LongArrayList.java b/java/core/src/main/java/com/google/protobuf/LongArrayList.java index 5a772e3a..fc146e23 100644 --- a/java/core/src/main/java/com/google/protobuf/LongArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/LongArrayList.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import com.google.protobuf.Internal.LongList; +import static com.google.protobuf.Internal.checkNotNull; +import com.google.protobuf.Internal.LongList; import java.util.Arrays; import java.util.Collection; import java.util.RandomAccess; @@ -41,9 +42,8 @@ import java.util.RandomAccess; * * @author dweis@google.com (Daniel Weis) */ -final class LongArrayList - extends AbstractProtobufList - implements LongList, RandomAccess { +final class LongArrayList extends AbstractProtobufList + implements LongList, RandomAccess, PrimitiveNonBoxingCollection { private static final LongArrayList EMPTY_LIST = new LongArrayList(); static { @@ -198,9 +198,7 @@ final class LongArrayList public boolean addAll(Collection collection) { ensureIsMutable(); - if (collection == null) { - throw new NullPointerException(); - } + checkNotNull(collection); // We specialize when adding another LongArrayList to avoid boxing elements. if (!(collection instanceof LongArrayList)) { diff --git a/java/core/src/main/java/com/google/protobuf/MapEntry.java b/java/core/src/main/java/com/google/protobuf/MapEntry.java index 7e8e9aad..0849b821 100644 --- a/java/core/src/main/java/com/google/protobuf/MapEntry.java +++ b/java/core/src/main/java/com/google/protobuf/MapEntry.java @@ -33,7 +33,6 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; - import java.io.IOException; import java.util.Collections; import java.util.Map; @@ -171,7 +170,7 @@ public final class MapEntry extends AbstractMessage { @Override public Builder toBuilder() { - return new Builder(metadata, key, value); + return new Builder(metadata, key, value, true, true); } @Override @@ -247,15 +246,19 @@ public final class MapEntry extends AbstractMessage { private final Metadata metadata; private K key; private V value; + private boolean hasKey; + private boolean hasValue; private Builder(Metadata metadata) { - this(metadata, metadata.defaultKey, metadata.defaultValue); + this(metadata, metadata.defaultKey, metadata.defaultValue, false, false); } - private Builder(Metadata metadata, K key, V value) { + private Builder(Metadata metadata, K key, V value, boolean hasKey, boolean hasValue) { this.metadata = metadata; this.key = key; this.value = value; + this.hasKey = hasKey; + this.hasValue = hasValue; } public K getKey() { @@ -268,21 +271,25 @@ public final class MapEntry extends AbstractMessage { public Builder setKey(K key) { this.key = key; + this.hasKey = true; return this; } public Builder clearKey() { this.key = metadata.defaultKey; + this.hasKey = false; return this; } public Builder setValue(V value) { this.value = value; + this.hasValue = true; return this; } public Builder clearValue() { this.value = metadata.defaultValue; + this.hasValue = false; return this; } @@ -404,7 +411,7 @@ public final class MapEntry extends AbstractMessage { @Override public boolean hasField(FieldDescriptor field) { checkFieldDescriptor(field); - return true; + return field.getNumber() == 1 ? hasKey : hasValue; } @Override @@ -438,7 +445,7 @@ public final class MapEntry extends AbstractMessage { @Override @SuppressWarnings("unchecked") public Builder clone() { - return new Builder(metadata, key, value); + return new Builder(metadata, key, value, hasKey, hasValue); } } @@ -448,4 +455,9 @@ public final class MapEntry extends AbstractMessage { } return true; } + + /** Returns the metadata only for experimental runtime. */ + final Metadata getMetadata() { + return metadata; + } } diff --git a/java/core/src/main/java/com/google/protobuf/MapEntryLite.java b/java/core/src/main/java/com/google/protobuf/MapEntryLite.java index 22aef8f9..dcb5dfad 100644 --- a/java/core/src/main/java/com/google/protobuf/MapEntryLite.java +++ b/java/core/src/main/java/com/google/protobuf/MapEntryLite.java @@ -223,4 +223,9 @@ public class MapEntryLite { input.popLimit(oldLimit); map.put(key, value); } + + /** For experimental runtime internal use only. */ + Metadata getMetadata() { + return metadata; + } } diff --git a/java/core/src/main/java/com/google/protobuf/MapField.java b/java/core/src/main/java/com/google/protobuf/MapField.java index 805defe2..ad8ceb02 100644 --- a/java/core/src/main/java/com/google/protobuf/MapField.java +++ b/java/core/src/main/java/com/google/protobuf/MapField.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -329,6 +331,8 @@ public class MapField implements MutabilityOracle { @Override public V put(K key, V value) { mutabilityOracle.ensureMutable(); + checkNotNull(key); + checkNotNull(value); return delegate.put(key, value); } @@ -341,6 +345,10 @@ public class MapField implements MutabilityOracle { @Override public void putAll(Map m) { mutabilityOracle.ensureMutable(); + for (K key : m.keySet()) { + checkNotNull(key); + checkNotNull(m.get(key)); + } delegate.putAll(m); } diff --git a/java/core/src/main/java/com/google/protobuf/MapFieldLite.java b/java/core/src/main/java/com/google/protobuf/MapFieldLite.java index 42640279..a8b3dd88 100644 --- a/java/core/src/main/java/com/google/protobuf/MapFieldLite.java +++ b/java/core/src/main/java/com/google/protobuf/MapFieldLite.java @@ -30,8 +30,9 @@ package com.google.protobuf; -import com.google.protobuf.Internal.EnumLite; +import static com.google.protobuf.Internal.checkNotNull; +import com.google.protobuf.Internal.EnumLite; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -88,6 +89,9 @@ public final class MapFieldLite extends LinkedHashMap { @Override public V put(K key, V value) { ensureMutable(); + checkNotNull(key); + + checkNotNull(value); return super.put(key, value); } @@ -97,6 +101,7 @@ public final class MapFieldLite extends LinkedHashMap { @Override public void putAll(Map m) { ensureMutable(); + checkForNullKeysAndValues(m); super.putAll(m); } @@ -105,6 +110,13 @@ public final class MapFieldLite extends LinkedHashMap { return super.remove(key); } + private static void checkForNullKeysAndValues(Map m) { + for (Object key : m.keySet()) { + checkNotNull(key); + checkNotNull(m.get(key)); + } + } + private static boolean equals(Object a, Object b) { if (a instanceof byte[] && b instanceof byte[]) { return Arrays.equals((byte[]) a, (byte[]) b); diff --git a/java/core/src/main/java/com/google/protobuf/Parser.java b/java/core/src/main/java/com/google/protobuf/Parser.java index cfbcb442..e07c6895 100644 --- a/java/core/src/main/java/com/google/protobuf/Parser.java +++ b/java/core/src/main/java/com/google/protobuf/Parser.java @@ -31,6 +31,7 @@ package com.google.protobuf; import java.io.InputStream; +import java.nio.ByteBuffer; /** * Abstract interface for parsing Protocol Messages. @@ -92,6 +93,18 @@ public interface Parser { // --------------------------------------------------------------- // Convenience methods. + /** + * Parses {@code data} as a message of {@code MessageType}. This is just a small wrapper around + * {@link #parseFrom(CodedInputStream)}. + */ + public MessageType parseFrom(ByteBuffer data) throws InvalidProtocolBufferException; + + /** + * Parses {@code data} as a message of {@code MessageType}. This is just a small wrapper around + * {@link #parseFrom(CodedInputStream, ExtensionRegistryLite)}. + */ + public MessageType parseFrom(ByteBuffer data, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException; /** * Parses {@code data} as a message of {@code MessageType}. * This is just a small wrapper around {@link #parseFrom(CodedInputStream)}. diff --git a/java/core/src/main/java/com/google/protobuf/PrimitiveNonBoxingCollection.java b/java/core/src/main/java/com/google/protobuf/PrimitiveNonBoxingCollection.java new file mode 100644 index 00000000..79b5769d --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/PrimitiveNonBoxingCollection.java @@ -0,0 +1,34 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +/** A marker interface indicating that the collection supports primitives and is non-boxing. */ +interface PrimitiveNonBoxingCollection {} diff --git a/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java b/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java index 77b61b5f..30c991d4 100644 --- a/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java +++ b/java/core/src/main/java/com/google/protobuf/RepeatedFieldBuilderV3.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import java.util.AbstractList; import java.util.ArrayList; import java.util.Collection; @@ -290,9 +292,7 @@ public class RepeatedFieldBuilderV3 */ public RepeatedFieldBuilderV3 setMessage( int index, MType message) { - if (message == null) { - throw new NullPointerException(); - } + checkNotNull(message); ensureMutableMessageList(); messages.set(index, message); if (builders != null) { @@ -315,9 +315,7 @@ public class RepeatedFieldBuilderV3 */ public RepeatedFieldBuilderV3 addMessage( MType message) { - if (message == null) { - throw new NullPointerException(); - } + checkNotNull(message); ensureMutableMessageList(); messages.add(message); if (builders != null) { @@ -339,9 +337,7 @@ public class RepeatedFieldBuilderV3 */ public RepeatedFieldBuilderV3 addMessage( int index, MType message) { - if (message == null) { - throw new NullPointerException(); - } + checkNotNull(message); ensureMutableMessageList(); messages.add(index, message); if (builders != null) { @@ -363,9 +359,7 @@ public class RepeatedFieldBuilderV3 public RepeatedFieldBuilderV3 addAllMessages( Iterable values) { for (final MType value : values) { - if (value == null) { - throw new NullPointerException(); - } + checkNotNull(value); } // If we can inspect the size, we can more efficiently add messages. diff --git a/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java b/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java index fb1f76a7..8ab0f26d 100644 --- a/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java +++ b/java/core/src/main/java/com/google/protobuf/SingleFieldBuilderV3.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + /** * {@code SingleFieldBuilderV3} implements a structure that a protocol * message uses to hold a single field of another protocol message. It supports @@ -84,10 +86,7 @@ public class SingleFieldBuilderV3 MType message, AbstractMessage.BuilderParent parent, boolean isClean) { - if (message == null) { - throw new NullPointerException(); - } - this.message = message; + this.message = checkNotNull(message); this.parent = parent; this.isClean = isClean; } @@ -169,10 +168,7 @@ public class SingleFieldBuilderV3 */ public SingleFieldBuilderV3 setMessage( MType message) { - if (message == null) { - throw new NullPointerException(); - } - this.message = message; + this.message = checkNotNull(message); if (builder != null) { builder.dispose(); builder = null; diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 49708242..53dead80 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -1442,7 +1442,7 @@ public final class TextFormat { /** * Parse a single field from {@code tokenizer} and merge it into - * {@code builder}. + * {@code target}. */ private void mergeField(final Tokenizer tokenizer, final ExtensionRegistry extensionRegistry, @@ -1712,6 +1712,8 @@ public final class TextFormat { } if (field.isRepeated()) { + // TODO(b/29122459): If field.isMapField() and FORBID_SINGULAR_OVERWRITES mode, + // check for duplicate map keys here. target.addRepeatedField(field, value); } else if ((singularOverwritePolicy == SingularOverwritePolicy.FORBID_SINGULAR_OVERWRITES) diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index 2bef27e9..d9381135 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -91,7 +91,7 @@ public final class UnknownFieldSet implements MessageLite { * Construct an {@code UnknownFieldSet} around the given map. The map is * expected to be immutable. */ - private UnknownFieldSet(final Map fields, + UnknownFieldSet(final Map fields, final Map fieldsDescending) { this.fields = fields; } @@ -715,7 +715,7 @@ public final class UnknownFieldSet implements MessageLite { * @see UnknownFieldSet */ public static final class Field { - private Field() {} + Field() {} /** Construct a new {@link Builder}. */ public static Builder newBuilder() { diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java index 5f7bafd6..ca80d946 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java @@ -33,19 +33,23 @@ package com.google.protobuf; import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedExceptionAction; -import sun.misc.Unsafe; +import java.util.logging.Level; +import java.util.logging.Logger; /** Utility class for working with unsafe operations. */ -// TODO(nathanmittler): Add support for Android Memory/MemoryBlock final class UnsafeUtil { + private static final Logger logger = Logger.getLogger(UnsafeUtil.class.getName()); private static final sun.misc.Unsafe UNSAFE = getUnsafe(); + private static final MemoryAccessor MEMORY_ACCESSOR = getMemoryAccessor(); private static final boolean HAS_UNSAFE_BYTEBUFFER_OPERATIONS = supportsUnsafeByteBufferOperations(); private static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = supportsUnsafeArrayOperations(); + private static final boolean HAS_UNSAFE_COPY_MEMORY = supportsUnsafeCopyMemory(); private static final long ARRAY_BASE_OFFSET = byteArrayBaseOffset(); - private static final long BUFFER_ADDRESS_OFFSET = fieldOffset(field(Buffer.class, "address")); + private static final long BUFFER_ADDRESS_OFFSET = fieldOffset(bufferAddressField()); private UnsafeUtil() {} @@ -53,20 +57,16 @@ final class UnsafeUtil { return HAS_UNSAFE_ARRAY_OPERATIONS; } - static boolean hasUnsafeByteBufferOperations() { - return HAS_UNSAFE_BYTEBUFFER_OPERATIONS; + static boolean hasUnsafeCopyMemory() { + return HAS_UNSAFE_COPY_MEMORY; } - static Object allocateInstance(Class clazz) { - try { - return UNSAFE.allocateInstance(clazz); - } catch (InstantiationException e) { - throw new RuntimeException(e); - } + static boolean hasUnsafeByteBufferOperations() { + return HAS_UNSAFE_BYTEBUFFER_OPERATIONS; } static long objectFieldOffset(Field field) { - return UNSAFE.objectFieldOffset(field); + return MEMORY_ACCESSOR.objectFieldOffset(field); } static long getArrayBaseOffset() { @@ -74,103 +74,103 @@ final class UnsafeUtil { } static byte getByte(Object target, long offset) { - return UNSAFE.getByte(target, offset); + return MEMORY_ACCESSOR.getByte(target, offset); } static void putByte(Object target, long offset, byte value) { - UNSAFE.putByte(target, offset, value); + MEMORY_ACCESSOR.putByte(target, offset, value); } static int getInt(Object target, long offset) { - return UNSAFE.getInt(target, offset); + return MEMORY_ACCESSOR.getInt(target, offset); } static void putInt(Object target, long offset, int value) { - UNSAFE.putInt(target, offset, value); + MEMORY_ACCESSOR.putInt(target, offset, value); } static long getLong(Object target, long offset) { - return UNSAFE.getLong(target, offset); + return MEMORY_ACCESSOR.getLong(target, offset); } static void putLong(Object target, long offset, long value) { - UNSAFE.putLong(target, offset, value); + MEMORY_ACCESSOR.putLong(target, offset, value); } static boolean getBoolean(Object target, long offset) { - return UNSAFE.getBoolean(target, offset); + return MEMORY_ACCESSOR.getBoolean(target, offset); } static void putBoolean(Object target, long offset, boolean value) { - UNSAFE.putBoolean(target, offset, value); + MEMORY_ACCESSOR.putBoolean(target, offset, value); } static float getFloat(Object target, long offset) { - return UNSAFE.getFloat(target, offset); + return MEMORY_ACCESSOR.getFloat(target, offset); } static void putFloat(Object target, long offset, float value) { - UNSAFE.putFloat(target, offset, value); + MEMORY_ACCESSOR.putFloat(target, offset, value); } static double getDouble(Object target, long offset) { - return UNSAFE.getDouble(target, offset); + return MEMORY_ACCESSOR.getDouble(target, offset); } static void putDouble(Object target, long offset, double value) { - UNSAFE.putDouble(target, offset, value); + MEMORY_ACCESSOR.putDouble(target, offset, value); } static Object getObject(Object target, long offset) { - return UNSAFE.getObject(target, offset); + return MEMORY_ACCESSOR.getObject(target, offset); } static void putObject(Object target, long offset, Object value) { - UNSAFE.putObject(target, offset, value); + MEMORY_ACCESSOR.putObject(target, offset, value); } static void copyMemory( Object src, long srcOffset, Object target, long targetOffset, long length) { - UNSAFE.copyMemory(src, srcOffset, target, targetOffset, length); + MEMORY_ACCESSOR.copyMemory(src, srcOffset, target, targetOffset, length); } static byte getByte(long address) { - return UNSAFE.getByte(address); + return MEMORY_ACCESSOR.getByte(address); } static void putByte(long address, byte value) { - UNSAFE.putByte(address, value); + MEMORY_ACCESSOR.putByte(address, value); } static int getInt(long address) { - return UNSAFE.getInt(address); + return MEMORY_ACCESSOR.getInt(address); } static void putInt(long address, int value) { - UNSAFE.putInt(address, value); + MEMORY_ACCESSOR.putInt(address, value); } static long getLong(long address) { - return UNSAFE.getLong(address); + return MEMORY_ACCESSOR.getLong(address); } static void putLong(long address, long value) { - UNSAFE.putLong(address, value); + MEMORY_ACCESSOR.putLong(address, value); } static void copyMemory(long srcAddress, long targetAddress, long length) { - UNSAFE.copyMemory(srcAddress, targetAddress, length); - } - - static void setMemory(long address, long numBytes, byte value) { - UNSAFE.setMemory(address, numBytes, value); + MEMORY_ACCESSOR.copyMemory(srcAddress, targetAddress, length); } /** * Gets the offset of the {@code address} field of the given direct {@link ByteBuffer}. */ static long addressOffset(ByteBuffer buffer) { - return UNSAFE.getLong(buffer, BUFFER_ADDRESS_OFFSET); + return MEMORY_ACCESSOR.getLong(buffer, BUFFER_ADDRESS_OFFSET); + } + + static Object getStaticObject(Field field) { + return MEMORY_ACCESSOR.getStaticObject(field); } /** @@ -181,7 +181,7 @@ final class UnsafeUtil { try { unsafe = AccessController.doPrivileged( - new PrivilegedExceptionAction() { + new PrivilegedExceptionAction() { @Override public sun.misc.Unsafe run() throws Exception { Class k = sun.misc.Unsafe.class; @@ -204,69 +204,114 @@ final class UnsafeUtil { return unsafe; } + /** Get a {@link MemoryAccessor} appropriate for the platform, or null if not supported. */ + private static MemoryAccessor getMemoryAccessor() { + if (UNSAFE == null) { + return null; + } + return new JvmMemoryAccessor(UNSAFE); + } + /** Indicates whether or not unsafe array operations are supported on this platform. */ private static boolean supportsUnsafeArrayOperations() { - boolean supported = false; - if (UNSAFE != null) { - try { - Class clazz = UNSAFE.getClass(); - clazz.getMethod("objectFieldOffset", Field.class); - clazz.getMethod("allocateInstance", Class.class); - clazz.getMethod("arrayBaseOffset", Class.class); - clazz.getMethod("getByte", Object.class, long.class); - clazz.getMethod("putByte", Object.class, long.class, byte.class); - clazz.getMethod("getBoolean", Object.class, long.class); - clazz.getMethod("putBoolean", Object.class, long.class, boolean.class); - clazz.getMethod("getInt", Object.class, long.class); - clazz.getMethod("putInt", Object.class, long.class, int.class); - clazz.getMethod("getLong", Object.class, long.class); - clazz.getMethod("putLong", Object.class, long.class, long.class); - clazz.getMethod("getFloat", Object.class, long.class); - clazz.getMethod("putFloat", Object.class, long.class, float.class); - clazz.getMethod("getDouble", Object.class, long.class); - clazz.getMethod("putDouble", Object.class, long.class, double.class); - clazz.getMethod("getObject", Object.class, long.class); - clazz.getMethod("putObject", Object.class, long.class, Object.class); - clazz.getMethod( - "copyMemory", Object.class, long.class, Object.class, long.class, long.class); - supported = true; - } catch (Throwable e) { - // Do nothing. - } - } - return supported; + if (UNSAFE == null) { + return false; + } + try { + Class clazz = UNSAFE.getClass(); + clazz.getMethod("objectFieldOffset", Field.class); + clazz.getMethod("arrayBaseOffset", Class.class); + clazz.getMethod("getInt", Object.class, long.class); + clazz.getMethod("putInt", Object.class, long.class, int.class); + clazz.getMethod("getLong", Object.class, long.class); + clazz.getMethod("putLong", Object.class, long.class, long.class); + clazz.getMethod("getObject", Object.class, long.class); + clazz.getMethod("putObject", Object.class, long.class, Object.class); + clazz.getMethod("getByte", Object.class, long.class); + clazz.getMethod("putByte", Object.class, long.class, byte.class); + clazz.getMethod("getBoolean", Object.class, long.class); + clazz.getMethod("putBoolean", Object.class, long.class, boolean.class); + clazz.getMethod("getFloat", Object.class, long.class); + clazz.getMethod("putFloat", Object.class, long.class, float.class); + clazz.getMethod("getDouble", Object.class, long.class); + clazz.getMethod("putDouble", Object.class, long.class, double.class); + + return true; + } catch (Throwable e) { + logger.log( + Level.WARNING, + "platform method missing - proto runtime falling back to safer methods: " + e); + } + return false; + } + + /** + * Indicates whether or not unsafe copyMemory(object, long, object, long, long) operations are + * supported on this platform. + */ + private static boolean supportsUnsafeCopyMemory() { + if (UNSAFE == null) { + return false; + } + try { + Class clazz = UNSAFE.getClass(); + clazz.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class); + + return true; + } catch (Throwable e) { + logger.log( + Level.WARNING, + "copyMemory is missing from platform - proto runtime falling back to safer methods."); + } + return false; } private static boolean supportsUnsafeByteBufferOperations() { - boolean supported = false; - if (UNSAFE != null) { - try { - Class clazz = UNSAFE.getClass(); - // Methods for getting direct buffer address. - clazz.getMethod("objectFieldOffset", Field.class); - clazz.getMethod("getLong", Object.class, long.class); - - clazz.getMethod("getByte", long.class); - clazz.getMethod("putByte", long.class, byte.class); - clazz.getMethod("getInt", long.class); - clazz.getMethod("putInt", long.class, int.class); - clazz.getMethod("getLong", long.class); - clazz.getMethod("putLong", long.class, long.class); - clazz.getMethod("setMemory", long.class, long.class, byte.class); - clazz.getMethod("copyMemory", long.class, long.class, long.class); - supported = true; - } catch (Throwable e) { - // Do nothing. - } - } - return supported; + if (UNSAFE == null) { + return false; + } + try { + Class clazz = UNSAFE.getClass(); + // Methods for getting direct buffer address. + clazz.getMethod("objectFieldOffset", Field.class); + clazz.getMethod("getLong", Object.class, long.class); + + clazz.getMethod("getByte", long.class); + clazz.getMethod("putByte", long.class, byte.class); + clazz.getMethod("getInt", long.class); + clazz.getMethod("putInt", long.class, int.class); + clazz.getMethod("getLong", long.class); + clazz.getMethod("putLong", long.class, long.class); + clazz.getMethod("copyMemory", long.class, long.class, long.class); + return true; + } catch (Throwable e) { + logger.log( + Level.WARNING, + "platform method missing - proto runtime falling back to safer methods: " + e); + } + return false; + } + + + @SuppressWarnings("unchecked") + private static Class getClassForName(String name) { + try { + return (Class) Class.forName(name); + } catch (Throwable e) { + return null; + } + } + + /** Finds the address field within a direct {@link Buffer}. */ + private static Field bufferAddressField() { + return field(Buffer.class, "address"); } /** * Get the base offset for byte arrays, or {@code -1} if {@code sun.misc.Unsafe} is not available. */ private static int byteArrayBaseOffset() { - return HAS_UNSAFE_ARRAY_OPERATIONS ? UNSAFE.arrayBaseOffset(byte[].class) : -1; + return HAS_UNSAFE_ARRAY_OPERATIONS ? MEMORY_ACCESSOR.arrayBaseOffset(byte[].class) : -1; } /** @@ -274,7 +319,7 @@ final class UnsafeUtil { * available. */ private static long fieldOffset(Field field) { - return field == null || UNSAFE == null ? -1 : UNSAFE.objectFieldOffset(field); + return field == null || MEMORY_ACCESSOR == null ? -1 : MEMORY_ACCESSOR.objectFieldOffset(field); } /** @@ -292,4 +337,174 @@ final class UnsafeUtil { } return field; } + + private abstract static class MemoryAccessor { + + sun.misc.Unsafe unsafe; + + MemoryAccessor(sun.misc.Unsafe unsafe) { + this.unsafe = unsafe; + } + + public final long objectFieldOffset(Field field) { + return unsafe.objectFieldOffset(field); + } + + public abstract byte getByte(Object target, long offset); + + public abstract void putByte(Object target, long offset, byte value); + + public final int getInt(Object target, long offset) { + return unsafe.getInt(target, offset); + } + + public final void putInt(Object target, long offset, int value) { + unsafe.putInt(target, offset, value); + } + + public final long getLong(Object target, long offset) { + return unsafe.getLong(target, offset); + } + + public final void putLong(Object target, long offset, long value) { + unsafe.putLong(target, offset, value); + } + + public abstract boolean getBoolean(Object target, long offset); + + public abstract void putBoolean(Object target, long offset, boolean value); + + public abstract float getFloat(Object target, long offset); + + public abstract void putFloat(Object target, long offset, float value); + + public abstract double getDouble(Object target, long offset); + + public abstract void putDouble(Object target, long offset, double value); + + public final Object getObject(Object target, long offset) { + return unsafe.getObject(target, offset); + } + + public final void putObject(Object target, long offset, Object value) { + unsafe.putObject(target, offset, value); + } + + public final int arrayBaseOffset(Class clazz) { + return unsafe.arrayBaseOffset(clazz); + } + + public abstract byte getByte(long address); + + public abstract void putByte(long address, byte value); + + public abstract int getInt(long address); + + public abstract void putInt(long address, int value); + + public abstract long getLong(long address); + + public abstract void putLong(long address, long value); + + public abstract void copyMemory(long srcAddress, long targetAddress, long length); + + public abstract void copyMemory( + Object src, long srcOffset, Object target, long targetOffset, long length); + + public abstract Object getStaticObject(Field field); + } + + private static final class JvmMemoryAccessor extends MemoryAccessor { + + JvmMemoryAccessor(sun.misc.Unsafe unsafe) { + super(unsafe); + } + + @Override + public byte getByte(long address) { + return unsafe.getByte(address); + } + + @Override + public void putByte(long address, byte value) { + unsafe.putByte(address, value); + } + + @Override + public int getInt(long address) { + return unsafe.getInt(address); + } + + @Override + public void putInt(long address, int value) { + unsafe.putInt(address, value); + } + + @Override + public long getLong(long address) { + return unsafe.getLong(address); + } + + @Override + public void putLong(long address, long value) { + unsafe.putLong(address, value); + } + + @Override + public byte getByte(Object target, long offset) { + return unsafe.getByte(target, offset); + } + + @Override + public void putByte(Object target, long offset, byte value) { + unsafe.putByte(target, offset, value); + } + + @Override + public boolean getBoolean(Object target, long offset) { + return unsafe.getBoolean(target, offset); + } + + @Override + public void putBoolean(Object target, long offset, boolean value) { + unsafe.putBoolean(target, offset, value); + } + + @Override + public float getFloat(Object target, long offset) { + return unsafe.getFloat(target, offset); + } + + @Override + public void putFloat(Object target, long offset, float value) { + unsafe.putFloat(target, offset, value); + } + + @Override + public double getDouble(Object target, long offset) { + return unsafe.getDouble(target, offset); + } + + @Override + public void putDouble(Object target, long offset, double value) { + unsafe.putDouble(target, offset, value); + } + + @Override + public void copyMemory( + Object src, long srcOffset, Object target, long targetOffset, long length) { + unsafe.copyMemory(src, srcOffset, target, targetOffset, length); + } + + @Override + public void copyMemory(long srcAddress, long targetAddress, long length) { + unsafe.copyMemory(srcAddress, targetAddress, length); + } + + @Override + public Object getStaticObject(Field field) { + return getObject(unsafe.staticFieldBase(field), unsafe.staticFieldOffset(field)); + } + } + } diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index 5b80d405..be7b746e 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -1332,7 +1332,7 @@ final class Utf8 { // the index (relative to the start of the array) is also 8-byte aligned. We do this by // ANDing the index with 7 to determine the number of bytes that need to be read before // we're 8-byte aligned. - final int unaligned = (int) offset & 7; + final int unaligned = 8 - ((int) offset & 7); for (int j = unaligned; j > 0; j--) { if (UnsafeUtil.getByte(bytes, offset++) < 0) { return unaligned - j; diff --git a/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java b/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java index 5f013f3c..f27e8e51 100644 --- a/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java +++ b/java/core/src/test/java/com/google/protobuf/LazyFieldTest.java @@ -32,7 +32,6 @@ package com.google.protobuf; import protobuf_unittest.UnittestProto.TestAllExtensions; import protobuf_unittest.UnittestProto.TestAllTypes; -import java.io.IOException; import junit.framework.TestCase; /** @@ -89,6 +88,7 @@ public class LazyFieldTest extends TestCase { assertFalse(message.equals(lazyField.getValue())); } + @SuppressWarnings("EqualsIncompatibleType") // LazyField.equals() is not symmetric public void testEqualsObjectEx() throws Exception { TestAllExtensions message = TestUtil.getAllExtensionsSet(); LazyField lazyField = createLazyFieldFromMessage(message); diff --git a/java/core/src/test/java/com/google/protobuf/LiteTest.java b/java/core/src/test/java/com/google/protobuf/LiteTest.java index 538432f7..98c637a9 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteTest.java @@ -52,6 +52,7 @@ import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof; +import java.nio.ByteBuffer; import junit.framework.TestCase; /** @@ -2174,6 +2175,24 @@ public class LiteTest extends TestCase { assertFalse(bar.equals(barPrime)); } + public void testEqualsAndHashCodeForTrickySchemaTypes() { + Foo foo1 = Foo.newBuilder() + .build(); + Foo foo2 = Foo.newBuilder() + .setSint64(1) + .build(); + Foo foo3 = Foo.newBuilder() + .putMyMap("key", "value2") + .build(); + Foo foo4 = Foo.newBuilder() + .setMyGroup(Foo.MyGroup.newBuilder().setValue(4).build()) + .build(); + + assertEqualsAndHashCodeAreFalse(foo1, foo2); + assertEqualsAndHashCodeAreFalse(foo1, foo3); + assertEqualsAndHashCodeAreFalse(foo1, foo4); + } + public void testOneofEquals() throws Exception { TestOneofEquals.Builder builder = TestOneofEquals.newBuilder(); TestOneofEquals message1 = builder.build(); @@ -2270,4 +2289,93 @@ public class LiteTest extends TestCase { // This tests that we don't infinite loop. TestRecursiveOneof.getDefaultInstance().hashCode(); } + + public void testParseFromByteBuffer() throws Exception { + TestAllTypesLite message = + TestAllTypesLite.newBuilder() + .setOptionalInt32(123) + .addRepeatedString("hello") + .setOptionalNestedMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(7)) + .build(); + + TestAllTypesLite copy = + TestAllTypesLite.parseFrom(message.toByteString().asReadOnlyByteBuffer()); + + assertEquals(message, copy); + } + + public void testParseFromByteBufferThrows() { + try { + TestAllTypesLite.parseFrom(ByteBuffer.wrap(new byte[] { 0x5 })); + fail(); + } catch (InvalidProtocolBufferException expected) { + } + + TestAllTypesLite message = + TestAllTypesLite.newBuilder() + .setOptionalInt32(123) + .addRepeatedString("hello") + .build(); + + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray(), 0, message.getSerializedSize() - 1); + try { + TestAllTypesLite.parseFrom(buffer); + fail(); + } catch (InvalidProtocolBufferException expected) { + assertEquals( + TestAllTypesLite.newBuilder() + .setOptionalInt32(123) + .build(), + expected.getUnfinishedMessage()); + } + } + + public void testParseFromByteBuffer_extensions() throws Exception { + TestAllExtensionsLite message = + TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) + .addExtension(UnittestLite.repeatedStringExtensionLite, "hello") + .setExtension( + UnittestLite.optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ) + .setExtension( + UnittestLite.optionalNestedMessageExtensionLite, + TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build()) + .build(); + + ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance(); + UnittestLite.registerAllExtensions(registry); + + TestAllExtensionsLite copy = + TestAllExtensionsLite.parseFrom(message.toByteString().asReadOnlyByteBuffer(), registry); + + assertEquals(message, copy); + } + + public void testParseFromByteBufferThrows_extensions() { + ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance(); + UnittestLite.registerAllExtensions(registry); + try { + TestAllExtensionsLite.parseFrom(ByteBuffer.wrap(new byte[] { 0x5 }), registry); + fail(); + } catch (InvalidProtocolBufferException expected) { + } + + TestAllExtensionsLite message = + TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) + .addExtension(UnittestLite.repeatedStringExtensionLite, "hello") + .build(); + + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray(), 0, message.getSerializedSize() - 1); + try { + TestAllExtensionsLite.parseFrom(buffer, registry); + fail(); + } catch (InvalidProtocolBufferException expected) { + assertEquals( + TestAllExtensionsLite.newBuilder() + .setExtension(UnittestLite.optionalInt32ExtensionLite, 123) + .build(), + expected.getUnfinishedMessage()); + } + } } diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java index cfe4c453..453d3928 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java @@ -759,6 +759,7 @@ public class MapForProto2Test extends TestCase { assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); } + // See additional coverage in TextFormatTest.java. public void testTextFormat() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); setMapValuesUsingAccessors(builder); diff --git a/java/core/src/test/java/com/google/protobuf/MapTest.java b/java/core/src/test/java/com/google/protobuf/MapTest.java index 81e951cc..01b371a3 100644 --- a/java/core/src/test/java/com/google/protobuf/MapTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapTest.java @@ -30,7 +30,7 @@ package com.google.protobuf; - +import static org.junit.Assert.assertArrayEquals; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; @@ -870,6 +870,7 @@ public class MapTest extends TestCase { assertEquals(55, message.getInt32ToInt32Field().get(55).intValue()); } + // See additional coverage in TextFormatTest.java. public void testTextFormat() throws Exception { TestMap.Builder builder = TestMap.newBuilder(); setMapValuesUsingAccessors(builder); @@ -1492,4 +1493,40 @@ public class MapTest extends TestCase { map.put(key3, value3); return map; } + + public void testMap_withNulls() { + TestMap.Builder builder = TestMap.newBuilder(); + + try { + builder.putStringToInt32Field(null, 3); + fail(); + } catch (NullPointerException expected) { + } + + try { + builder.putAllStringToInt32Field(newMap(null, 3, "hi", 4)); + fail(); + } catch (NullPointerException expected) { + } + + try { + builder.putInt32ToMessageField(3, null); + fail(); + } catch (NullPointerException expected) { + } + + try { + builder.putAllInt32ToMessageField(newMap(4, null, 5, null)); + fail(); + } catch (NullPointerException expected) { + } + + try { + builder.putAllInt32ToMessageField(null); + fail(); + } catch (NullPointerException expected) { + } + + assertArrayEquals(new byte[0], builder.build().toByteArray()); + } } diff --git a/java/core/src/test/java/com/google/protobuf/ParserTest.java b/java/core/src/test/java/com/google/protobuf/ParserTest.java index 8c2e4c26..4bd34112 100644 --- a/java/core/src/test/java/com/google/protobuf/ParserTest.java +++ b/java/core/src/test/java/com/google/protobuf/ParserTest.java @@ -79,6 +79,8 @@ public class ParserTest extends TestCase { new ByteArrayInputStream(data), registry)); assertMessageEquals(message, parser.parseFrom( CodedInputStream.newInstance(data), registry)); + assertMessageEquals( + message, parser.parseFrom(message.toByteString().asReadOnlyByteBuffer(), registry)); } @SuppressWarnings("unchecked") @@ -99,6 +101,7 @@ public class ParserTest extends TestCase { new ByteArrayInputStream(data))); assertMessageEquals(message, parser.parseFrom( CodedInputStream.newInstance(data))); + assertMessageEquals(message, parser.parseFrom(message.toByteString().asReadOnlyByteBuffer())); } private void assertMessageEquals( @@ -178,6 +181,9 @@ public class ParserTest extends TestCase { public void testParseExtensions() throws Exception { assertRoundTripEquals(TestUtil.getAllExtensionsSet(), TestUtil.getExtensionRegistry()); + } + + public void testParseExtensionsLite() throws Exception { assertRoundTripEquals( TestUtilLite.getAllLiteExtensionsSet(), TestUtilLite.getExtensionRegistryLite()); } @@ -186,6 +192,9 @@ public class ParserTest extends TestCase { assertRoundTripEquals(TestUtil.getPackedSet()); assertRoundTripEquals(TestUtil.getPackedExtensionsSet(), TestUtil.getExtensionRegistry()); + } + + public void testParsePackedLite() throws Exception { assertRoundTripEquals( TestUtilLite.getLitePackedExtensionsSet(), TestUtilLite.getExtensionRegistryLite()); } @@ -195,15 +204,26 @@ public class ParserTest extends TestCase { TestAllTypes normalMessage = TestUtil.getAllSet(); ByteArrayOutputStream output = new ByteArrayOutputStream(); normalMessage.writeDelimitedTo(output); + normalMessage.writeDelimitedTo(output); + InputStream input = new ByteArrayInputStream(output.toByteArray()); + assertMessageEquals(normalMessage, normalMessage.getParserForType().parseDelimitedFrom(input)); + assertMessageEquals(normalMessage, normalMessage.getParserForType().parseDelimitedFrom(input)); + } + + public void testParseDelimitedToLite() throws Exception { // Write MessageLite with packed extension fields. TestPackedExtensionsLite packedMessage = TestUtilLite.getLitePackedExtensionsSet(); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + packedMessage.writeDelimitedTo(output); packedMessage.writeDelimitedTo(output); InputStream input = new ByteArrayInputStream(output.toByteArray()); assertMessageEquals( - normalMessage, - normalMessage.getParserForType().parseDelimitedFrom(input)); + packedMessage, + packedMessage + .getParserForType() + .parseDelimitedFrom(input, TestUtilLite.getExtensionRegistryLite())); assertMessageEquals( packedMessage, packedMessage @@ -314,8 +334,7 @@ public class ParserTest extends TestCase { public void testParsingMergeLite() throws Exception { // Build messages. - TestAllTypesLite.Builder builder = - TestAllTypesLite.newBuilder(); + TestAllTypesLite.Builder builder = TestAllTypesLite.newBuilder(); TestAllTypesLite msg1 = builder.setOptionalInt32(1).build(); builder.clear(); TestAllTypesLite msg2 = builder.setOptionalInt64(2).build(); diff --git a/java/core/src/test/java/com/google/protobuf/TestUtil.java b/java/core/src/test/java/com/google/protobuf/TestUtil.java index c1bd21db..e96adf07 100644 --- a/java/core/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/core/src/test/java/com/google/protobuf/TestUtil.java @@ -2622,6 +2622,8 @@ public final class TestUtil { break; case FOO_NOT_SET: break; + default: + // TODO(b/18683919): go/enum-switch-lsc } } diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index 6a91b02f..249d7c5e 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -30,9 +30,12 @@ package com.google.protobuf; +import static com.google.common.truth.Truth.assertThat; + import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.TextFormat.Parser.SingularOverwritePolicy; +import map_test.MapTestProto.TestMap; import protobuf_unittest.UnittestMset.TestMessageSetExtension1; import protobuf_unittest.UnittestMset.TestMessageSetExtension2; import protobuf_unittest.UnittestProto.OneString; @@ -940,6 +943,7 @@ public class TextFormatTest extends TestCase { } + // See additional coverage in testOneofOverwriteForbidden and testMapOverwriteForbidden. public void testParseNonRepeatedFields() throws Exception { assertParseSuccessWithOverwriteForbidden( "repeated_int32: 1\n" + @@ -950,6 +954,7 @@ public class TextFormatTest extends TestCase { assertParseSuccessWithOverwriteForbidden( "repeated_nested_message { bb: 1 }\n" + "repeated_nested_message { bb: 2 }\n"); + assertParseErrorWithOverwriteForbidden( "3:17: Non-repeated field " + "\"protobuf_unittest.TestAllTypes.optional_int32\" " + @@ -988,6 +993,7 @@ public class TextFormatTest extends TestCase { assertParseSuccessWithOverwriteForbidden("repeated_int32: [ 1, 2 ]\n"); assertParseSuccessWithOverwriteForbidden("RepeatedGroup [{ a: 1 },{ a: 2 }]\n"); assertParseSuccessWithOverwriteForbidden("repeated_nested_message [{ bb: 1 }, { bb: 2 }]\n"); + // See also testMapShortForm. } public void testParseShortRepeatedFormOfEmptyRepeatedFields() throws Exception { @@ -995,6 +1001,7 @@ public class TextFormatTest extends TestCase { assertParseSuccessWithOverwriteForbidden("repeated_int32: []\n"); assertParseSuccessWithOverwriteForbidden("RepeatedGroup []\n"); assertParseSuccessWithOverwriteForbidden("repeated_nested_message []\n"); + // See also testMapShortFormEmpty. } public void testParseShortRepeatedFormWithTrailingComma() throws Exception { @@ -1010,6 +1017,7 @@ public class TextFormatTest extends TestCase { assertParseErrorWithOverwriteForbidden( "1:37: Expected \"{\".", "repeated_nested_message [{ bb: 1 }, ]\n"); + // See also testMapShortFormTrailingComma. } public void testParseShortRepeatedFormOfNonRepeatedFields() throws Exception { @@ -1057,6 +1065,90 @@ public class TextFormatTest extends TestCase { assertTrue(oneof.hasFooInt()); } + // ======================================================================= + // test map + + public void testMapTextFormat() throws Exception { + TestMap message = + TestMap.newBuilder() + .putInt32ToStringField(10, "apple") + .putInt32ToStringField(20, "banana") + .putInt32ToStringField(30, "cherry") + .build(); + String text = TextFormat.printToUnicodeString(message); + { + TestMap.Builder dest = TestMap.newBuilder(); + TextFormat.merge(text, dest); + assertThat(dest.build()).isEqualTo(message); + } + { + TestMap.Builder dest = TestMap.newBuilder(); + parserWithOverwriteForbidden.merge(text, dest); + assertThat(dest.build()).isEqualTo(message); + } + } + + public void testMapShortForm() throws Exception { + String text = + "string_to_int32_field [{ key: 'x' value: 10 }, { key: 'y' value: 20 }]\n" + + "int32_to_message_field " + + "[{ key: 1 value { value: 100 } }, { key: 2 value: { value: 200 } }]\n"; + TestMap.Builder dest = TestMap.newBuilder(); + parserWithOverwriteForbidden.merge(text, dest); + TestMap message = dest.build(); + assertThat(message.getStringToInt32Field().size()).isEqualTo(2); + assertThat(message.getInt32ToMessageField().size()).isEqualTo(2); + assertThat(message.getStringToInt32Field().get("x")).isEqualTo(10); + assertThat(message.getInt32ToMessageField().get(2).getValue()).isEqualTo(200); + } + + public void testMapShortFormEmpty() throws Exception { + String text = "string_to_int32_field []\n" + + "int32_to_message_field: []\n"; + TestMap.Builder dest = TestMap.newBuilder(); + parserWithOverwriteForbidden.merge(text, dest); + TestMap message = dest.build(); + assertThat(message.getStringToInt32Field().size()).isEqualTo(0); + assertThat(message.getInt32ToMessageField().size()).isEqualTo(0); + } + + public void testMapShortFormTrailingComma() throws Exception { + String text = "string_to_int32_field [{ key: 'x' value: 10 }, ]\n"; + TestMap.Builder dest = TestMap.newBuilder(); + try { + parserWithOverwriteForbidden.merge(text, dest); + fail("Expected parse exception."); + } catch (TextFormat.ParseException e) { + assertThat(e).hasMessageThat().isEqualTo("1:48: Expected \"{\"."); + } + } + + public void testMapOverwrite() throws Exception { + String text = + "int32_to_int32_field { key: 1 value: 10 }\n" + + "int32_to_int32_field { key: 2 value: 20 }\n" + + "int32_to_int32_field { key: 1 value: 30 }\n"; + + { + // With default parser, last value set for the key holds. + TestMap.Builder builder = TestMap.newBuilder(); + defaultParser.merge(text, builder); + TestMap map = builder.build(); + assertThat(map.getInt32ToInt32Field().size()).isEqualTo(2); + assertThat(map.getInt32ToInt32Field().get(1).intValue()).isEqualTo(30); + } + + { + // With overwrite forbidden, same behavior. + // TODO(b/29122459): Expect parse exception here. + TestMap.Builder builder = TestMap.newBuilder(); + defaultParser.merge(text, builder); + TestMap map = builder.build(); + assertThat(map.getInt32ToInt32Field().size()).isEqualTo(2); + assertThat(map.getInt32ToInt32Field().get(1).intValue()).isEqualTo(30); + } + } + // ======================================================================= // test location information diff --git a/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto b/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto index 6eef42c5..b18b0d79 100644 --- a/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto +++ b/java/core/src/test/proto/com/google/protobuf/lite_equals_and_hash.proto @@ -46,6 +46,14 @@ message TestOneofEquals { message Foo { optional int32 value = 1; repeated Bar bar = 2; + map my_map = 3; + oneof Single { + sint64 sint64 = 4; + // LINT: ALLOW_GROUPS + group MyGroup = 5 { + optional int32 value = 1; + } + } extensions 100 to max; } diff --git a/java/lite/pom.xml b/java/lite/pom.xml index d7b15097..c902f819 100644 --- a/java/lite/pom.xml +++ b/java/lite/pom.xml @@ -137,6 +137,7 @@ **/MutabilityOracle.java **/NioByteString.java **/Parser.java + **/PrimitiveNonBoxingCollection.java **/ProtobufArrayList.java **/ProtocolStringList.java **/RopeByteString.java diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index 838700f7..5e0f5fe3 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -30,6 +30,7 @@ package com.google.protobuf.util; +import com.google.common.base.Preconditions; import com.google.common.io.BaseEncoding; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -102,7 +103,8 @@ public class JsonFormat { * Creates a {@link Printer} with default configurations. */ public static Printer printer() { - return new Printer(TypeRegistry.getEmptyTypeRegistry(), false, false, false); + return new Printer( + TypeRegistry.getEmptyTypeRegistry(), false, Collections.emptySet(), false, false); } /** @@ -110,16 +112,27 @@ public class JsonFormat { */ public static class Printer { private final TypeRegistry registry; - private final boolean includingDefaultValueFields; + // NOTE: There are 3 states for these *defaultValueFields variables: + // 1) Default - alwaysOutput is false & including is empty set. Fields only output if they are + // set to non-default values. + // 2) No-args includingDefaultValueFields() called - alwaysOutput is true & including is + // irrelevant (but set to empty set). All fields are output regardless of their values. + // 3) includingDefaultValueFields(Set) called - alwaysOutput is false & + // including is set to the specified set. Fields in that set are always output & fields not + // in that set are only output if set to non-default values. + private boolean alwaysOutputDefaultValueFields; + private Set includingDefaultValueFields; private final boolean preservingProtoFieldNames; private final boolean omittingInsignificantWhitespace; private Printer( TypeRegistry registry, - boolean includingDefaultValueFields, + boolean alwaysOutputDefaultValueFields, + Set includingDefaultValueFields, boolean preservingProtoFieldNames, boolean omittingInsignificantWhitespace) { this.registry = registry; + this.alwaysOutputDefaultValueFields = alwaysOutputDefaultValueFields; this.includingDefaultValueFields = includingDefaultValueFields; this.preservingProtoFieldNames = preservingProtoFieldNames; this.omittingInsignificantWhitespace = omittingInsignificantWhitespace; @@ -137,6 +150,7 @@ public class JsonFormat { } return new Printer( registry, + alwaysOutputDefaultValueFields, includingDefaultValueFields, preservingProtoFieldNames, omittingInsignificantWhitespace); @@ -149,8 +163,41 @@ public class JsonFormat { * {@link Printer}. */ public Printer includingDefaultValueFields() { + checkUnsetIncludingDefaultValueFields(); return new Printer( - registry, true, preservingProtoFieldNames, omittingInsignificantWhitespace); + registry, + true, + Collections.emptySet(), + preservingProtoFieldNames, + omittingInsignificantWhitespace); + } + + /** + * Creates a new {@link Printer} that will also print default-valued fields if their + * FieldDescriptors are found in the supplied set. Empty repeated fields and map fields will be + * printed as well, if they match. The new Printer clones all other configurations from the + * current {@link Printer}. Call includingDefaultValueFields() with no args to unconditionally + * output all fields. + */ + public Printer includingDefaultValueFields(Set fieldsToAlwaysOutput) { + Preconditions.checkArgument( + null != fieldsToAlwaysOutput && !fieldsToAlwaysOutput.isEmpty(), + "Non-empty Set must be supplied for includingDefaultValueFields."); + + checkUnsetIncludingDefaultValueFields(); + return new Printer( + registry, + false, + fieldsToAlwaysOutput, + preservingProtoFieldNames, + omittingInsignificantWhitespace); + } + + private void checkUnsetIncludingDefaultValueFields() { + if (alwaysOutputDefaultValueFields || !includingDefaultValueFields.isEmpty()) { + throw new IllegalStateException( + "JsonFormat includingDefaultValueFields has already been set."); + } } /** @@ -161,15 +208,20 @@ public class JsonFormat { */ public Printer preservingProtoFieldNames() { return new Printer( - registry, includingDefaultValueFields, true, omittingInsignificantWhitespace); + registry, + alwaysOutputDefaultValueFields, + includingDefaultValueFields, + true, + omittingInsignificantWhitespace); } /** - * Create a new {@link Printer} that will omit all insignificant whitespace - * in the JSON output. This new Printer clones all other configurations from the - * current Printer. Insignificant whitespace is defined by the JSON spec as whitespace - * that appear between JSON structural elements: + * Create a new {@link Printer} that will omit all insignificant whitespace in the JSON output. + * This new Printer clones all other configurations from the current Printer. Insignificant + * whitespace is defined by the JSON spec as whitespace that appear between JSON structural + * elements: + * * 

      * ws = *(
      * %x20 /              ; Space
@@ -177,18 +229,24 @@ public class JsonFormat {
      * %x0A /              ; Line feed or New line
      * %x0D )              ; Carriage return
      *
+ * * See https://tools.ietf.org/html/rfc7159 * current {@link Printer}. */ public Printer omittingInsignificantWhitespace() { - return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames, true); + return new Printer( + registry, + alwaysOutputDefaultValueFields, + includingDefaultValueFields, + preservingProtoFieldNames, + true); } /** * Converts a protobuf message to JSON format. * - * @throws InvalidProtocolBufferException if the message contains Any types - * that can't be resolved. + * @throws InvalidProtocolBufferException if the message contains Any types that can't be + * resolved. * @throws IOException if writing to the output fails. */ public void appendTo(MessageOrBuilder message, Appendable output) throws IOException { @@ -196,6 +254,7 @@ public class JsonFormat { // mobile. new PrinterImpl( registry, + alwaysOutputDefaultValueFields, includingDefaultValueFields, preservingProtoFieldNames, output, @@ -428,19 +487,16 @@ public class JsonFormat { this.output = output; } - /** - * ignored by compact printer - */ + /** ignored by compact printer */ + @Override public void indent() {} - /** - * ignored by compact printer - */ + /** ignored by compact printer */ + @Override public void outdent() {} - /** - * Print text to the output stream. - */ + /** Print text to the output stream. */ + @Override public void print(final CharSequence text) throws IOException { output.append(text); } @@ -458,18 +514,17 @@ public class JsonFormat { } /** - * Indent text by two spaces. After calling Indent(), two spaces will be - * inserted at the beginning of each line of text. Indent() may be called - * multiple times to produce deeper indents. + * Indent text by two spaces. After calling Indent(), two spaces will be inserted at the + * beginning of each line of text. Indent() may be called multiple times to produce deeper + * indents. */ + @Override public void indent() { indent.append(" "); } - /** - * Reduces the current indent level by two spaces, or crashes if the indent - * level is zero. - */ + /** Reduces the current indent level by two spaces, or crashes if the indent level is zero. */ + @Override public void outdent() { final int length = indent.length(); if (length < 2) { @@ -478,9 +533,8 @@ public class JsonFormat { indent.delete(length - 2, length); } - /** - * Print text to the output stream. - */ + /** Print text to the output stream. */ + @Override public void print(final CharSequence text) throws IOException { final int size = text.length(); int pos = 0; @@ -512,7 +566,8 @@ public class JsonFormat { */ private static final class PrinterImpl { private final TypeRegistry registry; - private final boolean includingDefaultValueFields; + private final boolean alwaysOutputDefaultValueFields; + private final Set includingDefaultValueFields; private final boolean preservingProtoFieldNames; private final TextGenerator generator; // We use Gson to help handle string escapes. @@ -526,11 +581,13 @@ public class JsonFormat { PrinterImpl( TypeRegistry registry, - boolean includingDefaultValueFields, + boolean alwaysOutputDefaultValueFields, + Set includingDefaultValueFields, boolean preservingProtoFieldNames, Appendable jsonOutput, boolean omittingInsignificantWhitespace) { this.registry = registry; + this.alwaysOutputDefaultValueFields = alwaysOutputDefaultValueFields; this.includingDefaultValueFields = includingDefaultValueFields; this.preservingProtoFieldNames = preservingProtoFieldNames; this.gson = GsonHolder.DEFAULT_GSON; @@ -781,23 +838,26 @@ public class JsonFormat { printedField = true; } Map fieldsToPrint = null; - if (includingDefaultValueFields) { - fieldsToPrint = new TreeMap(); + if (alwaysOutputDefaultValueFields || !includingDefaultValueFields.isEmpty()) { + fieldsToPrint = new TreeMap(message.getAllFields()); for (FieldDescriptor field : message.getDescriptorForType().getFields()) { if (field.isOptional()) { if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE - && !message.hasField(field)){ + && !message.hasField(field)) { // Always skip empty optional message fields. If not we will recurse indefinitely if // a message has itself as a sub-field. continue; } OneofDescriptor oneof = field.getContainingOneof(); if (oneof != null && !message.hasField(field)) { - // Skip all oneof fields except the one that is actually set + // Skip all oneof fields except the one that is actually set continue; } } - fieldsToPrint.put(field, message.getField(field)); + if (!fieldsToPrint.containsKey(field) + && (alwaysOutputDefaultValueFields || includingDefaultValueFields.contains(field))) { + fieldsToPrint.put(field, message.getField(field)); + } } } else { fieldsToPrint = message.getAllFields(); @@ -1451,45 +1511,6 @@ public class JsonFormat { } } - /** - * Gets the default value for a field type. Note that we use proto3 - * language defaults and ignore any default values set through the - * proto "default" option. - */ - private Object getDefaultValue(FieldDescriptor field, Message.Builder builder) { - switch (field.getType()) { - case INT32: - case SINT32: - case SFIXED32: - case UINT32: - case FIXED32: - return 0; - case INT64: - case SINT64: - case SFIXED64: - case UINT64: - case FIXED64: - return 0L; - case FLOAT: - return 0.0f; - case DOUBLE: - return 0.0; - case BOOL: - return false; - case STRING: - return ""; - case BYTES: - return ByteString.EMPTY; - case ENUM: - return field.getEnumType().getValues().get(0); - case MESSAGE: - case GROUP: - return builder.newBuilderForField(field).getDefaultInstanceForType(); - default: - throw new IllegalStateException("Invalid field type: " + field.getType()); - } - } - private void mergeRepeatedField( FieldDescriptor field, JsonElement json, Message.Builder builder) throws InvalidProtocolBufferException { diff --git a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java index 1d631a2c..d0bac417 100644 --- a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java +++ b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java @@ -297,7 +297,7 @@ public final class Timestamps { * Convert a Timestamp to the number of microseconds elapsed from the epoch. * *

The result will be rounded down to the nearest microsecond. E.g., if the timestamp - * represents "1969-12-31T23:59:59.999999999Z", it will be rounded to -1 millisecond. + * represents "1969-12-31T23:59:59.999999999Z", it will be rounded to -1 microsecond. */ public static long toMicros(Timestamp timestamp) { checkValid(timestamp); diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index de02c117..6943093f 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -34,6 +34,7 @@ import com.google.protobuf.Any; import com.google.protobuf.BoolValue; import com.google.protobuf.ByteString; import com.google.protobuf.BytesValue; +import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.DoubleValue; import com.google.protobuf.FloatValue; import com.google.protobuf.Int32Value; @@ -68,9 +69,12 @@ import java.io.Reader; import java.io.StringReader; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; import java.util.Map; +import java.util.Set; import junit.framework.TestCase; public class JsonFormatTest extends TestCase { @@ -284,8 +288,8 @@ public class JsonFormatTest extends TestCase { assertEquals(9012, message.getOptionalSint32()); assertEquals(3456, message.getOptionalFixed32()); assertEquals(7890, message.getOptionalSfixed32()); - assertEquals(1.5f, message.getOptionalFloat()); - assertEquals(1.25, message.getOptionalDouble()); + assertEquals(1.5f, message.getOptionalFloat(), 0.0f); + assertEquals(1.25, message.getOptionalDouble(), 0.0); assertEquals(true, message.getOptionalBool()); } @@ -1215,6 +1219,115 @@ public class JsonFormatTest extends TestCase { + "}", JsonFormat.printer().includingDefaultValueFields().print(message)); + Set fixedFields = new HashSet(); + for (FieldDescriptor fieldDesc : TestAllTypes.getDescriptor().getFields()) { + if (fieldDesc.getName().contains("_fixed")) { + fixedFields.add(fieldDesc); + } + } + + assertEquals( + "{\n" + + " \"optionalFixed32\": 0,\n" + + " \"optionalFixed64\": \"0\",\n" + + " \"repeatedFixed32\": [],\n" + + " \"repeatedFixed64\": []\n" + + "}", + JsonFormat.printer().includingDefaultValueFields(fixedFields).print(message)); + + TestAllTypes messageNonDefaults = + message.toBuilder().setOptionalInt64(1234).setOptionalFixed32(3232).build(); + assertEquals( + "{\n" + + " \"optionalInt64\": \"1234\",\n" + + " \"optionalFixed32\": 3232,\n" + + " \"optionalFixed64\": \"0\",\n" + + " \"repeatedFixed32\": [],\n" + + " \"repeatedFixed64\": []\n" + + "}", + JsonFormat.printer().includingDefaultValueFields(fixedFields).print(messageNonDefaults)); + + try { + JsonFormat.printer().includingDefaultValueFields().includingDefaultValueFields(); + fail("IllegalStateException is expected."); + } catch (IllegalStateException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + + try { + JsonFormat.printer().includingDefaultValueFields().includingDefaultValueFields(fixedFields); + fail("IllegalStateException is expected."); + } catch (IllegalStateException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + + try { + JsonFormat.printer().includingDefaultValueFields(fixedFields).includingDefaultValueFields(); + fail("IllegalStateException is expected."); + } catch (IllegalStateException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + + try { + JsonFormat.printer() + .includingDefaultValueFields(fixedFields) + .includingDefaultValueFields(fixedFields); + fail("IllegalStateException is expected."); + } catch (IllegalStateException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + + Set intFields = new HashSet(); + for (FieldDescriptor fieldDesc : TestAllTypes.getDescriptor().getFields()) { + if (fieldDesc.getName().contains("_int")) { + intFields.add(fieldDesc); + } + } + + try { + JsonFormat.printer() + .includingDefaultValueFields(intFields) + .includingDefaultValueFields(fixedFields); + fail("IllegalStateException is expected."); + } catch (IllegalStateException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + + try { + JsonFormat.printer().includingDefaultValueFields(null); + fail("IllegalArgumentException is expected."); + } catch (IllegalArgumentException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + + try { + JsonFormat.printer().includingDefaultValueFields(Collections.emptySet()); + fail("IllegalArgumentException is expected."); + } catch (IllegalArgumentException e) { + // Expected. + assertTrue( + "Exception message should mention includingDefaultValueFields.", + e.getMessage().contains("includingDefaultValueFields")); + } + TestMap mapMessage = TestMap.getDefaultInstance(); assertEquals("{\n}", JsonFormat.printer().print(mapMessage)); assertEquals( @@ -1283,16 +1396,17 @@ public class JsonFormatTest extends TestCase { assertEquals("{\n}", JsonFormat.printer().includingDefaultValueFields().print(oneofMessage)); oneofMessage = TestOneof.newBuilder().setOneofInt32(42).build(); - assertEquals("{\n \"oneofInt32\": 42\n}", - JsonFormat.printer().print(oneofMessage)); - assertEquals("{\n \"oneofInt32\": 42\n}", + assertEquals("{\n \"oneofInt32\": 42\n}", JsonFormat.printer().print(oneofMessage)); + assertEquals( + "{\n \"oneofInt32\": 42\n}", JsonFormat.printer().includingDefaultValueFields().print(oneofMessage)); TestOneof.Builder oneofBuilder = TestOneof.newBuilder(); mergeFromJson("{\n" + " \"oneofNullValue\": null \n" + "}", oneofBuilder); oneofMessage = oneofBuilder.build(); assertEquals("{\n \"oneofNullValue\": null\n}", JsonFormat.printer().print(oneofMessage)); - assertEquals("{\n \"oneofNullValue\": null\n}", + assertEquals( + "{\n \"oneofNullValue\": null\n}", JsonFormat.printer().includingDefaultValueFields().print(oneofMessage)); } @@ -1424,11 +1538,12 @@ public class JsonFormatTest extends TestCase { // Test that we are not leaking out JSON exceptions. public void testJsonException() throws Exception { - InputStream throwingInputStream = new InputStream() { - public int read() throws IOException { - throw new IOException("12345"); - } - }; + InputStream throwingInputStream = + new InputStream() { + public int read() throws IOException { + throw new IOException("12345"); + } + }; InputStreamReader throwingReader = new InputStreamReader(throwingInputStream); // When the underlying reader throws IOException, JsonFormat should forward // through this IOException. -- cgit v1.2.3