diff options
Diffstat (limited to 'java/core/src/main/java/com')
19 files changed, 1053 insertions, 135 deletions
diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java index 908764df..fc3c2a5d 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java @@ -125,6 +125,16 @@ public abstract class AbstractMessage protected int memoizedSize = -1; @Override + int getMemoizedSerializedSize() { + return memoizedSize; + } + + @Override + void setMemoizedSerializedSize(int size) { + memoizedSize = size; + } + + @Override public int getSerializedSize() { int size = memoizedSize; if (size != -1) { diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java index 24830c0a..b22bbaab 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java @@ -99,6 +99,16 @@ public abstract class AbstractMessageLite< codedOutput.flush(); } + // We'd like these to be abstract but some folks are extending this class directly. They shouldn't + // be doing that and they should feel bad. + int getMemoizedSerializedSize() { + throw new UnsupportedOperationException(); + } + + void setMemoizedSerializedSize(int size) { + throw new UnsupportedOperationException(); + } + /** * Package private helper method for AbstractParser to create diff --git a/java/core/src/main/java/com/google/protobuf/Android.java b/java/core/src/main/java/com/google/protobuf/Android.java new file mode 100644 index 00000000..cad54783 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/Android.java @@ -0,0 +1,57 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +final class Android { + + private static final Class<?> MEMORY_CLASS = getClassForName("libcore.io.Memory"); + private static final boolean IS_ROBOLECTRIC = + getClassForName("org.robolectric.Robolectric") != null; + + /** Returns {@code true} if running on an Android device. */ + static boolean isOnAndroidDevice() { + return MEMORY_CLASS != null && !IS_ROBOLECTRIC; + } + + /** Returns the memory class or {@code null} if not on Android device. */ + static Class<?> getMemoryClass() { + return MEMORY_CLASS; + } + + @SuppressWarnings("unchecked") + private static <T> Class<T> getClassForName(String name) { + try { + return (Class<T>) Class.forName(name); + } catch (Throwable e) { + return null; + } + } +} diff --git a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java index fd4c142b..4d7a9727 100644 --- a/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/BooleanArrayList.java @@ -82,6 +82,18 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean> } @Override + protected void removeRange(int fromIndex, int toIndex) { + ensureIsMutable(); + if (toIndex < fromIndex) { + throw new IndexOutOfBoundsException("toIndex < fromIndex"); + } + + System.arraycopy(array, toIndex, array, fromIndex, size - toIndex); + size -= (toIndex - fromIndex); + modCount++; + } + + @Override public boolean equals(Object o) { if (this == o) { return true; @@ -246,7 +258,9 @@ final class BooleanArrayList extends AbstractProtobufList<Boolean> ensureIsMutable(); ensureIndexInRange(index); boolean value = array[index]; - System.arraycopy(array, index + 1, array, index, size - index); + if (index < size - 1) { + System.arraycopy(array, index + 1, array, index, size - index); + } size--; modCount++; return value; diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java index 99a31209..d67bb54a 100644 --- a/java/core/src/main/java/com/google/protobuf/ByteString.java +++ b/java/core/src/main/java/com/google/protobuf/ByteString.java @@ -124,14 +124,8 @@ public abstract class ByteString implements Iterable<Byte>, Serializable { private static final ByteArrayCopier byteArrayCopier; static { - boolean isAndroid = true; - try { - Class.forName("android.content.Context"); - } catch (ClassNotFoundException e) { - isAndroid = false; - } - - byteArrayCopier = isAndroid ? new SystemByteArrayCopier() : new ArraysByteArrayCopier(); + byteArrayCopier = + Android.isOnAndroidDevice() ? new SystemByteArrayCopier() : new ArraysByteArrayCopier(); } /** diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index e08a993b..1297462e 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -64,6 +64,12 @@ public abstract class CodedInputStream { // Integer.MAX_VALUE == 0x7FFFFFF == INT_MAX from limits.h private static final int DEFAULT_SIZE_LIMIT = Integer.MAX_VALUE; + /** + * Whether to enable our custom UTF-8 decode codepath which does not use {@link StringCoding}. + * Currently disabled. + */ + private static final boolean ENABLE_CUSTOM_UTF8_DECODE = false; + /** Visible for subclasses. See setRecursionLimit() */ int recursionDepth; @@ -825,13 +831,19 @@ public abstract class CodedInputStream { public String readStringRequireUtf8() throws IOException { final int size = readRawVarint32(); if (size > 0 && size <= (limit - pos)) { - // TODO(martinrb): We could save a pass by validating while decoding. - if (!Utf8.isValidUtf8(buffer, pos, pos + size)) { - throw InvalidProtocolBufferException.invalidUtf8(); + if (ENABLE_CUSTOM_UTF8_DECODE) { + String result = Utf8.decodeUtf8(buffer, pos, size); + pos += size; + return result; + } else { + // TODO(martinrb): We could save a pass by validating while decoding. + if (!Utf8.isValidUtf8(buffer, pos, pos + size)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + final int tempPos = pos; + pos += size; + return new String(buffer, tempPos, size, UTF_8); } - final int tempPos = pos; - pos += size; - return new String(buffer, tempPos, size, UTF_8); } if (size == 0) { @@ -1524,6 +1536,8 @@ public abstract class CodedInputStream { final int size = readRawVarint32(); if (size > 0 && size <= remaining()) { // TODO(nathanmittler): Is there a way to avoid this copy? + // TODO(anuraaga): It might be possible to share the optimized loop with + // readStringRequireUtf8 by implementing Java replacement logic there. // The same as readBytes' logic byte[] bytes = new byte[size]; UnsafeUtil.copyMemory(pos, bytes, 0, size); @@ -1544,19 +1558,26 @@ public abstract class CodedInputStream { @Override public String readStringRequireUtf8() throws IOException { final int size = readRawVarint32(); - if (size >= 0 && size <= remaining()) { - // TODO(nathanmittler): Is there a way to avoid this copy? - // The same as readBytes' logic - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(pos, bytes, 0, size); - // TODO(martinrb): We could save a pass by validating while decoding. - if (!Utf8.isValidUtf8(bytes)) { - throw InvalidProtocolBufferException.invalidUtf8(); - } + if (size > 0 && size <= remaining()) { + if (ENABLE_CUSTOM_UTF8_DECODE) { + final int bufferPos = bufferPos(pos); + String result = Utf8.decodeUtf8(buffer, bufferPos, size); + pos += size; + return result; + } else { + // TODO(nathanmittler): Is there a way to avoid this copy? + // The same as readBytes' logic + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); + // TODO(martinrb): We could save a pass by validating while decoding. + if (!Utf8.isValidUtf8(bytes)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } - String result = new String(bytes, UTF_8); - pos += size; - return result; + String result = new String(bytes, UTF_8); + pos += size; + return result; + } } if (size == 0) { @@ -2324,11 +2345,15 @@ public abstract class CodedInputStream { bytes = readRawBytesSlowPath(size); tempPos = 0; } - // TODO(martinrb): We could save a pass by validating while decoding. - if (!Utf8.isValidUtf8(bytes, tempPos, tempPos + size)) { - throw InvalidProtocolBufferException.invalidUtf8(); + if (ENABLE_CUSTOM_UTF8_DECODE) { + return Utf8.decodeUtf8(bytes, tempPos, size); + } else { + // TODO(martinrb): We could save a pass by validating while decoding. + if (!Utf8.isValidUtf8(bytes, tempPos, tempPos + size)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + return new String(bytes, tempPos, size, UTF_8); } - return new String(bytes, tempPos, size, UTF_8); } @Override @@ -3348,23 +3373,34 @@ public abstract class CodedInputStream { public String readStringRequireUtf8() throws IOException { final int size = readRawVarint32(); if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { - byte[] bytes = new byte[size]; - UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); - if (!Utf8.isValidUtf8(bytes)) { - throw InvalidProtocolBufferException.invalidUtf8(); + if (ENABLE_CUSTOM_UTF8_DECODE) { + final int bufferPos = (int) (currentByteBufferPos - currentByteBufferStartPos); + String result = Utf8.decodeUtf8(currentByteBuffer, bufferPos, size); + currentByteBufferPos += size; + return result; + } else { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + if (!Utf8.isValidUtf8(bytes)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + String result = new String(bytes, UTF_8); + currentByteBufferPos += size; + return result; } - String result = new String(bytes, UTF_8); - currentByteBufferPos += size; - return result; } if (size >= 0 && size <= remaining()) { byte[] bytes = new byte[size]; readRawBytesTo(bytes, 0, size); - if (!Utf8.isValidUtf8(bytes)) { - throw InvalidProtocolBufferException.invalidUtf8(); + if (ENABLE_CUSTOM_UTF8_DECODE) { + return Utf8.decodeUtf8(bytes, 0, size); + } else { + if (!Utf8.isValidUtf8(bytes)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + String result = new String(bytes, UTF_8); + return result; } - String result = new String(bytes, UTF_8); - return result; } if (size == 0) { diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index 093a5f61..7b1ac651 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -377,6 +377,7 @@ public abstract class CodedOutputStream extends ByteOutput { public abstract void writeMessage(final int fieldNumber, final MessageLite value) throws IOException; + /** * Write a MessageSet extension field to the stream. For historical reasons, * the wire format differs from normal fields. @@ -481,6 +482,7 @@ public abstract class CodedOutputStream extends ByteOutput { // Abstract to avoid overhead of additional virtual method calls. public abstract void writeMessageNoTag(final MessageLite value) throws IOException; + //================================================================= @ExperimentalApi @@ -666,6 +668,7 @@ public abstract class CodedOutputStream extends ByteOutput { return computeTagSize(fieldNumber) + computeMessageSizeNoTag(value); } + /** * Compute the number of bytes that would be needed to encode a * MessageSet extension to the stream. For historical reasons, @@ -913,6 +916,7 @@ public abstract class CodedOutputStream extends ByteOutput { return computeLengthDelimitedFieldSize(value.getSerializedSize()); } + static int computeLengthDelimitedFieldSize(int fieldLength) { return computeUInt32SizeNoTag(fieldLength) + fieldLength; } @@ -1049,6 +1053,7 @@ public abstract class CodedOutputStream extends ByteOutput { writeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP); } + /** * Write a {@code group} field to the stream. * @@ -1059,6 +1064,7 @@ public abstract class CodedOutputStream extends ByteOutput { value.writeTo(this); } + /** * Compute the number of bytes that would be needed to encode a * {@code group} field, including tag. @@ -1070,6 +1076,7 @@ public abstract class CodedOutputStream extends ByteOutput { return computeTagSize(fieldNumber) * 2 + computeGroupSizeNoTag(value); } + /** * Compute the number of bytes that would be needed to encode a * {@code group} field. @@ -1079,6 +1086,7 @@ public abstract class CodedOutputStream extends ByteOutput { return value.getSerializedSize(); } + /** * Encode and write a varint. {@code value} is treated as * unsigned, so it won't be sign-extended if negative. @@ -1273,6 +1281,7 @@ public abstract class CodedOutputStream extends ByteOutput { writeMessageNoTag(value); } + @Override public final void writeMessageSetExtension(final int fieldNumber, final MessageLite value) throws IOException { @@ -1297,6 +1306,7 @@ public abstract class CodedOutputStream extends ByteOutput { value.writeTo(this); } + @Override public final void write(byte value) throws IOException { try { @@ -1608,6 +1618,7 @@ public abstract class CodedOutputStream extends ByteOutput { writeMessageNoTag(value); } + @Override public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) throws IOException { @@ -1632,6 +1643,7 @@ public abstract class CodedOutputStream extends ByteOutput { value.writeTo(this); } + @Override public void write(byte value) throws IOException { try { @@ -1928,6 +1940,7 @@ public abstract class CodedOutputStream extends ByteOutput { writeMessageNoTag(value); } + @Override public void writeMessageSetExtension(int fieldNumber, MessageLite value) throws IOException { writeTag(WireFormat.MESSAGE_SET_ITEM, WireFormat.WIRETYPE_START_GROUP); @@ -1950,6 +1963,7 @@ public abstract class CodedOutputStream extends ByteOutput { value.writeTo(this); } + @Override public void write(byte value) throws IOException { if (position >= limit) { @@ -2456,6 +2470,7 @@ public abstract class CodedOutputStream extends ByteOutput { writeMessageNoTag(value); } + @Override public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) throws IOException { @@ -2480,6 +2495,7 @@ public abstract class CodedOutputStream extends ByteOutput { value.writeTo(this); } + @Override public void write(byte value) throws IOException { if (position == limit) { @@ -2759,6 +2775,7 @@ public abstract class CodedOutputStream extends ByteOutput { writeMessageNoTag(value); } + @Override public void writeMessageSetExtension(final int fieldNumber, final MessageLite value) throws IOException { @@ -2783,6 +2800,7 @@ public abstract class CodedOutputStream extends ByteOutput { value.writeTo(this); } + @Override public void write(byte value) throws IOException { if (position == limit) { diff --git a/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java b/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java index 867b85ce..5b28b4a8 100644 --- a/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/DoubleArrayList.java @@ -82,6 +82,18 @@ final class DoubleArrayList extends AbstractProtobufList<Double> } @Override + protected void removeRange(int fromIndex, int toIndex) { + ensureIsMutable(); + if (toIndex < fromIndex) { + throw new IndexOutOfBoundsException("toIndex < fromIndex"); + } + + System.arraycopy(array, toIndex, array, fromIndex, size - toIndex); + size -= (toIndex - fromIndex); + modCount++; + } + + @Override public boolean equals(Object o) { if (this == o) { return true; @@ -247,7 +259,9 @@ final class DoubleArrayList extends AbstractProtobufList<Double> ensureIsMutable(); ensureIndexInRange(index); double value = array[index]; - System.arraycopy(array, index + 1, array, index, size - index); + if (index < size - 1) { + System.arraycopy(array, index + 1, array, index, size - index); + } size--; modCount++; return value; diff --git a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java index ba532021..a6a774b7 100644 --- a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java +++ b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java @@ -339,6 +339,20 @@ public final class DynamicMessage extends AbstractMessage { this.fields = FieldSet.newFieldSet(); this.unknownFields = UnknownFieldSet.getDefaultInstance(); this.oneofCases = new FieldDescriptor[type.toProto().getOneofDeclCount()]; + // A MapEntry has all of its fields present at all times. + if (type.getOptions().getMapEntry()) { + populateMapEntry(); + } + } + + private void populateMapEntry() { + for (FieldDescriptor field : type.getFields()) { + if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { + fields.setField(field, getDefaultInstance(field.getMessageType())); + } else { + fields.setField(field, field.getDefaultValue()); + } + } } // --------------------------------------------------------------- @@ -351,6 +365,10 @@ public final class DynamicMessage extends AbstractMessage { } else { fields.clear(); } + // A MapEntry has all of its fields present at all times. + if (type.getOptions().getMapEntry()) { + populateMapEntry(); + } unknownFields = UnknownFieldSet.getDefaultInstance(); return this; } diff --git a/java/core/src/main/java/com/google/protobuf/FieldSet.java b/java/core/src/main/java/com/google/protobuf/FieldSet.java index 8a9239ed..c09daa32 100644 --- a/java/core/src/main/java/com/google/protobuf/FieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/FieldSet.java @@ -102,6 +102,11 @@ final class FieldSet<FieldDescriptorType extends @SuppressWarnings("rawtypes") private static final FieldSet DEFAULT_INSTANCE = new FieldSet(true); + /** Returns {@code true} if empty, {@code false} otherwise. */ + boolean isEmpty() { + return fields.isEmpty(); + } + /** Make this FieldSet immutable from this point forward. */ @SuppressWarnings("unchecked") public void makeImmutable() { diff --git a/java/core/src/main/java/com/google/protobuf/FloatArrayList.java b/java/core/src/main/java/com/google/protobuf/FloatArrayList.java index 76645583..7c080af3 100644 --- a/java/core/src/main/java/com/google/protobuf/FloatArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/FloatArrayList.java @@ -82,6 +82,18 @@ final class FloatArrayList extends AbstractProtobufList<Float> } @Override + protected void removeRange(int fromIndex, int toIndex) { + ensureIsMutable(); + if (toIndex < fromIndex) { + throw new IndexOutOfBoundsException("toIndex < fromIndex"); + } + + System.arraycopy(array, toIndex, array, fromIndex, size - toIndex); + size -= (toIndex - fromIndex); + modCount++; + } + + @Override public boolean equals(Object o) { if (this == o) { return true; @@ -246,7 +258,9 @@ final class FloatArrayList extends AbstractProtobufList<Float> ensureIsMutable(); ensureIndexInRange(index); float value = array[index]; - System.arraycopy(array, index + 1, array, index, size - index); + if (index < size - 1) { + System.arraycopy(array, index + 1, array, index, size - index); + } size--; modCount++; return value; diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 09084646..df01547e 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -230,9 +230,13 @@ public abstract class GeneratedMessageLite< * Called by subclasses to complete parsing. For use by generated code only. */ protected void makeImmutable() { + // BEGIN REGULAR dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE); - unknownFields.makeImmutable(); + // END REGULAR + // BEGIN EXPERIMENTAL + // Protobuf.getInstance().schemaFor(this).makeImmutable(this); + // END EXPERIMENTAL } protected final < @@ -269,15 +273,15 @@ public abstract class GeneratedMessageLite< * For use by generated code only. */ public static enum MethodToInvoke { - IS_INITIALIZED, // BEGIN REGULAR + IS_INITIALIZED, VISIT, + MERGE_FROM_STREAM, + MAKE_IMMUTABLE, // END REGULAR // Rely on/modify instance state GET_MEMOIZED_IS_INITIALIZED, SET_MEMOIZED_IS_INITIALIZED, - MERGE_FROM_STREAM, - MAKE_IMMUTABLE, // Rely on static state NEW_MUTABLE_INSTANCE, @@ -339,6 +343,16 @@ public abstract class GeneratedMessageLite< } // END REGULAR + @Override + int getMemoizedSerializedSize() { + return memoizedSerializedSize; + } + + @Override + void setMemoizedSerializedSize(int size) { + memoizedSerializedSize = size; + } + /** @@ -449,13 +463,41 @@ public abstract class GeneratedMessageLite< } @Override + public BuilderType mergeFrom(byte[] input, int offset, int length) + throws InvalidProtocolBufferException { + // BEGIN REGULAR + return super.mergeFrom(input, offset, length); + // END REGULAR + // BEGIN EXPERIMENTAL + // copyOnWrite(); + // try { + // Protobuf.getInstance().schemaFor(instance).mergeFrom( + // instance, input, offset, offset + length, new ArrayDecoders.Registers()); + // } catch (InvalidProtocolBufferException e) { + // throw e; + // } catch (IndexOutOfBoundsException e) { + // throw InvalidProtocolBufferException.truncatedMessage(); + // } catch (IOException e) { + // throw new RuntimeException("Reading from byte array should not throw IOException.", e); + // } + // return (BuilderType) this; + // END EXPERIMENTAL + } + + @Override public BuilderType mergeFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws IOException { copyOnWrite(); try { + // BEGIN REGULAR instance.dynamicMethod(MethodToInvoke.MERGE_FROM_STREAM, input, extensionRegistry); + // END REGULAR + // BEGIN EXPERIMENTAL + // Protobuf.getInstance().schemaFor(instance).mergeFrom( + // instance, CodedInputStreamReader.forCodedInput(input), extensionRegistry); + // END EXPERIMENTAL } catch (RuntimeException e) { if (e.getCause() instanceof IOException) { throw (IOException) e.getCause(); @@ -504,11 +546,8 @@ public abstract class GeneratedMessageLite< extends GeneratedMessageLite<MessageType, BuilderType> implements ExtendableMessageOrBuilder<MessageType, BuilderType> { - /** - * Represents the set of extensions on this message. For use by generated - * code only. - */ - protected FieldSet<ExtensionDescriptor> extensions = FieldSet.newFieldSet(); + /** Represents the set of extensions on this message. For use by generated code only. */ + protected FieldSet<ExtensionDescriptor> extensions = FieldSet.emptySet(); @SuppressWarnings("unchecked") protected final void mergeExtensionFields(final MessageType other) { @@ -578,7 +617,9 @@ public abstract class GeneratedMessageLite< if (unknown) { // Unknown field or wrong wire type. Skip. return parseUnknownField(tag, input); } - + + ensureExtensionsAreMutable(); + if (packed) { int length = input.readRawVarint32(); int limit = input.pushLimit(length); @@ -793,10 +834,18 @@ public abstract class GeneratedMessageLite< if (subBuilder == null) { subBuilder = extension.getMessageDefaultInstance().newBuilderForType(); } - rawBytes.newCodedInput().readMessage(subBuilder, extensionRegistry); + subBuilder.mergeFrom(rawBytes, extensionRegistry); MessageLite value = subBuilder.build(); - extensions.setField(extension.descriptor, extension.singularToFieldSetType(value)); + ensureExtensionsAreMutable().setField( + extension.descriptor, extension.singularToFieldSetType(value)); + } + + private FieldSet<ExtensionDescriptor> ensureExtensionsAreMutable() { + if (extensions.isImmutable()) { + extensions = extensions.clone(); + } + return extensions; } private void verifyExtensionContainingType( @@ -868,10 +917,12 @@ public abstract class GeneratedMessageLite< @Override protected final void makeImmutable() { super.makeImmutable(); - + // BEGIN REGULAR extensions.makeImmutable(); + // END REGULAR } + /** * Used by subclasses to serialize extensions. Extension ranges may be * interleaved with field numbers, but we must write them in canonical @@ -942,12 +993,6 @@ public abstract class GeneratedMessageLite< implements ExtendableMessageOrBuilder<MessageType, BuilderType> { protected ExtendableBuilder(MessageType defaultInstance) { super(defaultInstance); - - // TODO(dweis): This is kind of an unnecessary clone since we construct a - // new instance in the parent constructor which makes the extensions - // immutable. This extra allocation shouldn't matter in practice - // though. - instance.extensions = instance.extensions.clone(); } // For immutable message conversion. @@ -966,6 +1011,15 @@ public abstract class GeneratedMessageLite< instance.extensions = instance.extensions.clone(); } + private FieldSet<ExtensionDescriptor> ensureExtensionsAreMutable() { + FieldSet<ExtensionDescriptor> extensions = instance.extensions; + if (extensions.isImmutable()) { + extensions = extensions.clone(); + instance.extensions = extensions; + } + return extensions; + } + @Override public final MessageType buildPartial() { if (isBuilt) { @@ -1024,7 +1078,8 @@ public abstract class GeneratedMessageLite< verifyExtensionContainingType(extensionLite); copyOnWrite(); - instance.extensions.setField(extensionLite.descriptor, extensionLite.toFieldSetType(value)); + ensureExtensionsAreMutable() + .setField(extensionLite.descriptor, extensionLite.toFieldSetType(value)); return (BuilderType) this; } @@ -1037,8 +1092,9 @@ public abstract class GeneratedMessageLite< verifyExtensionContainingType(extensionLite); copyOnWrite(); - instance.extensions.setRepeatedField( - extensionLite.descriptor, index, extensionLite.singularToFieldSetType(value)); + ensureExtensionsAreMutable() + .setRepeatedField( + extensionLite.descriptor, index, extensionLite.singularToFieldSetType(value)); return (BuilderType) this; } @@ -1051,8 +1107,8 @@ public abstract class GeneratedMessageLite< verifyExtensionContainingType(extensionLite); copyOnWrite(); - instance.extensions.addRepeatedField( - extensionLite.descriptor, extensionLite.singularToFieldSetType(value)); + ensureExtensionsAreMutable() + .addRepeatedField(extensionLite.descriptor, extensionLite.singularToFieldSetType(value)); return (BuilderType) this; } @@ -1063,7 +1119,7 @@ public abstract class GeneratedMessageLite< verifyExtensionContainingType(extensionLite); copyOnWrite(); - instance.extensions.clearField(extensionLite.descriptor); + ensureExtensionsAreMutable().clearField(extensionLite.descriptor); return (BuilderType) this; } } @@ -1462,8 +1518,13 @@ public abstract class GeneratedMessageLite< if (memoizedIsInitialized == 0) { return false; } + // BEGIN EXPERIMENTAL + // boolean isInitialized = Protobuf.getInstance().schemaFor(message).isInitialized(message); + // END EXPERIMENTAL + // BEGIN REGULAR boolean isInitialized = message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.FALSE) != null; + // END REGULAR if (shouldMemoize) { message.dynamicMethod( MethodToInvoke.SET_MEMOIZED_IS_INITIALIZED, isInitialized ? message : null); @@ -1471,10 +1532,6 @@ public abstract class GeneratedMessageLite< return isInitialized; } - protected static final <T extends GeneratedMessageLite<T, ?>> void makeImmutable(T message) { - message.dynamicMethod(MethodToInvoke.MAKE_IMMUTABLE); - } - protected static IntList emptyIntList() { return IntArrayList.emptyList(); } @@ -1554,6 +1611,11 @@ public abstract class GeneratedMessageLite< throws InvalidProtocolBufferException { return GeneratedMessageLite.parsePartialFrom(defaultInstance, input, extensionRegistry); } + + @Override + public T parsePartialFrom(byte[] input) throws InvalidProtocolBufferException { + return GeneratedMessageLite.parsePartialFrom(defaultInstance, input); + } } /** @@ -1567,8 +1629,21 @@ public abstract class GeneratedMessageLite< @SuppressWarnings("unchecked") // Guaranteed by protoc T result = (T) instance.dynamicMethod(MethodToInvoke.NEW_MUTABLE_INSTANCE); try { + // BEGIN REGULAR result.dynamicMethod(MethodToInvoke.MERGE_FROM_STREAM, input, extensionRegistry); + // END REGULAR + // BEGIN EXPERIMENTAL + // Protobuf.getInstance().schemaFor(result).mergeFrom( + // result, CodedInputStreamReader.forCodedInput(input), extensionRegistry); + // END EXPERIMENTAL result.makeImmutable(); + // BEGIN EXPERIMENTAL + // } catch (IOException e) { + // if (e.getCause() instanceof InvalidProtocolBufferException) { + // throw (InvalidProtocolBufferException) e.getCause(); + // } + // throw new InvalidProtocolBufferException(e.getMessage()).setUnfinishedMessage(result); + // END EXPERIMENTAL } catch (RuntimeException e) { if (e.getCause() instanceof InvalidProtocolBufferException) { throw (InvalidProtocolBufferException) e.getCause(); @@ -1578,6 +1653,34 @@ public abstract class GeneratedMessageLite< return result; } + /** A static helper method for parsing a partial from byte array. */ + static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom(T instance, byte[] input) + throws InvalidProtocolBufferException { + // BEGIN REGULAR + return parsePartialFrom(instance, input, ExtensionRegistryLite.getEmptyRegistry()); + // END REGULAR + // BEGIN EXPERIMENTAL + // @SuppressWarnings("unchecked") // Guaranteed by protoc + // T result = (T) instance.dynamicMethod(MethodToInvoke.NEW_MUTABLE_INSTANCE); + // try { + // Protobuf.getInstance().schemaFor(result).mergeFrom( + // result, input, 0, input.length, new ArrayDecoders.Registers()); + // result.makeImmutable(); + // if (result.memoizedHashCode != 0) { + // throw new RuntimeException(); + // } + // } catch (IOException e) { + // if (e.getCause() instanceof InvalidProtocolBufferException) { + // throw (InvalidProtocolBufferException) e.getCause(); + // } + // throw new InvalidProtocolBufferException(e.getMessage()).setUnfinishedMessage(result); + // } catch (IndexOutOfBoundsException e) { + // throw InvalidProtocolBufferException.truncatedMessage().setUnfinishedMessage(result); + // } + // return result; + // END EXPERIMENTAL + } + protected static <T extends GeneratedMessageLite<T, ?>> T parsePartialFrom( T defaultInstance, CodedInputStream input) @@ -1674,8 +1777,7 @@ public abstract class GeneratedMessageLite< protected static <T extends GeneratedMessageLite<T, ?>> T parseFrom( T defaultInstance, byte[] data) throws InvalidProtocolBufferException { - return checkMessageInitialized( - parsePartialFrom(defaultInstance, data, ExtensionRegistryLite.getEmptyRegistry())); + return checkMessageInitialized(parsePartialFrom(defaultInstance, data)); } // Validates last tag. diff --git a/java/core/src/main/java/com/google/protobuf/IntArrayList.java b/java/core/src/main/java/com/google/protobuf/IntArrayList.java index aff5c21b..aacd71e1 100644 --- a/java/core/src/main/java/com/google/protobuf/IntArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/IntArrayList.java @@ -82,6 +82,18 @@ final class IntArrayList extends AbstractProtobufList<Integer> } @Override + protected void removeRange(int fromIndex, int toIndex) { + ensureIsMutable(); + if (toIndex < fromIndex) { + throw new IndexOutOfBoundsException("toIndex < fromIndex"); + } + + System.arraycopy(array, toIndex, array, fromIndex, size - toIndex); + size -= (toIndex - fromIndex); + modCount++; + } + + @Override public boolean equals(Object o) { if (this == o) { return true; @@ -246,7 +258,9 @@ final class IntArrayList extends AbstractProtobufList<Integer> ensureIsMutable(); ensureIndexInRange(index); int value = array[index]; - System.arraycopy(array, index + 1, array, index, size - index); + if (index < size - 1) { + System.arraycopy(array, index + 1, array, index, size - index); + } size--; modCount++; return value; diff --git a/java/core/src/main/java/com/google/protobuf/LongArrayList.java b/java/core/src/main/java/com/google/protobuf/LongArrayList.java index fc146e23..95945cb7 100644 --- a/java/core/src/main/java/com/google/protobuf/LongArrayList.java +++ b/java/core/src/main/java/com/google/protobuf/LongArrayList.java @@ -82,6 +82,18 @@ final class LongArrayList extends AbstractProtobufList<Long> } @Override + protected void removeRange(int fromIndex, int toIndex) { + ensureIsMutable(); + if (toIndex < fromIndex) { + throw new IndexOutOfBoundsException("toIndex < fromIndex"); + } + + System.arraycopy(array, toIndex, array, fromIndex, size - toIndex); + size -= (toIndex - fromIndex); + modCount++; + } + + @Override public boolean equals(Object o) { if (this == o) { return true; @@ -246,7 +258,9 @@ final class LongArrayList extends AbstractProtobufList<Long> ensureIsMutable(); ensureIndexInRange(index); long value = array[index]; - System.arraycopy(array, index + 1, array, index, size - index); + if (index < size - 1) { + System.arraycopy(array, index + 1, array, index, size - index); + } size--; modCount++; return value; diff --git a/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java b/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java index 23373ef4..8e265935 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java +++ b/java/core/src/main/java/com/google/protobuf/MessageLiteToString.java @@ -31,6 +31,7 @@ package com.google.protobuf; import java.lang.reflect.Method; +import java.lang.reflect.Modifier; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -38,20 +39,18 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -/** - * Helps generate {@link String} representations of {@link MessageLite} protos. - */ -// TODO(dweis): Fix map fields. +/** Helps generate {@link String} representations of {@link MessageLite} protos. */ final class MessageLiteToString { private static final String LIST_SUFFIX = "List"; private static final String BUILDER_LIST_SUFFIX = "OrBuilderList"; + private static final String MAP_SUFFIX = "Map"; private static final String BYTES_SUFFIX = "Bytes"; - + /** - * Returns a {@link String} representation of the {@link MessageLite} object. The first line of + * Returns a {@link String} representation of the {@link MessageLite} object. The first line of * the {@code String} representation representation includes a comment string to uniquely identify - * the objcet instance. This acts as an indicator that this should not be relied on for + * the object instance. This acts as an indicator that this should not be relied on for * comparisons. * * <p>For use by generated code only. @@ -71,8 +70,9 @@ final class MessageLiteToString { */ private static void reflectivePrintWithIndent( MessageLite messageLite, StringBuilder buffer, int indent) { - // Build a map of method name to method. We're looking for methods like getFoo(), hasFoo(), and - // getFooList() which might be useful for building an object's string representation. + // Build a map of method name to method. We're looking for methods like getFoo(), hasFoo(), + // getFooList() and getFooMap() which might be useful for building an object's string + // representation. Map<String, Method> nameToNoArgMethod = new HashMap<String, Method>(); Map<String, Method> nameToMethod = new HashMap<String, Method>(); Set<String> getters = new TreeSet<String>(); @@ -89,12 +89,16 @@ final class MessageLiteToString { for (String getter : getters) { String suffix = getter.replaceFirst("get", ""); - if (suffix.endsWith(LIST_SUFFIX) && !suffix.endsWith(BUILDER_LIST_SUFFIX)) { - String camelCase = suffix.substring(0, 1).toLowerCase() - + suffix.substring(1, suffix.length() - LIST_SUFFIX.length()); + if (suffix.endsWith(LIST_SUFFIX) + && !suffix.endsWith(BUILDER_LIST_SUFFIX) + // Sometimes people have fields named 'list' that aren't repeated. + && !suffix.equals(LIST_SUFFIX)) { + String camelCase = + suffix.substring(0, 1).toLowerCase() + + suffix.substring(1, suffix.length() - LIST_SUFFIX.length()); // Try to reflectively get the value and toString() the field as if it were repeated. This - // only works if the method names have not be proguarded out or renamed. - Method listMethod = nameToNoArgMethod.get("get" + suffix); + // only works if the method names have not been proguarded out or renamed. + Method listMethod = nameToNoArgMethod.get(getter); if (listMethod != null && listMethod.getReturnType().equals(List.class)) { printField( buffer, @@ -104,6 +108,30 @@ final class MessageLiteToString { continue; } } + if (suffix.endsWith(MAP_SUFFIX) + // Sometimes people have fields named 'map' that aren't maps. + && !suffix.equals(MAP_SUFFIX)) { + String camelCase = + suffix.substring(0, 1).toLowerCase() + + suffix.substring(1, suffix.length() - MAP_SUFFIX.length()); + // Try to reflectively get the value and toString() the field as if it were a map. This only + // works if the method names have not been proguarded out or renamed. + Method mapMethod = nameToNoArgMethod.get(getter); + if (mapMethod != null + && mapMethod.getReturnType().equals(Map.class) + // Skip the deprecated getter method with no prefix "Map" when the field name ends with + // "map". + && !mapMethod.isAnnotationPresent(Deprecated.class) + // Skip the internal mutable getter method. + && Modifier.isPublic(mapMethod.getModifiers())) { + printField( + buffer, + indent, + camelCaseToSnakeCase(camelCase), + GeneratedMessageLite.invokeOrDie(mapMethod, messageLite)); + continue; + } + } Method setter = nameToMethod.get("set" + suffix); if (setter == null) { @@ -119,22 +147,19 @@ final class MessageLiteToString { String camelCase = suffix.substring(0, 1).toLowerCase() + suffix.substring(1); // Try to reflectively get the value and toString() the field as if it were optional. This - // only works if the method names have not be proguarded out or renamed. + // only works if the method names have not been proguarded out or renamed. Method getMethod = nameToNoArgMethod.get("get" + suffix); Method hasMethod = nameToNoArgMethod.get("has" + suffix); // TODO(dweis): Fix proto3 semantics. if (getMethod != null) { Object value = GeneratedMessageLite.invokeOrDie(getMethod, messageLite); - final boolean hasValue = hasMethod == null - ? !isDefaultValue(value) - : (Boolean) GeneratedMessageLite.invokeOrDie(hasMethod, messageLite); - // TODO(dweis): This doesn't stop printing oneof case twice: value and enum style. + final boolean hasValue = + hasMethod == null + ? !isDefaultValue(value) + : (Boolean) GeneratedMessageLite.invokeOrDie(hasMethod, messageLite); + // TODO(dweis): This doesn't stop printing oneof case twice: value and enum style. if (hasValue) { - printField( - buffer, - indent, - camelCaseToSnakeCase(camelCase), - value); + printField(buffer, indent, camelCaseToSnakeCase(camelCase), value); } continue; } @@ -153,7 +178,7 @@ final class MessageLiteToString { ((GeneratedMessageLite<?, ?>) messageLite).unknownFields.printWithIndent(buffer, indent); } } - + private static boolean isDefaultValue(Object o) { if (o instanceof Boolean) { return !((Boolean) o); @@ -179,7 +204,7 @@ final class MessageLiteToString { if (o instanceof java.lang.Enum<?>) { // Catches oneof enums. return ((java.lang.Enum<?>) o).ordinal() == 0; } - + return false; } @@ -201,6 +226,13 @@ final class MessageLiteToString { } return; } + if (object instanceof Map<?, ?>) { + Map<?, ?> map = (Map<?, ?>) object; + for (Map.Entry<?, ?> entry : map.entrySet()) { + printField(buffer, indent, name, entry); + } + return; + } buffer.append('\n'); for (int i = 0; i < indent; i++) { @@ -220,11 +252,21 @@ final class MessageLiteToString { buffer.append(' '); } buffer.append("}"); + } else if (object instanceof Map.Entry<?, ?>) { + buffer.append(" {"); + Map.Entry<?, ?> entry = (Map.Entry<?, ?>) object; + printField(buffer, indent + 2, "key", entry.getKey()); + printField(buffer, indent + 2, "value", entry.getValue()); + buffer.append("\n"); + for (int i = 0; i < indent; i++) { + buffer.append(' '); + } + buffer.append("}"); } else { buffer.append(": ").append(object.toString()); } } - + private static final String camelCaseToSnakeCase(String camelCase) { StringBuilder builder = new StringBuilder(); for (int i = 0; i < camelCase.length(); i++) { diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index ab9acf2f..25c3474f 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -987,7 +987,7 @@ public final class TextFormat { nextToken(); return false; } else { - throw parseException("Expected \"true\" or \"false\"."); + throw parseException("Expected \"true\" or \"false\". Found \"" + currentToken + "\"."); } } @@ -1224,6 +1224,22 @@ public final class TextFormat { } /** + * Parse a text-format message from {@code input}. + * + * @return the parsed message, guaranteed initialized + */ + public static <T extends Message> T parse(final CharSequence input, + final Class<T> protoClass) + throws ParseException { + Message.Builder builder = + Internal.getDefaultInstance(protoClass).newBuilderForType(); + merge(input, builder); + @SuppressWarnings("unchecked") + T output = (T) builder.build(); + return output; + } + + /** * Parse a text-format message from {@code input} and merge the contents * into {@code builder}. Extensions will be recognized if they are * registered in {@code extensionRegistry}. @@ -1248,6 +1264,25 @@ public final class TextFormat { PARSER.merge(input, extensionRegistry, builder); } + /** + * Parse a text-format message from {@code input}. Extensions will be + * recognized if they are registered in {@code extensionRegistry}. + * + * @return the parsed message, guaranteed initialized + */ + public static <T extends Message> T parse( + final CharSequence input, + final ExtensionRegistry extensionRegistry, + final Class<T> protoClass) + throws ParseException { + Message.Builder builder = + Internal.getDefaultInstance(protoClass).newBuilderForType(); + merge(input, extensionRegistry, builder); + @SuppressWarnings("unchecked") + T output = (T) builder.build(); + return output; + } + /** * Parser for text-format proto2 instances. This class is thread-safe. @@ -1276,13 +1311,17 @@ public final class TextFormat { } private final boolean allowUnknownFields; + private final boolean allowUnknownEnumValues; private final SingularOverwritePolicy singularOverwritePolicy; private TextFormatParseInfoTree.Builder parseInfoTreeBuilder; private Parser( - boolean allowUnknownFields, SingularOverwritePolicy singularOverwritePolicy, + boolean allowUnknownFields, + boolean allowUnknownEnumValues, + SingularOverwritePolicy singularOverwritePolicy, TextFormatParseInfoTree.Builder parseInfoTreeBuilder) { this.allowUnknownFields = allowUnknownFields; + this.allowUnknownEnumValues = allowUnknownEnumValues; this.singularOverwritePolicy = singularOverwritePolicy; this.parseInfoTreeBuilder = parseInfoTreeBuilder; } @@ -1299,6 +1338,7 @@ public final class TextFormat { */ public static class Builder { private boolean allowUnknownFields = false; + private boolean allowUnknownEnumValues = false; private SingularOverwritePolicy singularOverwritePolicy = SingularOverwritePolicy.ALLOW_SINGULAR_OVERWRITES; private TextFormatParseInfoTree.Builder parseInfoTreeBuilder = null; @@ -1320,7 +1360,10 @@ public final class TextFormat { public Parser build() { return new Parser( - allowUnknownFields, singularOverwritePolicy, parseInfoTreeBuilder); + allowUnknownFields, + allowUnknownEnumValues, + singularOverwritePolicy, + parseInfoTreeBuilder); } } @@ -1384,7 +1427,7 @@ public final class TextFormat { return text; } - // Check both unknown fields and unknown extensions and log warming messages + // Check both unknown fields and unknown extensions and log warning messages // or throw exceptions according to the flag. private void checkUnknownFields(final List<String> unknownFields) throws ParseException { @@ -1702,17 +1745,40 @@ public final class TextFormat { final int number = tokenizer.consumeInt32(); value = enumType.findValueByNumber(number); if (value == null) { - throw tokenizer.parseExceptionPreviousToken( - "Enum type \"" + enumType.getFullName() - + "\" has no value with number " + number + '.'); + String unknownValueMsg = + "Enum type \"" + + enumType.getFullName() + + "\" has no value with number " + + number + + '.'; + if (allowUnknownEnumValues) { + logger.warning(unknownValueMsg); + return; + } else { + throw tokenizer.parseExceptionPreviousToken( + "Enum type \"" + + enumType.getFullName() + + "\" has no value with number " + + number + + '.'); + } } } else { final String id = tokenizer.consumeIdentifier(); value = enumType.findValueByName(id); if (value == null) { - throw tokenizer.parseExceptionPreviousToken( - "Enum type \"" + enumType.getFullName() - + "\" has no value named \"" + id + "\"."); + String unknownValueMsg = + "Enum type \"" + + enumType.getFullName() + + "\" has no value named \"" + + id + + "\"."; + if (allowUnknownEnumValues) { + logger.warning(unknownValueMsg); + return; + } else { + throw tokenizer.parseExceptionPreviousToken(unknownValueMsg); + } } } diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java index 2a614c84..f0b919ad 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java @@ -295,14 +295,30 @@ public final class UnknownFieldSetLite { return true; } + private static int hashCode(int[] tags, int count) { + int hashCode = 17; + for (int i = 0; i < count; ++i) { + hashCode = 31 * hashCode + tags[i]; + } + return hashCode; + } + + private static int hashCode(Object[] objects, int count) { + int hashCode = 17; + for (int i = 0; i < count; ++i) { + hashCode = 31 * hashCode + objects[i].hashCode(); + } + return hashCode; + } + @Override public int hashCode() { int hashCode = 17; - + hashCode = 31 * hashCode + count; - hashCode = 31 * hashCode + Arrays.hashCode(tags); - hashCode = 31 * hashCode + Arrays.deepHashCode(objects); - + hashCode = 31 * hashCode + hashCode(tags, count); + hashCode = 31 * hashCode + hashCode(objects, count); + return hashCode; } diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java index 88315cb6..d84ef3c5 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java @@ -33,7 +33,6 @@ package com.google.protobuf; import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedExceptionAction; import java.util.logging.Level; @@ -82,6 +81,7 @@ final class UnsafeUtil { return HAS_UNSAFE_BYTEBUFFER_OPERATIONS; } + static long objectFieldOffset(Field field) { return MEMORY_ACCESSOR.objectFieldOffset(field); } @@ -146,10 +146,6 @@ final class UnsafeUtil { return MEMORY_ACCESSOR.getObject(target, offset); } - static void putObject(Object target, long offset, Object value) { - MEMORY_ACCESSOR.putObject(target, offset, value); - } - static byte getByte(byte[] target, long index) { return MEMORY_ACCESSOR.getByte(target, BYTE_ARRAY_BASE_OFFSET + index); } @@ -266,7 +262,7 @@ final class UnsafeUtil { /** * Gets the {@code sun.misc.Unsafe} instance, or {@code null} if not available on this platform. */ - private static sun.misc.Unsafe getUnsafe() { + static sun.misc.Unsafe getUnsafe() { sun.misc.Unsafe unsafe = null; try { unsafe = @@ -346,6 +342,10 @@ final class UnsafeUtil { clazz.getMethod("objectFieldOffset", Field.class); clazz.getMethod("getLong", Object.class, long.class); + if (bufferAddressField() == null) { + return false; + } + clazz.getMethod("getByte", long.class); clazz.getMethod("putByte", long.class, byte.class); clazz.getMethod("getInt", long.class); @@ -364,18 +364,16 @@ final class UnsafeUtil { } - @SuppressWarnings("unchecked") - private static <T> Class<T> getClassForName(String name) { - try { - return (Class<T>) Class.forName(name); - } catch (Throwable e) { - return null; - } - } - /** Finds the address field within a direct {@link Buffer}. */ private static Field bufferAddressField() { - return field(Buffer.class, "address"); + Field field = field(Buffer.class, "address"); + return field != null && field.getType() == long.class ? field : null; + } + + /** Finds the value field within a {@link String}. */ + private static Field stringValueField() { + Field field = field(String.class, "value"); + return field != null && field.getType() == char[].class ? field : null; } /** diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index 1b136144..de75fe6b 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -34,11 +34,15 @@ import static com.google.protobuf.UnsafeUtil.addressOffset; import static com.google.protobuf.UnsafeUtil.hasUnsafeArrayOperations; import static com.google.protobuf.UnsafeUtil.hasUnsafeByteBufferOperations; import static java.lang.Character.MAX_SURROGATE; +import static java.lang.Character.MIN_HIGH_SURROGATE; +import static java.lang.Character.MIN_LOW_SURROGATE; +import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT; import static java.lang.Character.MIN_SURROGATE; import static java.lang.Character.isSurrogatePair; import static java.lang.Character.toCodePoint; import java.nio.ByteBuffer; +import java.util.Arrays; /** * A set of low-level, high-performance static utility methods related @@ -289,7 +293,7 @@ final class Utf8 { if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) { // Check that we have a well-formed surrogate pair. int cp = Character.codePointAt(sequence, i); - if (cp < Character.MIN_SUPPLEMENTARY_CODE_POINT) { + if (cp < MIN_SUPPLEMENTARY_CODE_POINT) { throw new UnpairedSurrogateException(i, utf16Length); } i++; @@ -331,6 +335,26 @@ final class Utf8 { } /** + * Decodes the given UTF-8 portion of the {@link ByteBuffer} into a {@link String}. + * + * @throws InvalidProtocolBufferException if the input is not valid UTF-8. + */ + static String decodeUtf8(ByteBuffer buffer, int index, int size) + throws InvalidProtocolBufferException { + return processor.decodeUtf8(buffer, index, size); + } + + /** + * Decodes the given UTF-8 encoded byte array slice into a {@link String}. + * + * @throws InvalidProtocolBufferException if the input is not valid UTF-8. + */ + static String decodeUtf8(byte[] bytes, int index, int size) + throws InvalidProtocolBufferException { + return processor.decodeUtf8(bytes, index, size); + } + + /** * Encodes the given characters to the target {@link ByteBuffer} using UTF-8 encoding. * * <p>Selects an optimal algorithm based on the type of {@link ByteBuffer} (i.e. heap or direct) @@ -610,6 +634,116 @@ final class Utf8 { } /** + * Decodes the given byte array slice into a {@link String}. + * + * @throws InvalidProtocolBufferException if the byte array slice is not valid UTF-8. + */ + abstract String decodeUtf8(byte[] bytes, int index, int size) + throws InvalidProtocolBufferException; + + /** + * Decodes the given portion of the {@link ByteBuffer} into a {@link String}. + * + * @throws InvalidProtocolBufferException if the portion of the buffer is not valid UTF-8. + */ + final String decodeUtf8(ByteBuffer buffer, int index, int size) + throws InvalidProtocolBufferException { + if (buffer.hasArray()) { + final int offset = buffer.arrayOffset(); + return decodeUtf8(buffer.array(), offset + index, size); + } else if (buffer.isDirect()) { + return decodeUtf8Direct(buffer, index, size); + } + return decodeUtf8Default(buffer, index, size); + } + + /** + * Decodes direct {@link ByteBuffer} instances into {@link String}. + */ + abstract String decodeUtf8Direct(ByteBuffer buffer, int index, int size) + throws InvalidProtocolBufferException; + + /** + * Decodes {@link ByteBuffer} instances using the {@link ByteBuffer} API rather than + * potentially faster approaches. + */ + final String decodeUtf8Default(ByteBuffer buffer, int index, int size) + throws InvalidProtocolBufferException { + // Bitwise OR combines the sign bits so any negative value fails the check. + if ((index | size | buffer.limit() - index - size) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, size)); + } + + int offset = index; + final int limit = offset + size; + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + char[] resultArr = new char[size]; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (offset < limit) { + byte b = buffer.get(offset); + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + + while (offset < limit) { + byte byte1 = buffer.get(offset++); + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (offset < limit) { + byte b = buffer.get(offset); + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (offset >= limit) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleTwoBytes( + byte1, /* byte2 */ buffer.get(offset++), resultArr, resultPos++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (offset >= limit - 1) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleThreeBytes( + byte1, + /* byte2 */ buffer.get(offset++), + /* byte3 */ buffer.get(offset++), + resultArr, + resultPos++); + } else { + if (offset >= limit - 2) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleFourBytes( + byte1, + /* byte2 */ buffer.get(offset++), + /* byte3 */ buffer.get(offset++), + /* byte4 */ buffer.get(offset++), + resultArr, + resultPos++); + // 4-byte case requires two chars. + resultPos++; + } + } + + return new String(resultArr, 0, resultPos); + } + + /** * Encodes an input character sequence ({@code in}) to UTF-8 in the target array ({@code out}). * For a string, this method is similar to * <pre>{@code @@ -851,6 +985,88 @@ final class Utf8 { } @Override + String decodeUtf8(byte[] bytes, int index, int size) throws InvalidProtocolBufferException { + // Bitwise OR combines the sign bits so any negative value fails the check. + if ((index | size | bytes.length - index - size) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size)); + } + + int offset = index; + final int limit = offset + size; + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + char[] resultArr = new char[size]; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (offset < limit) { + byte b = bytes[offset]; + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + + while (offset < limit) { + byte byte1 = bytes[offset++]; + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (offset < limit) { + byte b = bytes[offset]; + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (offset >= limit) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleTwoBytes(byte1, /* byte2 */ bytes[offset++], resultArr, resultPos++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (offset >= limit - 1) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleThreeBytes( + byte1, + /* byte2 */ bytes[offset++], + /* byte3 */ bytes[offset++], + resultArr, + resultPos++); + } else { + if (offset >= limit - 2) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleFourBytes( + byte1, + /* byte2 */ bytes[offset++], + /* byte3 */ bytes[offset++], + /* byte4 */ bytes[offset++], + resultArr, + resultPos++); + // 4-byte case requires two chars. + resultPos++; + } + } + + return new String(resultArr, 0, resultPos); + } + + @Override + String decodeUtf8Direct(ByteBuffer buffer, int index, int size) + throws InvalidProtocolBufferException { + // For safe processing, we have to use the ByteBufferAPI. + return decodeUtf8Default(buffer, index, size); + } + + @Override int encodeUtf8(CharSequence in, byte[] out, int offset, int length) { int utf16Length = in.length(); int j = offset; @@ -996,6 +1212,7 @@ final class Utf8 { @Override int partialIsValidUtf8(int state, byte[] bytes, final int index, final int limit) { + // Bitwise OR combines the sign bits so any negative value fails the check. if ((index | limit | bytes.length - limit) < 0) { throw new ArrayIndexOutOfBoundsException( String.format("Array length=%d, index=%d, limit=%d", bytes.length, index, limit)); @@ -1091,6 +1308,7 @@ final class Utf8 { @Override int partialIsValidUtf8Direct( final int state, ByteBuffer buffer, final int index, final int limit) { + // Bitwise OR combines the sign bits so any negative value fails the check. if ((index | limit | buffer.limit() - limit) < 0) { throw new ArrayIndexOutOfBoundsException( String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, limit)); @@ -1185,6 +1403,157 @@ final class Utf8 { } @Override + String decodeUtf8(byte[] bytes, int index, int size) throws InvalidProtocolBufferException { + if ((index | size | bytes.length - index - size) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer length=%d, index=%d, size=%d", bytes.length, index, size)); + } + + int offset = index; + final int limit = offset + size; + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + char[] resultArr = new char[size]; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (offset < limit) { + byte b = UnsafeUtil.getByte(bytes, offset); + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + + while (offset < limit) { + byte byte1 = UnsafeUtil.getByte(bytes, offset++); + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (offset < limit) { + byte b = UnsafeUtil.getByte(bytes, offset); + if (!DecodeUtil.isOneByte(b)) { + break; + } + offset++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (offset >= limit) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleTwoBytes( + byte1, /* byte2 */ UnsafeUtil.getByte(bytes, offset++), resultArr, resultPos++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (offset >= limit - 1) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleThreeBytes( + byte1, + /* byte2 */ UnsafeUtil.getByte(bytes, offset++), + /* byte3 */ UnsafeUtil.getByte(bytes, offset++), + resultArr, + resultPos++); + } else { + if (offset >= limit - 2) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleFourBytes( + byte1, + /* byte2 */ UnsafeUtil.getByte(bytes, offset++), + /* byte3 */ UnsafeUtil.getByte(bytes, offset++), + /* byte4 */ UnsafeUtil.getByte(bytes, offset++), + resultArr, + resultPos++); + // 4-byte case requires two chars. + resultPos++; + } + } + + return new String(resultArr, 0, resultPos); + } + + @Override + String decodeUtf8Direct(ByteBuffer buffer, int index, int size) + throws InvalidProtocolBufferException { + // Bitwise OR combines the sign bits so any negative value fails the check. + if ((index | size | buffer.limit() - index - size) < 0) { + throw new ArrayIndexOutOfBoundsException( + String.format("buffer limit=%d, index=%d, limit=%d", buffer.limit(), index, size)); + } + long address = UnsafeUtil.addressOffset(buffer) + index; + final long addressLimit = address + size; + + // The longest possible resulting String is the same as the number of input bytes, when it is + // all ASCII. For other cases, this over-allocates and we will truncate in the end. + char[] resultArr = new char[size]; + int resultPos = 0; + + // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this). + // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). + while (address < addressLimit) { + byte b = UnsafeUtil.getByte(address); + if (!DecodeUtil.isOneByte(b)) { + break; + } + address++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + + while (address < addressLimit) { + byte byte1 = UnsafeUtil.getByte(address++); + if (DecodeUtil.isOneByte(byte1)) { + DecodeUtil.handleOneByte(byte1, resultArr, resultPos++); + // It's common for there to be multiple ASCII characters in a run mixed in, so add an + // extra optimized loop to take care of these runs. + while (address < addressLimit) { + byte b = UnsafeUtil.getByte(address); + if (!DecodeUtil.isOneByte(b)) { + break; + } + address++; + DecodeUtil.handleOneByte(b, resultArr, resultPos++); + } + } else if (DecodeUtil.isTwoBytes(byte1)) { + if (address >= addressLimit) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleTwoBytes( + byte1, /* byte2 */ UnsafeUtil.getByte(address++), resultArr, resultPos++); + } else if (DecodeUtil.isThreeBytes(byte1)) { + if (address >= addressLimit - 1) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleThreeBytes( + byte1, + /* byte2 */ UnsafeUtil.getByte(address++), + /* byte3 */ UnsafeUtil.getByte(address++), + resultArr, + resultPos++); + } else { + if (address >= addressLimit - 2) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + DecodeUtil.handleFourBytes( + byte1, + /* byte2 */ UnsafeUtil.getByte(address++), + /* byte3 */ UnsafeUtil.getByte(address++), + /* byte4 */ UnsafeUtil.getByte(address++), + resultArr, + resultPos++); + // 4-byte case requires two chars. + resultPos++; + } + } + + return new String(resultArr, 0, resultPos); + } + + @Override int encodeUtf8(final CharSequence in, final byte[] out, final int offset, final int length) { long outIx = offset; final long outLimit = outIx + length; @@ -1554,5 +1923,112 @@ final class Utf8 { } } + /** + * Utility methods for decoding bytes into {@link String}. Callers are responsible for extracting + * bytes (possibly using Unsafe methods), and checking remaining bytes. All other UTF-8 validity + * checks and codepoint conversion happen in this class. + */ + private static class DecodeUtil { + + /** + * Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'. + */ + private static boolean isOneByte(byte b) { + return b >= 0; + } + + /** + * Returns whether this is a two-byte codepoint with the form '10XXXXXX'. + */ + private static boolean isTwoBytes(byte b) { + return b < (byte) 0xE0; + } + + /** + * Returns whether this is a three-byte codepoint with the form '110XXXXX'. + */ + private static boolean isThreeBytes(byte b) { + return b < (byte) 0xF0; + } + + private static void handleOneByte(byte byte1, char[] resultArr, int resultPos) { + resultArr[resultPos] = (char) byte1; + } + + private static void handleTwoBytes( + byte byte1, byte byte2, char[] resultArr, int resultPos) + throws InvalidProtocolBufferException { + // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and + // overlong 2-byte, '11000001'. + if (byte1 < (byte) 0xC2 + || isNotTrailingByte(byte2)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2)); + } + + private static void handleThreeBytes( + byte byte1, byte byte2, byte byte3, char[] resultArr, int resultPos) + throws InvalidProtocolBufferException { + if (isNotTrailingByte(byte2) + // overlong? 5 most significant bits must not all be zero + || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0) + // check for illegal surrogate codepoints + || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) + || isNotTrailingByte(byte3)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + resultArr[resultPos] = (char) + (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3)); + } + + private static void handleFourBytes( + byte byte1, byte byte2, byte byte3, byte byte4, char[] resultArr, int resultPos) + throws InvalidProtocolBufferException{ + if (isNotTrailingByte(byte2) + // Check that 1 <= plane <= 16. Tricky optimized form of: + // valid 4-byte leading byte? + // if (byte1 > (byte) 0xF4 || + // overlong? 4 most significant bits must not all be zero + // byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 || + // codepoint larger than the highest code point (U+10FFFF)? + // byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F) + || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 + || isNotTrailingByte(byte3) + || isNotTrailingByte(byte4)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + int codepoint = ((byte1 & 0x07) << 18) + | (trailingByteValue(byte2) << 12) + | (trailingByteValue(byte3) << 6) + | trailingByteValue(byte4); + resultArr[resultPos] = DecodeUtil.highSurrogate(codepoint); + resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codepoint); + } + + /** + * Returns whether the byte is not a valid continuation of the form '10XXXXXX'. + */ + private static boolean isNotTrailingByte(byte b) { + return b > (byte) 0xBF; + } + + /** + * Returns the actual value of the trailing byte (removes the prefix '10') for composition. + */ + private static int trailingByteValue(byte b) { + return b & 0x3F; + } + + private static char highSurrogate(int codePoint) { + return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + + (codePoint >>> 10)); + } + + private static char lowSurrogate(int codePoint) { + return (char) (MIN_LOW_SURROGATE + (codePoint & 0x3ff)); + } + } + private Utf8() {} } |