From 13fd045dbb2b4dacea32be162a41d5a4b0d1802f Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Tue, 12 Sep 2017 10:32:01 -0700 Subject: Integrated internal changes from Google --- .../java/com/google/protobuf/CodedInputStream.java | 882 +++++++++++++++++++++ .../com/google/protobuf/GeneratedMessageLite.java | 59 +- .../protobuf/IterableByteBufferInputStream.java | 150 ++++ .../java/com/google/protobuf/SmallSortedMap.java | 4 +- .../main/java/com/google/protobuf/TextFormat.java | 18 +- .../com/google/protobuf/UnknownFieldSetLite.java | 3 +- .../main/java/com/google/protobuf/UnsafeUtil.java | 11 - .../com/google/protobuf/CodedInputStreamTest.java | 46 +- .../com/google/protobuf/GeneratedMessageTest.java | 79 -- .../test/java/com/google/protobuf/LiteTest.java | 38 + .../test/java/com/google/protobuf/MessageTest.java | 8 + .../test/java/com/google/protobuf/ParserTest.java | 102 --- .../java/com/google/protobuf/TextFormatTest.java | 40 +- .../google/protobuf/UnknownFieldSetLiteTest.java | 247 +++++- .../com/google/protobuf/UnknownFieldSetTest.java | 204 ----- .../java/com/google/protobuf/WireFormatTest.java | 70 -- .../java/com/google/protobuf/util/JsonFormat.java | 3 +- 17 files changed, 1454 insertions(+), 510 deletions(-) create mode 100644 java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java (limited to 'java') diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index d6a941b1..511501d4 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -44,6 +44,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; /** @@ -85,6 +86,43 @@ public abstract class CodedInputStream { return new StreamDecoder(input, bufferSize); } + /** Create a new CodedInputStream wrapping the given {@code Iterable }. */ + public static CodedInputStream newInstance(final Iterable input) { + if (!UnsafeDirectNioDecoder.isSupported()) { + return newInstance(new IterableByteBufferInputStream(input)); + } + return newInstance(input, false); + } + + /** Create a new CodedInputStream wrapping the given {@code Iterable }. */ + static CodedInputStream newInstance( + final Iterable bufs, final boolean bufferIsImmutable) { + // flag is to check the type of input's ByteBuffers. + // flag equals 1: all ByteBuffers have array. + // flag equals 2: all ByteBuffers are direct ByteBuffers. + // flag equals 3: some ByteBuffers are direct and some have array. + // flag greater than 3: other cases. + int flag = 0; + // Total size of the input + int totalSize = 0; + for (ByteBuffer buf : bufs) { + totalSize += buf.remaining(); + if (buf.hasArray()) { + flag |= 1; + } else if (buf.isDirect()) { + flag |= 2; + } else { + flag |= 4; + } + } + if (flag == 2) { + return new IterableDirectByteBufferDecoder(bufs, totalSize, bufferIsImmutable); + } else { + // TODO(yilunchong): add another decoders to deal case 1 and 3. + return newInstance(new IterableByteBufferInputStream(bufs)); + } + } + /** Create a new CodedInputStream wrapping the given byte array. */ public static CodedInputStream newInstance(final byte[] buf) { return newInstance(buf, 0, buf.length); @@ -3022,4 +3060,848 @@ public abstract class CodedInputStream { pos = size - tempPos; } } + + /** + * Implementation of {@link CodedInputStream} that uses an {@link Iterable } as the + * data source. Requires the use of {@code sun.misc.Unsafe} to perform fast reads on the buffer. + */ + private static final class IterableDirectByteBufferDecoder extends CodedInputStream { + /** The object that need to decode. */ + private Iterable input; + /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */ + private Iterator iterator; + /** The current ByteBuffer; */ + private ByteBuffer currentByteBuffer; + /** + * If {@code true}, indicates that all the buffer are backing a {@link ByteString} and are + * therefore considered to be an immutable input source. + */ + private boolean immutable; + /** + * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]} + * may return slices of the underlying buffer, rather than copies. + */ + private boolean enableAliasing; + /** The global total message length limit */ + private int totalBufferSize; + /** The amount of available data in the input beyond {@link #currentLimit}. */ + private int bufferSizeAfterCurrentLimit; + /** The absolute position of the end of the current message. */ + private int currentLimit = Integer.MAX_VALUE; + /** The last tag that was read from this stream. */ + private int lastTag; + /** Total Bytes have been Read from the {@link Iterable} {@link ByteBuffer} */ + private int totalBytesRead; + /** The start position offset of the whole message, used as to reset the totalBytesRead */ + private int startOffset; + /** The current position for current ByteBuffer */ + private long currentByteBufferPos; + + private long currentByteBufferStartPos; + /** + * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this + * ByteBuffer; otherwise should be zero. + */ + private long currentAddress; + /** The limit position for current ByteBuffer */ + private long currentByteBufferLimit; + + /** + * The constructor of {@code Iterable} decoder. + * + * @param inputBufs The input data. + * @param size The total size of the input data. + * @param immutableFlag whether the input data is immutable. + */ + private IterableDirectByteBufferDecoder( + Iterable inputBufs, int size, boolean immutableFlag) { + totalBufferSize = size; + input = inputBufs; + iterator = input.iterator(); + immutable = immutableFlag; + startOffset = totalBytesRead = 0; + if (size == 0) { + currentByteBuffer = EMPTY_BYTE_BUFFER; + currentByteBufferPos = 0; + currentByteBufferStartPos = 0; + currentByteBufferLimit = 0; + currentAddress = 0; + } else { + tryGetNextByteBuffer(); + } + } + + /** To get the next ByteBuffer from {@code input}, and then update the parameters */ + private void getNextByteBuffer() throws InvalidProtocolBufferException { + if (!iterator.hasNext()) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + tryGetNextByteBuffer(); + } + + private void tryGetNextByteBuffer() { + currentByteBuffer = iterator.next(); + totalBytesRead += (int) (currentByteBufferPos - currentByteBufferStartPos); + currentByteBufferPos = currentByteBuffer.position(); + currentByteBufferStartPos = currentByteBufferPos; + currentByteBufferLimit = currentByteBuffer.limit(); + currentAddress = UnsafeUtil.addressOffset(currentByteBuffer); + currentByteBufferPos += currentAddress; + currentByteBufferStartPos += currentAddress; + currentByteBufferLimit += currentAddress; + } + + @Override + public int readTag() throws IOException { + if (isAtEnd()) { + lastTag = 0; + return 0; + } + + lastTag = readRawVarint32(); + if (WireFormat.getTagFieldNumber(lastTag) == 0) { + // If we actually read zero (or any tag number corresponding to field + // number zero), that's not a valid tag. + throw InvalidProtocolBufferException.invalidTag(); + } + return lastTag; + } + + @Override + public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { + if (lastTag != value) { + throw InvalidProtocolBufferException.invalidEndTag(); + } + } + + @Override + public int getLastTag() { + return lastTag; + } + + @Override + public boolean skipField(final int tag) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + skipRawVarint(); + return true; + case WireFormat.WIRETYPE_FIXED64: + skipRawBytes(FIXED64_SIZE); + return true; + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + skipRawBytes(readRawVarint32()); + return true; + case WireFormat.WIRETYPE_START_GROUP: + skipMessage(); + checkLastTagWas( + WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); + return true; + case WireFormat.WIRETYPE_END_GROUP: + return false; + case WireFormat.WIRETYPE_FIXED32: + skipRawBytes(FIXED32_SIZE); + return true; + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { + switch (WireFormat.getTagWireType(tag)) { + case WireFormat.WIRETYPE_VARINT: + { + long value = readInt64(); + output.writeRawVarint32(tag); + output.writeUInt64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_FIXED64: + { + long value = readRawLittleEndian64(); + output.writeRawVarint32(tag); + output.writeFixed64NoTag(value); + return true; + } + case WireFormat.WIRETYPE_LENGTH_DELIMITED: + { + ByteString value = readBytes(); + output.writeRawVarint32(tag); + output.writeBytesNoTag(value); + return true; + } + case WireFormat.WIRETYPE_START_GROUP: + { + output.writeRawVarint32(tag); + skipMessage(output); + int endtag = + WireFormat.makeTag( + WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); + checkLastTagWas(endtag); + output.writeRawVarint32(endtag); + return true; + } + case WireFormat.WIRETYPE_END_GROUP: + { + return false; + } + case WireFormat.WIRETYPE_FIXED32: + { + int value = readRawLittleEndian32(); + output.writeRawVarint32(tag); + output.writeFixed32NoTag(value); + return true; + } + default: + throw InvalidProtocolBufferException.invalidWireType(); + } + } + + @Override + public void skipMessage() throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag)) { + return; + } + } + } + + @Override + public void skipMessage(CodedOutputStream output) throws IOException { + while (true) { + final int tag = readTag(); + if (tag == 0 || !skipField(tag, output)) { + return; + } + } + } + + // ----------------------------------------------------------------- + + @Override + public double readDouble() throws IOException { + return Double.longBitsToDouble(readRawLittleEndian64()); + } + + @Override + public float readFloat() throws IOException { + return Float.intBitsToFloat(readRawLittleEndian32()); + } + + @Override + public long readUInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public long readInt64() throws IOException { + return readRawVarint64(); + } + + @Override + public int readInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public long readFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public boolean readBool() throws IOException { + return readRawVarint64() != 0; + } + + @Override + public String readString() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + String result = new String(bytes, UTF_8); + currentByteBufferPos += size; + return result; + } else if (size > 0 && size <= remaining()) { + // TODO(yilunchong): To use an underlying bytes[] instead of allocating a new bytes[] + byte[] bytes = new byte[size]; + readRawBytesTo(bytes, 0, size); + String result = new String(bytes, UTF_8); + return result; + } + + if (size == 0) { + return ""; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public String readStringRequireUtf8() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + if (!Utf8.isValidUtf8(bytes)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + String result = new String(bytes, UTF_8); + currentByteBufferPos += size; + return result; + } + if (size >= 0 && size <= remaining()) { + byte[] bytes = new byte[size]; + readRawBytesTo(bytes, 0, size); + if (!Utf8.isValidUtf8(bytes)) { + throw InvalidProtocolBufferException.invalidUtf8(); + } + String result = new String(bytes, UTF_8); + return result; + } + + if (size == 0) { + return ""; + } + if (size <= 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void readGroup( + final int fieldNumber, + final MessageLite.Builder builder, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + } + + + @Override + public T readGroup( + final int fieldNumber, + final Parser parser, + final ExtensionRegistryLite extensionRegistry) + throws IOException { + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); + --recursionDepth; + return result; + } + + @Deprecated + @Override + public void readUnknownGroup(final int fieldNumber, final MessageLite.Builder builder) + throws IOException { + readGroup(fieldNumber, builder, ExtensionRegistryLite.getEmptyRegistry()); + } + + @Override + public void readMessage( + final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) + throws IOException { + final int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + builder.mergeFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + } + + + @Override + public T readMessage( + final Parser parser, final ExtensionRegistryLite extensionRegistry) throws IOException { + int length = readRawVarint32(); + if (recursionDepth >= recursionLimit) { + throw InvalidProtocolBufferException.recursionLimitExceeded(); + } + final int oldLimit = pushLimit(length); + ++recursionDepth; + T result = parser.parsePartialFrom(this, extensionRegistry); + checkLastTagWas(0); + --recursionDepth; + popLimit(oldLimit); + return result; + } + + @Override + public ByteString readBytes() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentByteBufferLimit - currentByteBufferPos) { + if (immutable && enableAliasing) { + final int idx = (int) (currentByteBufferPos - currentAddress); + final ByteString result = ByteString.wrap(slice(idx, idx + size)); + currentByteBufferPos += size; + return result; + } else { + byte[] bytes; + bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + currentByteBufferPos += size; + return ByteString.wrap(bytes); + } + } else if (size > 0 && size <= remaining()) { + byte[] temp = new byte[size]; + readRawBytesTo(temp, 0, size); + return ByteString.wrap(temp); + } + + if (size == 0) { + return ByteString.EMPTY; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public byte[] readByteArray() throws IOException { + return readRawBytes(readRawVarint32()); + } + + @Override + public ByteBuffer readByteBuffer() throws IOException { + final int size = readRawVarint32(); + if (size > 0 && size <= currentRemaining()) { + if (!immutable && enableAliasing) { + currentByteBufferPos += size; + return slice( + (int) (currentByteBufferPos - currentAddress - size), + (int) (currentByteBufferPos - currentAddress)); + } else { + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size); + currentByteBufferPos += size; + return ByteBuffer.wrap(bytes); + } + } else if (size > 0 && size <= remaining()) { + byte[] temp = new byte[size]; + readRawBytesTo(temp, 0, size); + return ByteBuffer.wrap(temp); + } + + if (size == 0) { + return EMPTY_BYTE_BUFFER; + } + if (size < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public int readUInt32() throws IOException { + return readRawVarint32(); + } + + @Override + public int readEnum() throws IOException { + return readRawVarint32(); + } + + @Override + public int readSFixed32() throws IOException { + return readRawLittleEndian32(); + } + + @Override + public long readSFixed64() throws IOException { + return readRawLittleEndian64(); + } + + @Override + public int readSInt32() throws IOException { + return decodeZigZag32(readRawVarint32()); + } + + @Override + public long readSInt64() throws IOException { + return decodeZigZag64(readRawVarint64()); + } + + @Override + public int readRawVarint32() throws IOException { + fastpath: + { + long tempPos = currentByteBufferPos; + + if (currentByteBufferLimit == currentByteBufferPos) { + break fastpath; + } + + int x; + if ((x = UnsafeUtil.getByte(tempPos++)) >= 0) { + currentByteBufferPos++; + return x; + } else if (currentByteBufferLimit - currentByteBufferPos < 10) { + break fastpath; + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x ^= (~0 << 7); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x ^= (~0 << 7) ^ (~0 << 14); + } else if ((x ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); + } else { + int y = UnsafeUtil.getByte(tempPos++); + x ^= y << 28; + x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); + if (y < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0 + && UnsafeUtil.getByte(tempPos++) < 0) { + break fastpath; // Will throw malformedVarint() + } + } + currentByteBufferPos = tempPos; + return x; + } + return (int) readRawVarint64SlowPath(); + } + + @Override + public long readRawVarint64() throws IOException { + fastpath: + { + long tempPos = currentByteBufferPos; + + if (currentByteBufferLimit == currentByteBufferPos) { + break fastpath; + } + + long x; + int y; + if ((y = UnsafeUtil.getByte(tempPos++)) >= 0) { + currentByteBufferPos++; + return y; + } else if (currentByteBufferLimit - currentByteBufferPos < 10) { + break fastpath; + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 7)) < 0) { + x = y ^ (~0 << 7); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 14)) >= 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14)); + } else if ((y ^= (UnsafeUtil.getByte(tempPos++) << 21)) < 0) { + x = y ^ ((~0 << 7) ^ (~0 << 14) ^ (~0 << 21)); + } else if ((x = y ^ ((long) UnsafeUtil.getByte(tempPos++) << 28)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 35)) < 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 42)) >= 0L) { + x ^= (~0L << 7) ^ (~0L << 14) ^ (~0L << 21) ^ (~0L << 28) ^ (~0L << 35) ^ (~0L << 42); + } else if ((x ^= ((long) UnsafeUtil.getByte(tempPos++) << 49)) < 0L) { + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49); + } else { + x ^= ((long) UnsafeUtil.getByte(tempPos++) << 56); + x ^= + (~0L << 7) + ^ (~0L << 14) + ^ (~0L << 21) + ^ (~0L << 28) + ^ (~0L << 35) + ^ (~0L << 42) + ^ (~0L << 49) + ^ (~0L << 56); + if (x < 0L) { + if (UnsafeUtil.getByte(tempPos++) < 0L) { + break fastpath; // Will throw malformedVarint() + } + } + } + currentByteBufferPos = tempPos; + return x; + } + return readRawVarint64SlowPath(); + } + + @Override + long readRawVarint64SlowPath() throws IOException { + long result = 0; + for (int shift = 0; shift < 64; shift += 7) { + final byte b = readRawByte(); + result |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + @Override + public int readRawLittleEndian32() throws IOException { + if (currentRemaining() >= FIXED32_SIZE) { + long tempPos = currentByteBufferPos; + currentByteBufferPos += FIXED32_SIZE; + return (((UnsafeUtil.getByte(tempPos) & 0xff)) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24)); + } + return ((readRawByte() & 0xff) + | ((readRawByte() & 0xff) << 8) + | ((readRawByte() & 0xff) << 16) + | ((readRawByte() & 0xff) << 24)); + } + + @Override + public long readRawLittleEndian64() throws IOException { + if (currentRemaining() >= FIXED64_SIZE) { + long tempPos = currentByteBufferPos; + currentByteBufferPos += FIXED64_SIZE; + return (((UnsafeUtil.getByte(tempPos) & 0xffL)) + | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) + | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) + | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24) + | ((UnsafeUtil.getByte(tempPos + 4) & 0xffL) << 32) + | ((UnsafeUtil.getByte(tempPos + 5) & 0xffL) << 40) + | ((UnsafeUtil.getByte(tempPos + 6) & 0xffL) << 48) + | ((UnsafeUtil.getByte(tempPos + 7) & 0xffL) << 56)); + } + return ((readRawByte() & 0xffL) + | ((readRawByte() & 0xffL) << 8) + | ((readRawByte() & 0xffL) << 16) + | ((readRawByte() & 0xffL) << 24) + | ((readRawByte() & 0xffL) << 32) + | ((readRawByte() & 0xffL) << 40) + | ((readRawByte() & 0xffL) << 48) + | ((readRawByte() & 0xffL) << 56)); + } + + @Override + public void enableAliasing(boolean enabled) { + this.enableAliasing = enabled; + } + + @Override + public void resetSizeCounter() { + startOffset = (int) (totalBytesRead + currentByteBufferPos - currentByteBufferStartPos); + } + + @Override + public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { + if (byteLimit < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + byteLimit += getTotalBytesRead(); + final int oldLimit = currentLimit; + if (byteLimit > oldLimit) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + currentLimit = byteLimit; + + recomputeBufferSizeAfterLimit(); + + return oldLimit; + } + + private void recomputeBufferSizeAfterLimit() { + totalBufferSize += bufferSizeAfterCurrentLimit; + final int bufferEnd = totalBufferSize - startOffset; + if (bufferEnd > currentLimit) { + // Limit is in current buffer. + bufferSizeAfterCurrentLimit = bufferEnd - currentLimit; + totalBufferSize -= bufferSizeAfterCurrentLimit; + } else { + bufferSizeAfterCurrentLimit = 0; + } + } + + @Override + public void popLimit(final int oldLimit) { + currentLimit = oldLimit; + recomputeBufferSizeAfterLimit(); + } + + @Override + public int getBytesUntilLimit() { + if (currentLimit == Integer.MAX_VALUE) { + return -1; + } + + return currentLimit - getTotalBytesRead(); + } + + @Override + public boolean isAtEnd() throws IOException { + return totalBytesRead + currentByteBufferPos - currentByteBufferStartPos == totalBufferSize; + } + + @Override + public int getTotalBytesRead() { + return (int) + (totalBytesRead - startOffset + currentByteBufferPos - currentByteBufferStartPos); + } + + @Override + public byte readRawByte() throws IOException { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + return UnsafeUtil.getByte(currentByteBufferPos++); + } + + @Override + public byte[] readRawBytes(final int length) throws IOException { + if (length >= 0 && length <= currentRemaining()) { + byte[] bytes = new byte[length]; + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, length); + currentByteBufferPos += length; + return bytes; + } + if (length >= 0 && length <= remaining()) { + byte[] bytes = new byte[length]; + readRawBytesTo(bytes, 0, length); + return bytes; + } + + if (length <= 0) { + if (length == 0) { + return EMPTY_BYTE_ARRAY; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + + throw InvalidProtocolBufferException.truncatedMessage(); + } + + /** + * Try to get raw bytes from {@code input} with the size of {@code length} and copy to {@code + * bytes} array. If the size is bigger than the number of remaining bytes in the input, then + * throw {@code truncatedMessage} exception. + * + * @param bytes + * @param offset + * @param length + * @throws IOException + */ + private void readRawBytesTo(byte[] bytes, int offset, final int length) throws IOException { + if (length >= 0 && length <= remaining()) { + int l = length; + while (l > 0) { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + int bytesToCopy = Math.min(l, (int) currentRemaining()); + UnsafeUtil.copyMemory(currentByteBufferPos, bytes, length - l + offset, bytesToCopy); + l -= bytesToCopy; + currentByteBufferPos += bytesToCopy; + } + return; + } + + if (length <= 0) { + if (length == 0) { + return; + } else { + throw InvalidProtocolBufferException.negativeSize(); + } + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + @Override + public void skipRawBytes(final int length) throws IOException { + if (length >= 0 + && length + <= (totalBufferSize + - totalBytesRead + - currentByteBufferPos + + currentByteBufferStartPos)) { + // We have all the bytes we need already. + int l = length; + while (l > 0) { + if (currentRemaining() == 0) { + getNextByteBuffer(); + } + int rl = Math.min(l, (int) currentRemaining()); + l -= rl; + currentByteBufferPos += rl; + } + return; + } + + if (length < 0) { + throw InvalidProtocolBufferException.negativeSize(); + } + throw InvalidProtocolBufferException.truncatedMessage(); + } + + // TODO: optimize to fastpath + private void skipRawVarint() throws IOException { + for (int i = 0; i < MAX_VARINT_SIZE; i++) { + if (readRawByte() >= 0) { + return; + } + } + throw InvalidProtocolBufferException.malformedVarint(); + } + + /** + * Try to get the number of remaining bytes in {@code input}. + * + * @return the number of remaining bytes in {@code input}. + */ + private int remaining() { + return (int) + (totalBufferSize - totalBytesRead - currentByteBufferPos + currentByteBufferStartPos); + } + + /** + * Try to get the number of remaining bytes in {@code currentByteBuffer}. + * + * @return the number of remaining bytes in {@code currentByteBuffer} + */ + private long currentRemaining() { + return (currentByteBufferLimit - currentByteBufferPos); + } + + private ByteBuffer slice(int begin, int end) throws IOException { + int prevPos = currentByteBuffer.position(); + int prevLimit = currentByteBuffer.limit(); + try { + currentByteBuffer.position(begin); + currentByteBuffer.limit(end); + return currentByteBuffer.slice(); + } catch (IllegalArgumentException e) { + throw InvalidProtocolBufferException.truncatedMessage(); + } finally { + currentByteBuffer.position(prevPos); + currentByteBuffer.limit(prevLimit); + } + } + } } diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 7116ae1c..99864964 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -220,7 +220,7 @@ public abstract class GeneratedMessageLite< @Override public final boolean isInitialized() { - return dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.TRUE) != null; + return isInitialized((MessageType) this, Boolean.TRUE); } @Override @@ -240,6 +240,8 @@ public abstract class GeneratedMessageLite< public static enum MethodToInvoke { // Rely on/modify instance state IS_INITIALIZED, + GET_MEMOIZED_IS_INITIALIZED, + SET_MEMOIZED_IS_INITIALIZED, VISIT, MERGE_FROM_STREAM, MAKE_IMMUTABLE, @@ -256,25 +258,30 @@ public abstract class GeneratedMessageLite< * Theses different kinds of operations are required to implement message-level operations for * builders in the runtime. This method bundles those operations to reduce the generated methods * count. + * *
    - *
  • {@code MERGE_FROM_STREAM} is parameterized with an {@link CodedInputStream} and - * {@link ExtensionRegistryLite}. It consumes the input stream, parsing the contents into the - * returned protocol buffer. If parsing throws an {@link InvalidProtocolBufferException}, the - * implementation wraps it in a RuntimeException. - *
  • {@code NEW_INSTANCE} returns a new instance of the protocol buffer that has not yet been - * made immutable. See {@code MAKE_IMMUTABLE}. - *
  • {@code IS_INITIALIZED} is parameterized with a {@code Boolean} detailing whether to - * memoize. It returns {@code null} for false and the default instance for true. We optionally - * memoize to support the Builder case, where memoization is not desired. - *
  • {@code NEW_BUILDER} returns a {@code BuilderType} instance. - *
  • {@code VISIT} is parameterized with a {@code Visitor} and a {@code MessageType} and - * recursively iterates through the fields side by side between this and the instance. - *
  • {@code MAKE_IMMUTABLE} sets all internal fields to an immutable state. + *
  • {@code MERGE_FROM_STREAM} is parameterized with an {@link CodedInputStream} and {@link + * ExtensionRegistryLite}. It consumes the input stream, parsing the contents into the + * returned protocol buffer. If parsing throws an {@link InvalidProtocolBufferException}, + * the implementation wraps it in a RuntimeException. + *
  • {@code NEW_INSTANCE} returns a new instance of the protocol buffer that has not yet been + * made immutable. See {@code MAKE_IMMUTABLE}. + *
  • {@code IS_INITIALIZED} returns {@code null} for false and the default instance for true. + * It doesn't use or modify any memoized value. + *
  • {@code GET_MEMOIZED_IS_INITIALIZED} returns the memoized {@code isInitialized} byte + * value. + *
  • {@code SET_MEMOIZED_IS_INITIALIZED} sets the memoized {@code isInitilaized} byte value to + * 1 if the first parameter is not null, or to 0 if the first parameter is null. + *
  • {@code NEW_BUILDER} returns a {@code BuilderType} instance. + *
  • {@code VISIT} is parameterized with a {@code Visitor} and a {@code MessageType} and + * recursively iterates through the fields side by side between this and the instance. + *
  • {@code MAKE_IMMUTABLE} sets all internal fields to an immutable state. *
+ * * This method, plus the implementation of the Builder, enables the Builder class to be proguarded * away entirely on Android. - *

- * For use by generated code only. + * + *

For use by generated code only. */ protected abstract Object dynamicMethod(MethodToInvoke method, Object arg0, Object arg1); @@ -297,9 +304,9 @@ public abstract class GeneratedMessageLite< unknownFields = visitor.visitUnknownFields(unknownFields, other.unknownFields); } + /** - * Merge some unknown fields into the {@link UnknownFieldSetLite} for this - * message. + * Merge some unknown fields into the {@link UnknownFieldSetLite} for this message. * *

For use by generated code only. */ @@ -1403,7 +1410,21 @@ public abstract class GeneratedMessageLite< */ protected static final > boolean isInitialized( T message, boolean shouldMemoize) { - return message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, shouldMemoize) != null; + byte memoizedIsInitialized = + (Byte) message.dynamicMethod(MethodToInvoke.GET_MEMOIZED_IS_INITIALIZED); + if (memoizedIsInitialized == 1) { + return true; + } + if (memoizedIsInitialized == 0) { + return false; + } + boolean isInitialized = + message.dynamicMethod(MethodToInvoke.IS_INITIALIZED, Boolean.FALSE) != null; + if (shouldMemoize) { + message.dynamicMethod( + MethodToInvoke.SET_MEMOIZED_IS_INITIALIZED, isInitialized ? message : null); + } + return isInitialized; } protected static final > void makeImmutable(T message) { diff --git a/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java b/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java new file mode 100644 index 00000000..713e8064 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/IterableByteBufferInputStream.java @@ -0,0 +1,150 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import static com.google.protobuf.Internal.EMPTY_BYTE_BUFFER; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Iterator; + +class IterableByteBufferInputStream extends InputStream { + /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */ + private Iterator iterator; + /** The current ByteBuffer; */ + private ByteBuffer currentByteBuffer; + /** The number of ByteBuffers in the input data. */ + private int dataSize; + /** + * Current {@code ByteBuffer}'s index + * + *

If index equals dataSize, then all the data in the InputStream has been consumed + */ + private int currentIndex; + /** The current position for current ByteBuffer */ + private int currentByteBufferPos; + /** Whether current ByteBuffer has an array */ + private boolean hasArray; + /** + * If the current ByteBuffer is unsafe-direct based, currentArray is null; otherwise should be the + * array inside ByteBuffer. + */ + private byte[] currentArray; + /** Current ByteBuffer's array offset */ + private int currentArrayOffset; + /** + * If the current ByteBuffer is unsafe-direct based, currentAddress is the start address of this + * ByteBuffer; otherwise should be zero. + */ + private long currentAddress; + + IterableByteBufferInputStream(Iterable data) { + iterator = data.iterator(); + dataSize = 0; + for (ByteBuffer unused : data) { + dataSize++; + } + currentIndex = -1; + + if (!getNextByteBuffer()) { + currentByteBuffer = EMPTY_BYTE_BUFFER; + currentIndex = 0; + currentByteBufferPos = 0; + currentAddress = 0; + } + } + + private boolean getNextByteBuffer() { + currentIndex++; + if (!iterator.hasNext()) { + return false; + } + currentByteBuffer = iterator.next(); + currentByteBufferPos = currentByteBuffer.position(); + if (currentByteBuffer.hasArray()) { + hasArray = true; + currentArray = currentByteBuffer.array(); + currentArrayOffset = currentByteBuffer.arrayOffset(); + } else { + hasArray = false; + currentAddress = UnsafeUtil.addressOffset(currentByteBuffer); + currentArray = null; + } + return true; + } + + private void updateCurrentByteBufferPos(int numberOfBytesRead) { + currentByteBufferPos += numberOfBytesRead; + if (currentByteBufferPos == currentByteBuffer.limit()) { + getNextByteBuffer(); + } + } + + @Override + public int read() throws IOException { + if (currentIndex == dataSize) { + return -1; + } + if (hasArray) { + int result = currentArray[currentByteBufferPos + currentArrayOffset] & 0xFF; + updateCurrentByteBufferPos(1); + return result; + } else { + int result = UnsafeUtil.getByte(currentByteBufferPos + currentAddress) & 0xFF; + updateCurrentByteBufferPos(1); + return result; + } + } + + @Override + public int read(byte[] output, int offset, int length) throws IOException { + if (currentIndex == dataSize) { + return -1; + } + int remaining = currentByteBuffer.limit() - currentByteBufferPos; + if (length > remaining) { + length = remaining; + } + if (hasArray) { + System.arraycopy( + currentArray, currentByteBufferPos + currentArrayOffset, output, offset, length); + updateCurrentByteBufferPos(length); + } else { + int prevPos = currentByteBuffer.position(); + currentByteBuffer.position(currentByteBufferPos); + currentByteBuffer.get(output, offset, length); + currentByteBuffer.position(prevPos); + updateCurrentByteBufferPos(length); + } + return length; + } +} diff --git a/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java b/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java index 66033f58..279edc4d 100644 --- a/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java +++ b/java/core/src/main/java/com/google/protobuf/SmallSortedMap.java @@ -540,8 +540,8 @@ class SmallSortedMap, V> extends AbstractMap { @Override public boolean hasNext() { - return (pos + 1) < entryList.size() || - getOverflowIterator().hasNext(); + return (pos + 1) < entryList.size() + || (!overflowEntries.isEmpty() && getOverflowIterator().hasNext()); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 64094d09..ab9acf2f 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -279,9 +279,21 @@ public final class TextFormat { generator.print(String.format((Locale) null, "0x%016x", (Long) value)); break; case WireFormat.WIRETYPE_LENGTH_DELIMITED: - generator.print("\""); - generator.print(escapeBytes((ByteString) value)); - generator.print("\""); + try { + // Try to parse and print the field as an embedded message + UnknownFieldSet message = UnknownFieldSet.parseFrom((ByteString) value); + generator.print("{"); + generator.eol(); + generator.indent(); + Printer.DEFAULT.printUnknownFields(message, generator); + generator.outdent(); + generator.print("}"); + } catch (InvalidProtocolBufferException e) { + // If not parseable as a message, print as a String + generator.print("\""); + generator.print(escapeBytes((ByteString) value)); + generator.print("\""); + } break; case WireFormat.WIRETYPE_START_GROUP: Printer.DEFAULT.printUnknownFields((UnknownFieldSet) value, generator); diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java index d6226fc7..2a614c84 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSetLite.java @@ -81,7 +81,7 @@ public final class UnknownFieldSetLite { System.arraycopy(second.objects, 0, objects, first.count, second.count); return new UnknownFieldSetLite(count, tags, objects, true /* isMutable */); } - + /** * The number of elements in the set. */ @@ -323,6 +323,7 @@ public final class UnknownFieldSetLite { // Package private for unsafe experimental runtime. void storeField(int tag, Object value) { + checkMutable(); ensureCapacity(); tags[count] = tag; diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java index acc03a7c..88315cb6 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java @@ -252,10 +252,6 @@ final class UnsafeUtil { MEMORY_ACCESSOR.putLong(address, value); } - static void copyMemory(long srcAddress, long targetAddress, long length) { - MEMORY_ACCESSOR.copyMemory(srcAddress, targetAddress, length); - } - /** * Gets the offset of the {@code address} field of the given direct {@link ByteBuffer}. */ @@ -478,8 +474,6 @@ final class UnsafeUtil { public abstract void putLong(long address, long value); - public abstract void copyMemory(long srcAddress, long targetAddress, long length); - public abstract Object getStaticObject(Field field); public abstract void copyMemory(long srcOffset, byte[] target, long targetIndex, long length); @@ -562,11 +556,6 @@ final class UnsafeUtil { public void putDouble(Object target, long offset, double value) { unsafe.putDouble(target, offset, value); } - - @Override - public void copyMemory(long srcAddress, long targetAddress, long length) { - unsafe.copyMemory(srcAddress, targetAddress, length); - } @Override public void copyMemory(long srcOffset, byte[] target, long targetIndex, long length) { diff --git a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java index da2c067e..5ea6b79c 100644 --- a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java +++ b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java @@ -41,6 +41,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import junit.framework.TestCase; @@ -50,6 +51,9 @@ import junit.framework.TestCase; * @author kenton@google.com Kenton Varda */ public class CodedInputStreamTest extends TestCase { + + private static final int DEFAULT_BLOCK_SIZE = 4096; + private enum InputType { ARRAY { @Override @@ -77,7 +81,43 @@ public class CodedInputStreamTest extends TestCase { CodedInputStream newDecoder(byte[] data, int blockSize) { return CodedInputStream.newInstance(new SmallBlockInputStream(data, blockSize)); } + }, + ITER_DIRECT { + @Override + CodedInputStream newDecoder(byte[] data, int blockSize) { + if (blockSize > DEFAULT_BLOCK_SIZE) { + blockSize = DEFAULT_BLOCK_SIZE; + } + ArrayList input = new ArrayList (); + for (int i = 0; i < data.length; i += blockSize) { + int rl = Math.min(blockSize, data.length - i); + ByteBuffer rb = ByteBuffer.allocateDirect(rl); + rb.put(data, i, rl); + rb.flip(); + input.add(rb); + } + return CodedInputStream.newInstance(input); + } + }, + STREAM_ITER_DIRECT { + @Override + CodedInputStream newDecoder(byte[] data, int blockSize) { + if (blockSize > DEFAULT_BLOCK_SIZE) { + blockSize = DEFAULT_BLOCK_SIZE; + } + ArrayList input = new ArrayList (); + for (int i = 0; i < data.length; i += blockSize) { + int rl = Math.min(blockSize, data.length - i); + ByteBuffer rb = ByteBuffer.allocateDirect(rl); + rb.put(data, i, rl); + rb.flip(); + input.add(rb); + } + return CodedInputStream.newInstance(new IterableByteBufferInputStream(input)); + } }; + + CodedInputStream newDecoder(byte[] data) { return newDecoder(data, data.length); @@ -994,7 +1034,9 @@ public class CodedInputStreamTest extends TestCase { byte[] data = byteArrayStream.toByteArray(); for (InputType inputType : InputType.values()) { - if (inputType == InputType.STREAM) { + if (inputType == InputType.STREAM + || inputType == InputType.STREAM_ITER_DIRECT + || inputType == InputType.ITER_DIRECT) { // Aliasing doesn't apply to stream-backed CIS. continue; } @@ -1019,7 +1061,7 @@ public class CodedInputStreamTest extends TestCase { assertEquals(inputType.name(), (byte) 89, result.get()); // Enable aliasing - inputStream = inputType.newDecoder(data); + inputStream = inputType.newDecoder(data, data.length); inputStream.enableAliasing(true); result = inputStream.readByteBuffer(); assertEquals(inputType.name(), 0, result.capacity()); diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index a4311d17..c9ebe7f5 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -32,19 +32,14 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.UnittestLite.TestAllExtensionsLite; import com.google.protobuf.test.UnittestImport; import protobuf_unittest.EnumWithNoOuter; import protobuf_unittest.MessageWithNoOuter; import protobuf_unittest.MultipleFilesTestProto; import protobuf_unittest.NestedExtension.MyNestedExtension; -import protobuf_unittest.NestedExtensionLite.MyNestedExtensionLite; import protobuf_unittest.NonNestedExtension; import protobuf_unittest.NonNestedExtension.MessageToBeExtended; import protobuf_unittest.NonNestedExtension.MyNonNestedExtension; -import protobuf_unittest.NonNestedExtensionLite; -import protobuf_unittest.NonNestedExtensionLite.MessageLiteToBeExtended; -import protobuf_unittest.NonNestedExtensionLite.MyNonNestedExtensionLite; import protobuf_unittest.OuterClassNameTest2OuterClass; import protobuf_unittest.OuterClassNameTest3OuterClass; import protobuf_unittest.OuterClassNameTestOuterClass; @@ -711,70 +706,6 @@ public class GeneratedMessageTest extends TestCase { 1, (int) merged.getExtension(UnittestProto.optionalInt32Extension)); } - // ================================================================= - // Lite Extensions. - - // We test lite extensions directly because they have a separate - // implementation from full extensions. In contrast, we do not test - // lite fields directly since they are implemented exactly the same as - // regular fields. - - public void testLiteExtensionMessageOrBuilder() throws Exception { - TestAllExtensionsLite.Builder builder = TestAllExtensionsLite.newBuilder(); - TestUtilLite.setAllExtensions(builder); - TestUtil.assertAllExtensionsSet(builder); - - TestAllExtensionsLite message = builder.build(); - TestUtil.assertAllExtensionsSet(message); - } - - public void testLiteExtensionRepeatedSetters() throws Exception { - TestAllExtensionsLite.Builder builder = TestAllExtensionsLite.newBuilder(); - TestUtilLite.setAllExtensions(builder); - TestUtilLite.modifyRepeatedExtensions(builder); - TestUtil.assertRepeatedExtensionsModified(builder); - - TestAllExtensionsLite message = builder.build(); - TestUtil.assertRepeatedExtensionsModified(message); - } - - public void testLiteExtensionDefaults() throws Exception { - TestUtil.assertExtensionsClear(TestAllExtensionsLite.getDefaultInstance()); - TestUtil.assertExtensionsClear(TestAllExtensionsLite.newBuilder().build()); - } - - public void testClearLiteExtension() throws Exception { - // clearExtension() is not actually used in TestUtil, so try it manually. - assertFalse( - TestAllExtensionsLite.newBuilder() - .setExtension(UnittestLite.optionalInt32ExtensionLite, 1) - .clearExtension(UnittestLite.optionalInt32ExtensionLite) - .hasExtension(UnittestLite.optionalInt32ExtensionLite)); - assertEquals(0, - TestAllExtensionsLite.newBuilder() - .addExtension(UnittestLite.repeatedInt32ExtensionLite, 1) - .clearExtension(UnittestLite.repeatedInt32ExtensionLite) - .getExtensionCount(UnittestLite.repeatedInt32ExtensionLite)); - } - - public void testLiteExtensionCopy() throws Exception { - TestAllExtensionsLite original = TestUtilLite.getAllLiteExtensionsSet(); - TestAllExtensionsLite copy = - TestAllExtensionsLite.newBuilder(original).build(); - TestUtil.assertAllExtensionsSet(copy); - } - - public void testLiteExtensionMergeFrom() throws Exception { - TestAllExtensionsLite original = - TestAllExtensionsLite.newBuilder() - .setExtension(UnittestLite.optionalInt32ExtensionLite, 1).build(); - TestAllExtensionsLite merged = - TestAllExtensionsLite.newBuilder().mergeFrom(original).build(); - assertTrue(merged.hasExtension(UnittestLite.optionalInt32ExtensionLite)); - assertEquals( - 1, (int) merged.getExtension(UnittestLite.optionalInt32ExtensionLite)); - } - // ================================================================= // multiple_files_test @@ -942,16 +873,6 @@ public class GeneratedMessageTest extends TestCase { MyNestedExtension.recursiveExtension.getDescriptor().getName()); } - public void testNonNestedExtensionLiteInitialization() { - assertTrue(NonNestedExtensionLite.nonNestedExtensionLite - .getMessageDefaultInstance() instanceof MyNonNestedExtensionLite); - } - - public void testNestedExtensionLiteInitialization() { - assertTrue(MyNestedExtensionLite.recursiveExtensionLite - .getMessageDefaultInstance() instanceof MessageLiteToBeExtended); - } - public void testInvalidations() throws Exception { GeneratedMessage.enableAlwaysUseFieldBuildersForTesting(); TestAllTypes.NestedMessage nestedMessage1 = diff --git a/java/core/src/test/java/com/google/protobuf/LiteTest.java b/java/core/src/test/java/com/google/protobuf/LiteTest.java index ba8bcb1c..ba4d539d 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteTest.java @@ -1453,6 +1453,36 @@ public class LiteTest extends TestCase { UnittestLite.optionalFixed32ExtensionLite)); } + // Builder.mergeFrom() should keep existing extensions. + public void testBuilderMergeFromWithExtensions() throws Exception { + TestAllExtensionsLite message = + TestAllExtensionsLite.newBuilder() + .addExtension(UnittestLite.repeatedInt32ExtensionLite, 12) + .build(); + + ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance(); + UnittestLite.registerAllExtensions(registry); + + TestAllExtensionsLite.Builder builder = TestAllExtensionsLite.newBuilder(); + builder.mergeFrom(message.toByteArray(), registry); + builder.mergeFrom(message.toByteArray(), registry); + TestAllExtensionsLite result = builder.build(); + assertEquals(2, result.getExtensionCount(UnittestLite.repeatedInt32ExtensionLite)); + assertEquals(12, result.getExtension(UnittestLite.repeatedInt32ExtensionLite, 0).intValue()); + assertEquals(12, result.getExtension(UnittestLite.repeatedInt32ExtensionLite, 1).intValue()); + } + + // Builder.mergeFrom() should keep existing unknown fields. + public void testBuilderMergeFromWithUnknownFields() throws Exception { + TestAllTypesLite message = TestAllTypesLite.newBuilder().addRepeatedInt32(1).build(); + + NestedMessage.Builder builder = NestedMessage.newBuilder(); + builder.mergeFrom(message.toByteArray()); + builder.mergeFrom(message.toByteArray()); + NestedMessage result = builder.build(); + assertEquals(message.getSerializedSize() * 2, result.getSerializedSize()); + } + public void testToStringDefaultInstance() throws Exception { assertToStringEquals("", TestAllTypesLite.getDefaultInstance()); } @@ -2592,4 +2622,12 @@ public class LiteTest extends TestCase { return list.iterator(); } } + + public void testNullExtensionRegistry() throws Exception { + try { + TestAllTypesLite.parseFrom(new byte[] {}, null); + fail(); + } catch (NullPointerException expected) { + } + } } diff --git a/java/core/src/test/java/com/google/protobuf/MessageTest.java b/java/core/src/test/java/com/google/protobuf/MessageTest.java index 9d55d0dd..4fc8f78e 100644 --- a/java/core/src/test/java/com/google/protobuf/MessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/MessageTest.java @@ -74,6 +74,14 @@ public class MessageTest extends TestCase { "repeated_string: \"qux\"\n" + "repeated_string: \"bar\"\n"; + public void testParsingWithNullExtensionRegistry() throws Exception { + try { + TestAllTypes.parseFrom(new byte[] {}, null); + fail(); + } catch (NullPointerException expected) { + } + } + public void testMergeFrom() throws Exception { TestAllTypes result = TestAllTypes.newBuilder(MERGE_DEST) diff --git a/java/core/src/test/java/com/google/protobuf/ParserTest.java b/java/core/src/test/java/com/google/protobuf/ParserTest.java index 4bd34112..1e891112 100644 --- a/java/core/src/test/java/com/google/protobuf/ParserTest.java +++ b/java/core/src/test/java/com/google/protobuf/ParserTest.java @@ -30,9 +30,6 @@ package com.google.protobuf; -import com.google.protobuf.UnittestLite.TestAllTypesLite; -import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; -import com.google.protobuf.UnittestLite.TestParsingMergeLite; import protobuf_unittest.UnittestOptimizeFor; import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize; import protobuf_unittest.UnittestOptimizeFor.TestRequiredOptimizedForSize; @@ -183,22 +180,12 @@ public class ParserTest extends TestCase { TestUtil.getExtensionRegistry()); } - public void testParseExtensionsLite() throws Exception { - assertRoundTripEquals( - TestUtilLite.getAllLiteExtensionsSet(), TestUtilLite.getExtensionRegistryLite()); - } - public void testParsePacked() throws Exception { assertRoundTripEquals(TestUtil.getPackedSet()); assertRoundTripEquals(TestUtil.getPackedExtensionsSet(), TestUtil.getExtensionRegistry()); } - public void testParsePackedLite() throws Exception { - assertRoundTripEquals( - TestUtilLite.getLitePackedExtensionsSet(), TestUtilLite.getExtensionRegistryLite()); - } - public void testParseDelimitedTo() throws Exception { // Write normal Message. TestAllTypes normalMessage = TestUtil.getAllSet(); @@ -211,26 +198,6 @@ public class ParserTest extends TestCase { assertMessageEquals(normalMessage, normalMessage.getParserForType().parseDelimitedFrom(input)); } - public void testParseDelimitedToLite() throws Exception { - // Write MessageLite with packed extension fields. - TestPackedExtensionsLite packedMessage = TestUtilLite.getLitePackedExtensionsSet(); - ByteArrayOutputStream output = new ByteArrayOutputStream(); - packedMessage.writeDelimitedTo(output); - packedMessage.writeDelimitedTo(output); - - InputStream input = new ByteArrayInputStream(output.toByteArray()); - assertMessageEquals( - packedMessage, - packedMessage - .getParserForType() - .parseDelimitedFrom(input, TestUtilLite.getExtensionRegistryLite())); - assertMessageEquals( - packedMessage, - packedMessage - .getParserForType() - .parseDelimitedFrom(input, TestUtilLite.getExtensionRegistryLite())); - } - public void testParseUnknownFields() throws Exception { // All fields will be treated as unknown fields in emptyMessage. TestEmptyMessage emptyMessage = @@ -263,14 +230,6 @@ public class ParserTest extends TestCase { assertEquals("hello", allTypes.getOptionalString()); } - /** Helper method for {@link #testParsingMergeLite()}.*/ - private void assertMessageMerged(TestAllTypesLite allTypes) - throws Exception { - assertEquals(3, allTypes.getOptionalInt32()); - assertEquals(2, allTypes.getOptionalInt64()); - assertEquals("hello", allTypes.getOptionalString()); - } - public void testParsingMerge() throws Exception { // Build messages. TestAllTypes.Builder builder = TestAllTypes.newBuilder(); @@ -332,67 +291,6 @@ public class ParserTest extends TestCase { TestParsingMerge.repeatedExt)); } - public void testParsingMergeLite() throws Exception { - // Build messages. - TestAllTypesLite.Builder builder = TestAllTypesLite.newBuilder(); - TestAllTypesLite msg1 = builder.setOptionalInt32(1).build(); - builder.clear(); - TestAllTypesLite msg2 = builder.setOptionalInt64(2).build(); - builder.clear(); - TestAllTypesLite msg3 = builder.setOptionalInt32(3) - .setOptionalString("hello").build(); - - // Build groups. - TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG1 = - TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder() - .setField1(msg1).build(); - TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG2 = - TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder() - .setField1(msg2).build(); - TestParsingMergeLite.RepeatedFieldsGenerator.Group1 optionalG3 = - TestParsingMergeLite.RepeatedFieldsGenerator.Group1.newBuilder() - .setField1(msg3).build(); - TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG1 = - TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder() - .setField1(msg1).build(); - TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG2 = - TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder() - .setField1(msg2).build(); - TestParsingMergeLite.RepeatedFieldsGenerator.Group2 repeatedG3 = - TestParsingMergeLite.RepeatedFieldsGenerator.Group2.newBuilder() - .setField1(msg3).build(); - - // Assign and serialize RepeatedFieldsGenerator. - ByteString data = TestParsingMergeLite.RepeatedFieldsGenerator.newBuilder() - .addField1(msg1).addField1(msg2).addField1(msg3) - .addField2(msg1).addField2(msg2).addField2(msg3) - .addField3(msg1).addField3(msg2).addField3(msg3) - .addGroup1(optionalG1).addGroup1(optionalG2).addGroup1(optionalG3) - .addGroup2(repeatedG1).addGroup2(repeatedG2).addGroup2(repeatedG3) - .addExt1(msg1).addExt1(msg2).addExt1(msg3) - .addExt2(msg1).addExt2(msg2).addExt2(msg3) - .build().toByteString(); - - // Parse TestParsingMergeLite. - ExtensionRegistry registry = ExtensionRegistry.newInstance(); - UnittestLite.registerAllExtensions(registry); - TestParsingMergeLite parsingMerge = TestParsingMergeLite.parser().parseFrom(data, registry); - - // Required and optional fields should be merged. - assertMessageMerged(parsingMerge.getRequiredAllTypes()); - assertMessageMerged(parsingMerge.getOptionalAllTypes()); - assertMessageMerged( - parsingMerge.getOptionalGroup().getOptionalGroupAllTypes()); - assertMessageMerged(parsingMerge.getExtension( - TestParsingMergeLite.optionalExt)); - - // Repeated fields should not be merged. - assertEquals(3, parsingMerge.getRepeatedAllTypesCount()); - assertEquals(3, parsingMerge.getRepeatedGroupCount()); - assertEquals(3, parsingMerge.getExtensionCount( - TestParsingMergeLite.repeatedExt)); - } - public void testParseDelimitedFrom_firstByteInterrupted_preservesCause() { try { TestUtil.getAllSet().parseDelimitedFrom( diff --git a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java index 910f360f..28c4fdea 100644 --- a/java/core/src/test/java/com/google/protobuf/TextFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/TextFormatTest.java @@ -168,6 +168,7 @@ public class TextFormatTest extends TestCase { // Creates an example unknown field set. private UnknownFieldSet makeUnknownFieldSet() { + return UnknownFieldSet.newBuilder() .addField(5, UnknownFieldSet.Field.newBuilder() @@ -175,6 +176,12 @@ public class TextFormatTest extends TestCase { .addFixed32(2) .addFixed64(3) .addLengthDelimited(ByteString.copyFromUtf8("4")) + .addLengthDelimited(UnknownFieldSet.newBuilder() + .addField(12, + UnknownFieldSet.Field.newBuilder() + .addVarint(6) + .build()) + .build().toByteString()) .addGroup( UnknownFieldSet.newBuilder() .addField(10, @@ -207,20 +214,23 @@ public class TextFormatTest extends TestCase { .build(); assertEquals( - "5: 1\n" + - "5: 0x00000002\n" + - "5: 0x0000000000000003\n" + - "5: \"4\"\n" + - "5 {\n" + - " 10: 5\n" + - "}\n" + - "8: 1\n" + - "8: 2\n" + - "8: 3\n" + - "15: 12379813812177893520\n" + - "15: 0xabcd1234\n" + - "15: 0xabcdef1234567890\n", - TextFormat.printToString(message)); + "5: 1\n" + + "5: 0x00000002\n" + + "5: 0x0000000000000003\n" + + "5: \"4\"\n" + + "5: {\n" + + " 12: 6\n" + + "}\n" + + "5 {\n" + + " 10: 5\n" + + "}\n" + + "8: 1\n" + + "8: 2\n" + + "8: 3\n" + + "15: 12379813812177893520\n" + + "15: 0xabcd1234\n" + + "15: 0xabcdef1234567890\n", + TextFormat.printToString(message)); } public void testPrintField() throws Exception { @@ -861,7 +871,7 @@ public class TextFormatTest extends TestCase { } public void testShortDebugString_unknown() { - assertEquals("5: 1 5: 0x00000002 5: 0x0000000000000003 5: \"4\" 5 { 10: 5 }" + assertEquals("5: 1 5: 0x00000002 5: 0x0000000000000003 5: \"4\" 5: { 12: 6 } 5 { 10: 5 }" + " 8: 1 8: 2 8: 3 15: 12379813812177893520 15: 0xabcd1234 15:" + " 0xabcdef1234567890", TextFormat.shortDebugString(makeUnknownFieldSet())); diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java index f8cb0aab..9928d44a 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetLiteTest.java @@ -32,11 +32,20 @@ package com.google.protobuf; import com.google.protobuf.UnittestLite.TestAllExtensionsLite; import com.google.protobuf.UnittestLite.TestAllTypesLite; +import protobuf_unittest.UnittestProto; +import protobuf_unittest.UnittestProto.ForeignEnum; +import protobuf_unittest.UnittestProto.TestAllExtensions; +import protobuf_unittest.UnittestProto.TestAllTypes; +import protobuf_unittest.UnittestProto.TestEmptyMessage; +import protobuf_unittest.UnittestProto.TestPackedExtensions; +import protobuf_unittest.UnittestProto.TestPackedTypes; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Bar; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Arrays; +import java.util.Map; import junit.framework.TestCase; /** @@ -45,7 +54,44 @@ import junit.framework.TestCase; * @author dweis@google.com (Daniel Weis) */ public class UnknownFieldSetLiteTest extends TestCase { - + @Override + public void setUp() throws Exception { + allFields = TestUtil.getAllSet(); + allFieldsData = allFields.toByteString(); + emptyMessage = TestEmptyMessage.parseFrom(allFieldsData); + unknownFields = emptyMessage.getUnknownFields(); + } + + TestAllTypes allFields; + ByteString allFieldsData; + + // Constructs a protocol buffer which contains fields with all the same + // numbers as allFieldsData except that each field is some other wire + // type. + private ByteString getBizarroData() throws Exception { + UnknownFieldSet.Builder bizarroFields = UnknownFieldSet.newBuilder(); + + UnknownFieldSet.Field varintField = UnknownFieldSet.Field.newBuilder().addVarint(1).build(); + UnknownFieldSet.Field fixed32Field = UnknownFieldSet.Field.newBuilder().addFixed32(1).build(); + + for (Map.Entry entry : unknownFields.asMap().entrySet()) { + if (entry.getValue().getVarintList().isEmpty()) { + // Original field is not a varint, so use a varint. + bizarroFields.addField(entry.getKey(), varintField); + } else { + // Original field *is* a varint, so use something else. + bizarroFields.addField(entry.getKey(), fixed32Field); + } + } + + return bizarroFields.build().toByteString(); + } + + // An empty message that has been parsed from allFieldsData. So, it has + // unknown fields of every type. + TestEmptyMessage emptyMessage; + UnknownFieldSet unknownFields; + public void testDefaultInstance() { UnknownFieldSetLite unknownFields = UnknownFieldSetLite.getDefaultInstance(); @@ -331,4 +377,203 @@ public class UnknownFieldSetLiteTest extends TestCase { } return ByteString.copyFrom(byteArrayOutputStream.toByteArray()); } + + public void testSerializeLite() throws Exception { + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData); + assertEquals(allFieldsData.size(), emptyMessageLite.getSerializedSize()); + ByteString data = emptyMessageLite.toByteString(); + TestAllTypes message = TestAllTypes.parseFrom(data); + TestUtil.assertAllFieldsSet(message); + assertEquals(allFieldsData, data); + } + + public void testAllExtensionsLite() throws Exception { + TestAllExtensions allExtensions = TestUtil.getAllExtensionsSet(); + ByteString allExtensionsData = allExtensions.toByteString(); + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parser().parseFrom(allExtensionsData); + ByteString data = emptyMessageLite.toByteString(); + TestAllExtensions message = TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry()); + TestUtil.assertAllExtensionsSet(message); + assertEquals(allExtensionsData, data); + } + + public void testAllPackedFieldsLite() throws Exception { + TestPackedTypes allPackedFields = TestUtil.getPackedSet(); + ByteString allPackedData = allPackedFields.toByteString(); + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(allPackedData); + ByteString data = emptyMessageLite.toByteString(); + TestPackedTypes message = TestPackedTypes.parseFrom(data, TestUtil.getExtensionRegistry()); + TestUtil.assertPackedFieldsSet(message); + assertEquals(allPackedData, data); + } + + public void testAllPackedExtensionsLite() throws Exception { + TestPackedExtensions allPackedExtensions = TestUtil.getPackedExtensionsSet(); + ByteString allPackedExtensionsData = allPackedExtensions.toByteString(); + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(allPackedExtensionsData); + ByteString data = emptyMessageLite.toByteString(); + TestPackedExtensions message = + TestPackedExtensions.parseFrom(data, TestUtil.getExtensionRegistry()); + TestUtil.assertPackedExtensionsSet(message); + assertEquals(allPackedExtensionsData, data); + } + + public void testCopyFromLite() throws Exception { + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData); + UnittestLite.TestEmptyMessageLite emptyMessageLite2 = + UnittestLite.TestEmptyMessageLite.newBuilder().mergeFrom(emptyMessageLite).build(); + assertEquals(emptyMessageLite.toByteString(), emptyMessageLite2.toByteString()); + } + + public void testMergeFromLite() throws Exception { + TestAllTypes message1 = + TestAllTypes.newBuilder() + .setOptionalInt32(1) + .setOptionalString("foo") + .addRepeatedString("bar") + .setOptionalNestedEnum(TestAllTypes.NestedEnum.BAZ) + .build(); + + TestAllTypes message2 = + TestAllTypes.newBuilder() + .setOptionalInt64(2) + .setOptionalString("baz") + .addRepeatedString("qux") + .setOptionalForeignEnum(ForeignEnum.FOREIGN_BAZ) + .build(); + + ByteString data1 = message1.toByteString(); + UnittestLite.TestEmptyMessageLite emptyMessageLite1 = + UnittestLite.TestEmptyMessageLite.parseFrom(data1); + ByteString data2 = message2.toByteString(); + UnittestLite.TestEmptyMessageLite emptyMessageLite2 = + UnittestLite.TestEmptyMessageLite.parseFrom(data2); + + message1 = TestAllTypes.newBuilder(message1).mergeFrom(message2).build(); + emptyMessageLite1 = + UnittestLite.TestEmptyMessageLite.newBuilder(emptyMessageLite1) + .mergeFrom(emptyMessageLite2) + .build(); + + data1 = emptyMessageLite1.toByteString(); + message2 = TestAllTypes.parseFrom(data1); + + assertEquals(message1, message2); + } + + public void testWrongTypeTreatedAsUnknownLite() throws Exception { + // Test that fields of the wrong wire type are treated like unknown fields + // when parsing. + + ByteString bizarroData = getBizarroData(); + TestAllTypes allTypesMessage = TestAllTypes.parseFrom(bizarroData); + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData); + ByteString data = emptyMessageLite.toByteString(); + TestAllTypes allTypesMessage2 = TestAllTypes.parseFrom(data); + + assertEquals(allTypesMessage.toString(), allTypesMessage2.toString()); + } + + public void testUnknownExtensionsLite() throws Exception { + // Make sure fields are properly parsed to the UnknownFieldSet even when + // they are declared as extension numbers. + + UnittestLite.TestEmptyMessageWithExtensionsLite message = + UnittestLite.TestEmptyMessageWithExtensionsLite.parseFrom(allFieldsData); + + assertEquals(allFieldsData, message.toByteString()); + } + + public void testWrongExtensionTypeTreatedAsUnknownLite() throws Exception { + // Test that fields of the wrong wire type are treated like unknown fields + // when parsing extensions. + + ByteString bizarroData = getBizarroData(); + TestAllExtensions allExtensionsMessage = TestAllExtensions.parseFrom(bizarroData); + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData); + + // All fields should have been interpreted as unknown, so the byte strings + // should be the same. + assertEquals(emptyMessageLite.toByteString(), allExtensionsMessage.toByteString()); + } + + public void testParseUnknownEnumValueLite() throws Exception { + Descriptors.FieldDescriptor singularField = + TestAllTypes.getDescriptor().findFieldByName("optional_nested_enum"); + Descriptors.FieldDescriptor repeatedField = + TestAllTypes.getDescriptor().findFieldByName("repeated_nested_enum"); + assertNotNull(singularField); + assertNotNull(repeatedField); + + ByteString data = + UnknownFieldSet.newBuilder() + .addField( + singularField.getNumber(), + UnknownFieldSet.Field.newBuilder() + .addVarint(TestAllTypes.NestedEnum.BAR.getNumber()) + .addVarint(5) // not valid + .build()) + .addField( + repeatedField.getNumber(), + UnknownFieldSet.Field.newBuilder() + .addVarint(TestAllTypes.NestedEnum.FOO.getNumber()) + .addVarint(4) // not valid + .addVarint(TestAllTypes.NestedEnum.BAZ.getNumber()) + .addVarint(6) // not valid + .build()) + .build() + .toByteString(); + + UnittestLite.TestEmptyMessageLite emptyMessageLite = + UnittestLite.TestEmptyMessageLite.parseFrom(data); + data = emptyMessageLite.toByteString(); + + { + TestAllTypes message = TestAllTypes.parseFrom(data); + assertEquals(TestAllTypes.NestedEnum.BAR, message.getOptionalNestedEnum()); + assertEquals( + Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ), + message.getRepeatedNestedEnumList()); + assertEquals( + Arrays.asList(5L), + message.getUnknownFields().getField(singularField.getNumber()).getVarintList()); + assertEquals( + Arrays.asList(4L, 6L), + message.getUnknownFields().getField(repeatedField.getNumber()).getVarintList()); + } + + { + TestAllExtensions message = + TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry()); + assertEquals( + TestAllTypes.NestedEnum.BAR, + message.getExtension(UnittestProto.optionalNestedEnumExtension)); + assertEquals( + Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ), + message.getExtension(UnittestProto.repeatedNestedEnumExtension)); + assertEquals( + Arrays.asList(5L), + message.getUnknownFields().getField(singularField.getNumber()).getVarintList()); + assertEquals( + Arrays.asList(4L, 6L), + message.getUnknownFields().getField(repeatedField.getNumber()).getVarintList()); + } + } + + public void testClearLite() throws Exception { + UnittestLite.TestEmptyMessageLite emptyMessageLite1 = + UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData); + UnittestLite.TestEmptyMessageLite emptyMessageLite2 = + UnittestLite.TestEmptyMessageLite.newBuilder().mergeFrom(emptyMessageLite1).clear().build(); + assertEquals(0, emptyMessageLite2.getSerializedSize()); + ByteString data = emptyMessageLite2.toByteString(); + assertEquals(0, data.size()); + } } diff --git a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java index f81e90b4..1a84806a 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownFieldSetTest.java @@ -445,208 +445,4 @@ public class UnknownFieldSetTest extends TestCase { } // ================================================================= - - public void testSerializeLite() throws Exception { - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData); - assertEquals(allFieldsData.size(), emptyMessageLite.getSerializedSize()); - ByteString data = emptyMessageLite.toByteString(); - TestAllTypes message = TestAllTypes.parseFrom(data); - TestUtil.assertAllFieldsSet(message); - assertEquals(allFieldsData, data); - } - - public void testAllExtensionsLite() throws Exception { - TestAllExtensions allExtensions = TestUtil.getAllExtensionsSet(); - ByteString allExtensionsData = allExtensions.toByteString(); - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parser().parseFrom(allExtensionsData); - ByteString data = emptyMessageLite.toByteString(); - TestAllExtensions message = - TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry()); - TestUtil.assertAllExtensionsSet(message); - assertEquals(allExtensionsData, data); - } - - public void testAllPackedFieldsLite() throws Exception { - TestPackedTypes allPackedFields = TestUtil.getPackedSet(); - ByteString allPackedData = allPackedFields.toByteString(); - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(allPackedData); - ByteString data = emptyMessageLite.toByteString(); - TestPackedTypes message = - TestPackedTypes.parseFrom(data, TestUtil.getExtensionRegistry()); - TestUtil.assertPackedFieldsSet(message); - assertEquals(allPackedData, data); - } - - public void testAllPackedExtensionsLite() throws Exception { - TestPackedExtensions allPackedExtensions = TestUtil.getPackedExtensionsSet(); - ByteString allPackedExtensionsData = allPackedExtensions.toByteString(); - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(allPackedExtensionsData); - ByteString data = emptyMessageLite.toByteString(); - TestPackedExtensions message = - TestPackedExtensions.parseFrom(data, TestUtil.getExtensionRegistry()); - TestUtil.assertPackedExtensionsSet(message); - assertEquals(allPackedExtensionsData, data); - } - - public void testCopyFromLite() throws Exception { - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData); - UnittestLite.TestEmptyMessageLite emptyMessageLite2 = - UnittestLite.TestEmptyMessageLite.newBuilder() - .mergeFrom(emptyMessageLite).build(); - assertEquals(emptyMessageLite.toByteString(), emptyMessageLite2.toByteString()); - } - - public void testMergeFromLite() throws Exception { - TestAllTypes message1 = TestAllTypes.newBuilder() - .setOptionalInt32(1) - .setOptionalString("foo") - .addRepeatedString("bar") - .setOptionalNestedEnum(TestAllTypes.NestedEnum.BAZ) - .build(); - - TestAllTypes message2 = TestAllTypes.newBuilder() - .setOptionalInt64(2) - .setOptionalString("baz") - .addRepeatedString("qux") - .setOptionalForeignEnum(ForeignEnum.FOREIGN_BAZ) - .build(); - - ByteString data1 = message1.toByteString(); - UnittestLite.TestEmptyMessageLite emptyMessageLite1 = - UnittestLite.TestEmptyMessageLite.parseFrom(data1); - ByteString data2 = message2.toByteString(); - UnittestLite.TestEmptyMessageLite emptyMessageLite2 = - UnittestLite.TestEmptyMessageLite.parseFrom(data2); - - message1 = TestAllTypes.newBuilder(message1).mergeFrom(message2).build(); - emptyMessageLite1 = UnittestLite.TestEmptyMessageLite.newBuilder(emptyMessageLite1) - .mergeFrom(emptyMessageLite2).build(); - - data1 = emptyMessageLite1.toByteString(); - message2 = TestAllTypes.parseFrom(data1); - - assertEquals(message1, message2); - } - - public void testWrongTypeTreatedAsUnknownLite() throws Exception { - // Test that fields of the wrong wire type are treated like unknown fields - // when parsing. - - ByteString bizarroData = getBizarroData(); - TestAllTypes allTypesMessage = TestAllTypes.parseFrom(bizarroData); - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData); - ByteString data = emptyMessageLite.toByteString(); - TestAllTypes allTypesMessage2 = TestAllTypes.parseFrom(data); - - assertEquals(allTypesMessage.toString(), allTypesMessage2.toString()); - } - - public void testUnknownExtensionsLite() throws Exception { - // Make sure fields are properly parsed to the UnknownFieldSet even when - // they are declared as extension numbers. - - UnittestLite.TestEmptyMessageWithExtensionsLite message = - UnittestLite.TestEmptyMessageWithExtensionsLite.parseFrom(allFieldsData); - - assertEquals(allFieldsData, message.toByteString()); - } - - public void testWrongExtensionTypeTreatedAsUnknownLite() throws Exception { - // Test that fields of the wrong wire type are treated like unknown fields - // when parsing extensions. - - ByteString bizarroData = getBizarroData(); - TestAllExtensions allExtensionsMessage = - TestAllExtensions.parseFrom(bizarroData); - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(bizarroData); - - // All fields should have been interpreted as unknown, so the byte strings - // should be the same. - assertEquals(emptyMessageLite.toByteString(), - allExtensionsMessage.toByteString()); - } - - public void testParseUnknownEnumValueLite() throws Exception { - Descriptors.FieldDescriptor singularField = - TestAllTypes.getDescriptor().findFieldByName("optional_nested_enum"); - Descriptors.FieldDescriptor repeatedField = - TestAllTypes.getDescriptor().findFieldByName("repeated_nested_enum"); - assertNotNull(singularField); - assertNotNull(repeatedField); - - ByteString data = - UnknownFieldSet.newBuilder() - .addField(singularField.getNumber(), - UnknownFieldSet.Field.newBuilder() - .addVarint(TestAllTypes.NestedEnum.BAR.getNumber()) - .addVarint(5) // not valid - .build()) - .addField(repeatedField.getNumber(), - UnknownFieldSet.Field.newBuilder() - .addVarint(TestAllTypes.NestedEnum.FOO.getNumber()) - .addVarint(4) // not valid - .addVarint(TestAllTypes.NestedEnum.BAZ.getNumber()) - .addVarint(6) // not valid - .build()) - .build() - .toByteString(); - - UnittestLite.TestEmptyMessageLite emptyMessageLite = - UnittestLite.TestEmptyMessageLite.parseFrom(data); - data = emptyMessageLite.toByteString(); - - { - TestAllTypes message = TestAllTypes.parseFrom(data); - assertEquals(TestAllTypes.NestedEnum.BAR, - message.getOptionalNestedEnum()); - assertEquals( - Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ), - message.getRepeatedNestedEnumList()); - assertEquals(Arrays.asList(5L), - message.getUnknownFields() - .getField(singularField.getNumber()) - .getVarintList()); - assertEquals(Arrays.asList(4L, 6L), - message.getUnknownFields() - .getField(repeatedField.getNumber()) - .getVarintList()); - } - - { - TestAllExtensions message = - TestAllExtensions.parseFrom(data, TestUtil.getExtensionRegistry()); - assertEquals(TestAllTypes.NestedEnum.BAR, - message.getExtension(UnittestProto.optionalNestedEnumExtension)); - assertEquals( - Arrays.asList(TestAllTypes.NestedEnum.FOO, TestAllTypes.NestedEnum.BAZ), - message.getExtension(UnittestProto.repeatedNestedEnumExtension)); - assertEquals(Arrays.asList(5L), - message.getUnknownFields() - .getField(singularField.getNumber()) - .getVarintList()); - assertEquals(Arrays.asList(4L, 6L), - message.getUnknownFields() - .getField(repeatedField.getNumber()) - .getVarintList()); - } - } - - public void testClearLite() throws Exception { - UnittestLite.TestEmptyMessageLite emptyMessageLite1 = - UnittestLite.TestEmptyMessageLite.parseFrom(allFieldsData); - UnittestLite.TestEmptyMessageLite emptyMessageLite2 = - UnittestLite.TestEmptyMessageLite.newBuilder() - .mergeFrom(emptyMessageLite1).clear().build(); - assertEquals(0, emptyMessageLite2.getSerializedSize()); - ByteString data = emptyMessageLite2.toByteString(); - assertEquals(0, data.size()); - } - } diff --git a/java/core/src/test/java/com/google/protobuf/WireFormatTest.java b/java/core/src/test/java/com/google/protobuf/WireFormatTest.java index 370860c2..625d4b42 100644 --- a/java/core/src/test/java/com/google/protobuf/WireFormatTest.java +++ b/java/core/src/test/java/com/google/protobuf/WireFormatTest.java @@ -30,8 +30,6 @@ package com.google.protobuf; -import com.google.protobuf.UnittestLite.TestAllExtensionsLite; -import com.google.protobuf.UnittestLite.TestPackedExtensionsLite; import protobuf_unittest.UnittestMset.RawMessageSet; import protobuf_unittest.UnittestMset.TestMessageSetExtension1; import protobuf_unittest.UnittestMset.TestMessageSetExtension2; @@ -125,32 +123,6 @@ public class WireFormatTest extends TestCase { TestUtil.assertPackedFieldsSet(message2); } - public void testSerializeExtensionsLite() throws Exception { - // TestAllTypes and TestAllExtensions should have compatible wire formats, - // so if we serialize a TestAllExtensions then parse it as TestAllTypes - // it should work. - - TestAllExtensionsLite message = TestUtilLite.getAllLiteExtensionsSet(); - ByteString rawBytes = message.toByteString(); - assertEquals(rawBytes.size(), message.getSerializedSize()); - - TestAllTypes message2 = TestAllTypes.parseFrom(rawBytes); - - TestUtil.assertAllFieldsSet(message2); - } - - public void testSerializePackedExtensionsLite() throws Exception { - // TestPackedTypes and TestPackedExtensions should have compatible wire - // formats; check that they serialize to the same string. - TestPackedExtensionsLite message = TestUtilLite.getLitePackedExtensionsSet(); - ByteString rawBytes = message.toByteString(); - - TestPackedTypes message2 = TestUtil.getPackedSet(); - ByteString rawBytes2 = message2.toByteString(); - - assertEquals(rawBytes, rawBytes2); - } - public void testParseExtensions() throws Exception { // TestAllTypes and TestAllExtensions should have compatible wire formats, // so if we serialize a TestAllTypes then parse it as TestAllExtensions @@ -180,48 +152,6 @@ public class WireFormatTest extends TestCase { TestUtil.assertPackedExtensionsSet(message2); } - public void testParseExtensionsLite() throws Exception { - // TestAllTypes and TestAllExtensions should have compatible wire formats, - // so if we serialize a TestAllTypes then parse it as TestAllExtensions - // it should work. - - TestAllTypes message = TestUtil.getAllSet(); - ByteString rawBytes = message.toByteString(); - - ExtensionRegistryLite registry_lite = TestUtilLite.getExtensionRegistryLite(); - - TestAllExtensionsLite message2 = - TestAllExtensionsLite.parseFrom(rawBytes, registry_lite); - - TestUtil.assertAllExtensionsSet(message2); - - // Try again using a full extension registry. - ExtensionRegistry registry = TestUtil.getExtensionRegistry(); - - TestAllExtensionsLite message3 = - TestAllExtensionsLite.parseFrom(rawBytes, registry); - - TestUtil.assertAllExtensionsSet(message3); - } - - public void testParsePackedExtensionsLite() throws Exception { - // Ensure that packed extensions can be properly parsed. - TestPackedExtensionsLite message = TestUtilLite.getLitePackedExtensionsSet(); - ByteString rawBytes = message.toByteString(); - - ExtensionRegistryLite registry = TestUtilLite.getExtensionRegistryLite(); - - TestPackedExtensionsLite message2 = - TestPackedExtensionsLite.parseFrom(rawBytes, registry); - - TestUtil.assertPackedExtensionsSet(message2); - } - - public void testExtensionsSerializedSize() throws Exception { - assertNotSame(TestUtil.getAllSet().getSerializedSize(), - TestUtil.getAllExtensionsSet().getSerializedSize()); - } - public void testSerializeDelimited() throws Exception { ByteArrayOutputStream output = new ByteArrayOutputStream(); TestUtil.getAllSet().writeDelimitedTo(output); diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index a603d96a..a26dbc2d 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -1522,7 +1522,8 @@ public class JsonFormat { for (int i = 0; i < array.size(); ++i) { Object value = parseFieldValue(field, array.get(i), builder); if (value == null) { - throw new InvalidProtocolBufferException("Repeated field elements cannot be null"); + throw new InvalidProtocolBufferException( + "Repeated field elements cannot be null in field: " + field.getFullName()); } builder.addRepeatedField(field, value); } -- cgit v1.2.3