diff options
| author |  Jisi Liu <jisi.liu@gmail.com> | 2017-08-18 16:25:35 -0700 |
|---|---|---|
| committer | Jisi Liu <jisi.liu@gmail.com> | 2017-08-18 16:25:35 -0700 |
| commit | 139775ccc040a07e07c5407e34834dab27928cbc (patch) | |
| tree | c500f4eabfea9d2e059f18b7d612c6e2a2db3ad5 /java | |
| parent | 1825d6d8f0aedcd7966c5c137a71e46ae650b3da (diff) | |
| parent | 26ac3e8e242f1cd6de15291a9973905a14b546ba (diff) | |
Merge remote-tracking branch 'origin/3.4.x' into mergemaster
Diffstat (limited to 'java')
40 files changed, 1471 insertions, 577 deletions
diff --git a/java/core/generate-test-sources-build.xml b/java/core/generate-test-sources-build.xml index ab415db6..68951747 100644 --- a/java/core/generate-test-sources-build.xml +++ b/java/core/generate-test-sources-build.xml @@ -5,6 +5,7 @@ <arg value="--proto_path=${protobuf.source.dir}"/> <arg value="--proto_path=${test.proto.dir}"/> <arg value="${protobuf.source.dir}/google/protobuf/unittest.proto"/> + <arg value="${protobuf.source.dir}/google/protobuf/unittest_proto3.proto"/> <arg value="${protobuf.source.dir}/google/protobuf/unittest_import.proto"/> <arg value="${protobuf.source.dir}/google/protobuf/unittest_import_public.proto"/> <arg value="${protobuf.source.dir}/google/protobuf/unittest_mset.proto"/> @@ -40,4 +41,4 @@ <arg value="${test.proto.dir}/com/google/protobuf/map_test.proto"/> <arg value="${test.proto.dir}/com/google/protobuf/map_initialization_order_test.proto"/> </exec> -</project>
\ No newline at end of file +</project> diff --git a/java/core/pom.xml b/java/core/pom.xml index e4f50dd1..4608fce6 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-parent</artifactId> - <version>3.3.2</version> + <version>3.4.0</version> </parent> <artifactId>protobuf-java</artifactId> diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java index b5043eb5..065fa1a9 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessage.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessage.java @@ -32,6 +32,7 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Descriptors.FileDescriptor.Syntax; import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.Internal.EnumLite; import java.io.IOException; @@ -162,7 +163,7 @@ public abstract class AbstractMessage } return hash; } - + private static ByteString toByteString(Object value) { if (value instanceof byte[]) { return ByteString.copyFrom((byte[]) value); @@ -170,7 +171,7 @@ public abstract class AbstractMessage return (ByteString) value; } } - + /** * Compares two bytes fields. The parameters must be either a byte array or a * ByteString object. They can be of different type though. @@ -181,7 +182,7 @@ public abstract class AbstractMessage } return toByteString(a).equals(toByteString(b)); } - + /** * Converts a list of MapEntry messages into a Map used for equals() and * hashCode(). @@ -212,7 +213,7 @@ public abstract class AbstractMessage } return result; } - + /** * Compares two map fields. The parameters must be a list of MapEntry * messages. @@ -223,13 +224,13 @@ public abstract class AbstractMessage Map mb = convertMapEntryListToMap((List) b); return MapFieldLite.equals(ma, mb); } - + /** * Compares two set of fields. * This method is used to implement {@link AbstractMessage#equals(Object)} * and {@link AbstractMutableMessage#equals(Object)}. It takes special care * of bytes fields because immutable messages and mutable messages use - * different Java type to reprensent a bytes field and this method should be + * different Java type to represent a bytes field and this method should be * able to compare immutable messages, mutable messages and also an immutable * message to a mutable message. */ @@ -275,7 +276,7 @@ public abstract class AbstractMessage } return true; } - + /** * Calculates the hash code of a map field. {@code value} must be a list of * MapEntry messages. @@ -371,7 +372,7 @@ public abstract class AbstractMessage public String getInitializationErrorString() { return MessageReflection.delimitWithCommas(findInitializationErrors()); } - + @Override protected BuilderType internalMergeFrom(AbstractMessageLite other) { return mergeFrom((Message) other); @@ -432,8 +433,12 @@ public abstract class AbstractMessage final CodedInputStream input, final ExtensionRegistryLite extensionRegistry) throws IOException { + boolean discardUnknown = + getDescriptorForType().getFile().getSyntax() == Syntax.PROTO3 + ? input.shouldDiscardUnknownFieldsProto3() + : input.shouldDiscardUnknownFields(); final UnknownFieldSet.Builder unknownFields = - UnknownFieldSet.newBuilder(getUnknownFields()); + discardUnknown ? null : UnknownFieldSet.newBuilder(getUnknownFields()); while (true) { final int tag = input.readTag(); if (tag == 0) { @@ -451,7 +456,9 @@ public abstract class AbstractMessage break; } } - setUnknownFields(unknownFields.build()); + if (unknownFields != null) { + setUnknownFields(unknownFields.build()); + } return (BuilderType) this; } diff --git a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java index 99787fcc..24830c0a 100644 --- a/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java @@ -36,7 +36,9 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; /** * A partial implementation of the {@link MessageLite} interface which @@ -118,8 +120,13 @@ public abstract class AbstractMessageLite< } } - protected static <T> void addAll(final Iterable<T> values, - final Collection<? super T> list) { + // For binary compatibility + @Deprecated + protected static <T> void addAll(final Iterable<T> values, final Collection<? super T> list) { + Builder.addAll(values, (List) list); + } + + protected static <T> void addAll(final Iterable<T> values, final List<? super T> list) { Builder.addAll(values, list); } @@ -334,6 +341,25 @@ public abstract class AbstractMessageLite< + " threw an IOException (should never happen)."; } + // We check nulls as we iterate to avoid iterating over values twice. + private static <T> void addAllCheckingNulls(Iterable<T> values, List<? super T> list) { + if (list instanceof ArrayList && values instanceof Collection) { + ((ArrayList<T>) list).ensureCapacity(list.size() + ((Collection<T>) values).size()); + } + int begin = list.size(); + for (T value : values) { + if (value == null) { + // encountered a null value so we must undo our modifications prior to throwing + String message = "Element at index " + (list.size() - begin) + " is null."; + for (int i = list.size() - 1; i >= begin; i--) { + list.remove(i); + } + throw new NullPointerException(message); + } + list.add(value); + } + } + /** * Construct an UninitializedMessageException reporting missing fields in * the given message. @@ -343,16 +369,20 @@ public abstract class AbstractMessageLite< return new UninitializedMessageException(message); } + // For binary compatibility. + @Deprecated + protected static <T> void addAll(final Iterable<T> values, final Collection<? super T> list) { + addAll(values, (List<T>) list); + } + /** - * Adds the {@code values} to the {@code list}. This is a helper method - * used by generated code. Users should ignore it. + * Adds the {@code values} to the {@code list}. This is a helper method used by generated code. + * Users should ignore it. * - * @throws NullPointerException if {@code values} or any of the elements of - * {@code values} is null. When that happens, some elements of - * {@code values} may have already been added to the result {@code list}. + * @throws NullPointerException if {@code values} or any of the elements of {@code values} is + * null. */ - protected static <T> void addAll(final Iterable<T> values, - final Collection<? super T> list) { + protected static <T> void addAll(final Iterable<T> values, final List<? super T> list) { checkNotNull(values); if (values instanceof LazyStringList) { // For StringOrByteStringLists, check the underlying elements to avoid @@ -360,25 +390,31 @@ public abstract class AbstractMessageLite< // TODO(dweis): Could we just prohibit nulls in all protobuf lists and get rid of this? Is // if even possible to hit this condition as all protobuf methods check for null first, // right? - checkForNullValues(((LazyStringList) values).getUnderlyingElements()); - list.addAll((Collection<T>) values); - } else if (values instanceof Collection) { - if (!(values instanceof PrimitiveNonBoxingCollection)) { - checkForNullValues(values); + List<?> lazyValues = ((LazyStringList) values).getUnderlyingElements(); + LazyStringList lazyList = (LazyStringList) list; + int begin = list.size(); + for (Object value : lazyValues) { + if (value == null) { + // encountered a null value so we must undo our modifications prior to throwing + String message = "Element at index " + (lazyList.size() - begin) + " is null."; + for (int i = lazyList.size() - 1; i >= begin; i--) { + lazyList.remove(i); + } + throw new NullPointerException(message); + } + if (value instanceof ByteString) { + lazyList.add((ByteString) value); + } else { + lazyList.add((String) value); + } } - list.addAll((Collection<T>) values); } else { - for (final T value : values) { - checkNotNull(value); - list.add(value); + if (values instanceof PrimitiveNonBoxingCollection) { + list.addAll((Collection<T>) values); + } else { + addAllCheckingNulls(values, list); } } } - - private static void checkForNullValues(final Iterable<?> values) { - for (final Object value : values) { - checkNotNull(value); - } - } } } diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java index 3dfbcb0a..d6a941b1 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java @@ -34,8 +34,8 @@ import static com.google.protobuf.Internal.EMPTY_BYTE_ARRAY; import static com.google.protobuf.Internal.EMPTY_BYTE_BUFFER; import static com.google.protobuf.Internal.UTF_8; import static com.google.protobuf.Internal.checkNotNull; -import static com.google.protobuf.WireFormat.FIXED_32_SIZE; -import static com.google.protobuf.WireFormat.FIXED_64_SIZE; +import static com.google.protobuf.WireFormat.FIXED32_SIZE; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; import java.io.ByteArrayOutputStream; @@ -372,6 +372,64 @@ public abstract class CodedInputStream { return oldLimit; } + + private boolean explicitDiscardUnknownFields = false; + + /** TODO(liujisi): flip the default.*/ + private static volatile boolean proto3DiscardUnknownFieldsDefault = true; + + static void setProto3DiscardUnknownsByDefaultForTest() { + proto3DiscardUnknownFieldsDefault = true; + } + + static void setProto3KeepUnknownsByDefaultForTest() { + proto3DiscardUnknownFieldsDefault = false; + } + + static boolean getProto3DiscardUnknownFieldsDefault() { + return proto3DiscardUnknownFieldsDefault; + } + + /** + * Sets this {@code CodedInputStream} to discard unknown fields. Only applies to full runtime + * messages; lite messages will always preserve unknowns. + * + * <p>Note calling this function alone will have NO immediate effect on the underlying input data. + * The unknown fields will be discarded during parsing. This affects both Proto2 and Proto3 full + * runtime. + */ + final void discardUnknownFields() { + explicitDiscardUnknownFields = true; + } + + /** + * Reverts the unknown fields preservation behavior for Proto2 and Proto3 full runtime to their + * default. + */ + final void unsetDiscardUnknownFields() { + explicitDiscardUnknownFields = false; + } + + /** + * Whether unknown fields in this input stream should be discarded during parsing into full + * runtime messages. + */ + final boolean shouldDiscardUnknownFields() { + return explicitDiscardUnknownFields; + } + + /** + * Whether unknown fields in this input stream should be discarded during parsing for proto3 full + * runtime messages. + * + * <p>This function was temporarily introduced before proto3 unknown fields behavior is changed. + * TODO(liujisi): remove this and related code in GeneratedMessage after proto3 unknown + * fields migration is done. + */ + final boolean shouldDiscardUnknownFieldsProto3() { + return explicitDiscardUnknownFields ? true : proto3DiscardUnknownFieldsDefault; + } + /** * Resets the current size counter to zero (see {@link #setSizeLimit(int)}). Only valid for {@link * InputStream}-backed streams. @@ -572,7 +630,7 @@ public abstract class CodedInputStream { skipRawVarint(); return true; case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED_64_SIZE); + skipRawBytes(FIXED64_SIZE); return true; case WireFormat.WIRETYPE_LENGTH_DELIMITED: skipRawBytes(readRawVarint32()); @@ -585,7 +643,7 @@ public abstract class CodedInputStream { case WireFormat.WIRETYPE_END_GROUP: return false; case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED_32_SIZE); + skipRawBytes(FIXED32_SIZE); return true; default: throw InvalidProtocolBufferException.invalidWireType(); @@ -1064,12 +1122,12 @@ public abstract class CodedInputStream { public int readRawLittleEndian32() throws IOException { int tempPos = pos; - if (limit - tempPos < FIXED_32_SIZE) { + if (limit - tempPos < FIXED32_SIZE) { throw InvalidProtocolBufferException.truncatedMessage(); } final byte[] buffer = this.buffer; - pos = tempPos + FIXED_32_SIZE; + pos = tempPos + FIXED32_SIZE; return (((buffer[tempPos] & 0xff)) | ((buffer[tempPos + 1] & 0xff) << 8) | ((buffer[tempPos + 2] & 0xff) << 16) @@ -1080,12 +1138,12 @@ public abstract class CodedInputStream { public long readRawLittleEndian64() throws IOException { int tempPos = pos; - if (limit - tempPos < FIXED_64_SIZE) { + if (limit - tempPos < FIXED64_SIZE) { throw InvalidProtocolBufferException.truncatedMessage(); } final byte[] buffer = this.buffer; - pos = tempPos + FIXED_64_SIZE; + pos = tempPos + FIXED64_SIZE; return (((buffer[tempPos] & 0xffL)) | ((buffer[tempPos + 1] & 0xffL) << 8) | ((buffer[tempPos + 2] & 0xffL) << 16) @@ -1290,7 +1348,7 @@ public abstract class CodedInputStream { skipRawVarint(); return true; case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED_64_SIZE); + skipRawBytes(FIXED64_SIZE); return true; case WireFormat.WIRETYPE_LENGTH_DELIMITED: skipRawBytes(readRawVarint32()); @@ -1303,7 +1361,7 @@ public abstract class CodedInputStream { case WireFormat.WIRETYPE_END_GROUP: return false; case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED_32_SIZE); + skipRawBytes(FIXED32_SIZE); return true; default: throw InvalidProtocolBufferException.invalidWireType(); @@ -1429,7 +1487,9 @@ public abstract class CodedInputStream { final int size = readRawVarint32(); if (size > 0 && size <= remaining()) { // TODO(nathanmittler): Is there a way to avoid this copy? - byte[] bytes = copyToArray(pos, pos + size); + // The same as readBytes' logic + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); String result = new String(bytes, UTF_8); pos += size; return result; @@ -1449,7 +1509,9 @@ public abstract class CodedInputStream { final int size = readRawVarint32(); if (size >= 0 && size <= remaining()) { // TODO(nathanmittler): Is there a way to avoid this copy? - byte[] bytes = copyToArray(pos, pos + size); + // The same as readBytes' logic + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); // TODO(martinrb): We could save a pass by validating while decoding. if (!Utf8.isValidUtf8(bytes)) { throw InvalidProtocolBufferException.invalidUtf8(); @@ -1545,14 +1607,17 @@ public abstract class CodedInputStream { public ByteString readBytes() throws IOException { final int size = readRawVarint32(); if (size > 0 && size <= remaining()) { - ByteBuffer result; if (immutable && enableAliasing) { - result = slice(pos, pos + size); + final ByteBuffer result = slice(pos, pos + size); + pos += size; + return ByteString.wrap(result); } else { - result = copy(pos, pos + size); + // Use UnsafeUtil to copy the memory to bytes instead of using ByteBuffer ways. + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); + pos += size; + return ByteString.wrap(bytes); } - pos += size; - return ByteString.wrap(result); } if (size == 0) { @@ -1573,18 +1638,21 @@ public abstract class CodedInputStream { public ByteBuffer readByteBuffer() throws IOException { final int size = readRawVarint32(); if (size > 0 && size <= remaining()) { - ByteBuffer result; // "Immutable" implies that buffer is backing a ByteString. // Disallow slicing in this case to prevent the caller from modifying the contents // of the ByteString. if (!immutable && enableAliasing) { - result = slice(pos, pos + size); + final ByteBuffer result = slice(pos, pos + size); + pos += size; + return result; } else { - result = copy(pos, pos + size); + // The same as readBytes' logic + byte[] bytes = new byte[size]; + UnsafeUtil.copyMemory(pos, bytes, 0, size); + pos += size; + return ByteBuffer.wrap(bytes); } - pos += size; // TODO(nathanmittler): Investigate making the ByteBuffer be made read-only - return result; } if (size == 0) { @@ -1785,11 +1853,11 @@ public abstract class CodedInputStream { public int readRawLittleEndian32() throws IOException { long tempPos = pos; - if (limit - tempPos < FIXED_32_SIZE) { + if (limit - tempPos < FIXED32_SIZE) { throw InvalidProtocolBufferException.truncatedMessage(); } - pos = tempPos + FIXED_32_SIZE; + pos = tempPos + FIXED32_SIZE; return (((UnsafeUtil.getByte(tempPos) & 0xff)) | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8) | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16) @@ -1800,11 +1868,11 @@ public abstract class CodedInputStream { public long readRawLittleEndian64() throws IOException { long tempPos = pos; - if (limit - tempPos < FIXED_64_SIZE) { + if (limit - tempPos < FIXED64_SIZE) { throw InvalidProtocolBufferException.truncatedMessage(); } - pos = tempPos + FIXED_64_SIZE; + pos = tempPos + FIXED64_SIZE; return (((UnsafeUtil.getByte(tempPos) & 0xffL)) | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8) | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16) @@ -1943,27 +2011,6 @@ public abstract class CodedInputStream { buffer.limit(prevLimit); } } - - private ByteBuffer copy(long begin, long end) throws IOException { - return ByteBuffer.wrap(copyToArray(begin, end)); - } - - private byte[] copyToArray(long begin, long end) throws IOException { - int prevPos = buffer.position(); - int prevLimit = buffer.limit(); - try { - buffer.position(bufferPos(begin)); - buffer.limit(bufferPos(end)); - byte[] bytes = new byte[(int) (end - begin)]; - buffer.get(bytes); - return bytes; - } catch (IllegalArgumentException e) { - throw InvalidProtocolBufferException.truncatedMessage(); - } finally { - buffer.position(prevPos); - buffer.limit(prevLimit); - } - } } /** @@ -2034,7 +2081,7 @@ public abstract class CodedInputStream { skipRawVarint(); return true; case WireFormat.WIRETYPE_FIXED64: - skipRawBytes(FIXED_64_SIZE); + skipRawBytes(FIXED64_SIZE); return true; case WireFormat.WIRETYPE_LENGTH_DELIMITED: skipRawBytes(readRawVarint32()); @@ -2047,7 +2094,7 @@ public abstract class CodedInputStream { case WireFormat.WIRETYPE_END_GROUP: return false; case WireFormat.WIRETYPE_FIXED32: - skipRawBytes(FIXED_32_SIZE); + skipRawBytes(FIXED32_SIZE); return true; default: throw InvalidProtocolBufferException.invalidWireType(); @@ -2332,8 +2379,7 @@ public abstract class CodedInputStream { if (size == 0) { return ByteString.EMPTY; } - // Slow path: Build a byte array first then copy it. - return ByteString.wrap(readRawBytesSlowPath(size)); + return readBytesSlowPath(size); } @Override @@ -2558,13 +2604,13 @@ public abstract class CodedInputStream { public int readRawLittleEndian32() throws IOException { int tempPos = pos; - if (bufferSize - tempPos < FIXED_32_SIZE) { - refillBuffer(FIXED_32_SIZE); + if (bufferSize - tempPos < FIXED32_SIZE) { + refillBuffer(FIXED32_SIZE); tempPos = pos; } final byte[] buffer = this.buffer; - pos = tempPos + FIXED_32_SIZE; + pos = tempPos + FIXED32_SIZE; return (((buffer[tempPos] & 0xff)) | ((buffer[tempPos + 1] & 0xff) << 8) | ((buffer[tempPos + 2] & 0xff) << 16) @@ -2575,13 +2621,13 @@ public abstract class CodedInputStream { public long readRawLittleEndian64() throws IOException { int tempPos = pos; - if (bufferSize - tempPos < FIXED_64_SIZE) { - refillBuffer(FIXED_64_SIZE); + if (bufferSize - tempPos < FIXED64_SIZE) { + refillBuffer(FIXED64_SIZE); tempPos = pos; } final byte[] buffer = this.buffer; - pos = tempPos + FIXED_64_SIZE; + pos = tempPos + FIXED64_SIZE; return (((buffer[tempPos] & 0xffL)) | ((buffer[tempPos + 1] & 0xffL) << 8) | ((buffer[tempPos + 2] & 0xffL) << 16) @@ -2675,7 +2721,13 @@ public abstract class CodedInputStream { */ private void refillBuffer(int n) throws IOException { if (!tryRefillBuffer(n)) { - throw InvalidProtocolBufferException.truncatedMessage(); + // We have to distinguish the exception between sizeLimitExceeded and truncatedMessage. So + // we just throw an sizeLimitExceeded exception here if it exceeds the sizeLimit + if (n > sizeLimit - totalBytesRetired - pos) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } else { + throw InvalidProtocolBufferException.truncatedMessage(); + } } } @@ -2684,8 +2736,8 @@ public abstract class CodedInputStream { * buffer. Caller must ensure that the requested space is not yet available, and that the * requested space is less than BUFFER_SIZE. * - * @return {@code true} if the bytes could be made available; {@code false} if the end of the - * stream or the current limit was reached. + * @return {@code true} If the bytes could be made available; {@code false} 1. Current at the + * end of the stream 2. The current limit was reached 3. The total size limit was reached */ private boolean tryRefillBuffer(int n) throws IOException { if (pos + n <= bufferSize) { @@ -2693,6 +2745,14 @@ public abstract class CodedInputStream { "refillBuffer() called when " + n + " bytes were already available in buffer"); } + // Check whether the size of total message needs to read is bigger than the size limit. + // We shouldn't throw an exception here as isAtEnd() function needs to get this function's + // return as the result. + if (n > sizeLimit - totalBytesRetired - pos) { + return false; + } + + // Shouldn't throw the exception here either. if (totalBytesRetired + pos + n > currentLimit) { // Oops, we hit a limit. return false; @@ -2712,7 +2772,16 @@ public abstract class CodedInputStream { pos = 0; } - int bytesRead = input.read(buffer, bufferSize, buffer.length - bufferSize); + // Here we should refill the buffer as many bytes as possible. + int bytesRead = + input.read( + buffer, + bufferSize, + Math.min( + // the size of allocated but unused bytes in the buffer + buffer.length - bufferSize, + // do not exceed the total bytes limit + sizeLimit - totalBytesRetired - bufferSize)); if (bytesRead == 0 || bytesRead < -1 || bytesRead > buffer.length) { throw new IllegalStateException( "InputStream#read(byte[]) returned invalid result: " @@ -2721,10 +2790,6 @@ public abstract class CodedInputStream { } if (bytesRead > 0) { bufferSize += bytesRead; - // Integer-overflow-conscious check against sizeLimit - if (totalBytesRetired + n - sizeLimit > 0) { - throw InvalidProtocolBufferException.sizeLimitExceeded(); - } recomputeBufferSizeAfterLimit(); return (bufferSize >= n) ? true : tryRefillBuffer(n); } @@ -2756,6 +2821,49 @@ public abstract class CodedInputStream { * (bufferSize - pos) && size > 0) */ private byte[] readRawBytesSlowPath(final int size) throws IOException { + // Attempt to read the data in one byte array when it's safe to do. + byte[] result = readRawBytesSlowPathOneChunk(size); + if (result != null) { + return result; + } + + final int originalBufferPos = pos; + final int bufferedBytes = bufferSize - pos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; + + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + + // The size is very large. For security reasons we read them in small + // chunks. + List<byte[]> chunks = readRawBytesSlowPathRemainingChunks(sizeLeft); + + // OK, got everything. Now concatenate it all into one buffer. + final byte[] bytes = new byte[size]; + + // Start by copying the leftover bytes from this.buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // And now all the chunks. + int tempPos = bufferedBytes; + for (final byte[] chunk : chunks) { + System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); + tempPos += chunk.length; + } + + // Done. + return bytes; + } + + /** + * Attempts to read the data in one byte array when it's safe to do. Returns null if the size to + * read is too large and needs to be allocated in smaller chunks for security reasons. + */ + private byte[] readRawBytesSlowPathOneChunk(final int size) throws IOException { if (size == 0) { return Internal.EMPTY_BYTE_ARRAY; } @@ -2776,14 +2884,7 @@ public abstract class CodedInputStream { throw InvalidProtocolBufferException.truncatedMessage(); } - final int originalBufferPos = pos; final int bufferedBytes = bufferSize - pos; - - // Mark the current buffer consumed. - totalBytesRetired += bufferSize; - pos = 0; - bufferSize = 0; - // Determine the number of bytes we need to read from the input stream. int sizeLeft = size - bufferedBytes; // TODO(nathanmittler): Consider using a value larger than DEFAULT_BUFFER_SIZE. @@ -2793,7 +2894,10 @@ public abstract class CodedInputStream { final byte[] bytes = new byte[size]; // Copy all of the buffered bytes to the result buffer. - System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + System.arraycopy(buffer, pos, bytes, 0, bufferedBytes); + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; // Fill the remaining bytes from the input stream. int tempPos = bufferedBytes; @@ -2809,6 +2913,11 @@ public abstract class CodedInputStream { return bytes; } + return null; + } + + /** Reads the remaining data in small chunks from the input stream. */ + private List<byte[]> readRawBytesSlowPathRemainingChunks(int sizeLeft) throws IOException { // The size is very large. For security reasons, we can't allocate the // entire byte array yet. The size comes directly from the input, so a // maliciously-crafted message could provide a bogus very large size in @@ -2834,21 +2943,41 @@ public abstract class CodedInputStream { chunks.add(chunk); } - // OK, got everything. Now concatenate it all into one buffer. - final byte[] bytes = new byte[size]; - - // Start by copying the leftover bytes from this.buffer. - System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + return chunks; + } - // And now all the chunks. - int tempPos = bufferedBytes; - for (final byte[] chunk : chunks) { - System.arraycopy(chunk, 0, bytes, tempPos, chunk.length); - tempPos += chunk.length; + /** + * Like readBytes, but caller must have already checked the fast path: (size <= (bufferSize - + * pos) && size > 0 || size == 0) + */ + private ByteString readBytesSlowPath(final int size) throws IOException { + final byte[] result = readRawBytesSlowPathOneChunk(size); + if (result != null) { + return ByteString.wrap(result); } - // Done. - return bytes; + final int originalBufferPos = pos; + final int bufferedBytes = bufferSize - pos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + pos = 0; + bufferSize = 0; + + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + + // The size is very large. For security reasons we read them in small + // chunks. + List<byte[]> chunks = readRawBytesSlowPathRemainingChunks(sizeLeft); + + // Wrap the byte arrays into a single ByteString. + List<ByteString> byteStrings = new ArrayList<ByteString>(1 + chunks.size()); + byteStrings.add(ByteString.copyFrom(buffer, originalBufferPos, bufferedBytes)); + for (byte[] chunk : chunks) { + byteStrings.add(ByteString.wrap(chunk)); + } + return ByteString.copyFrom(byteStrings); } @Override diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java index da0e9b12..093a5f61 100644 --- a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -30,8 +30,8 @@ package com.google.protobuf; -import static com.google.protobuf.WireFormat.FIXED_32_SIZE; -import static com.google.protobuf.WireFormat.FIXED_64_SIZE; +import static com.google.protobuf.WireFormat.FIXED32_SIZE; +import static com.google.protobuf.WireFormat.FIXED64_SIZE; import static com.google.protobuf.WireFormat.MAX_VARINT_SIZE; import static java.lang.Math.max; @@ -59,13 +59,12 @@ import java.util.logging.Logger; public abstract class CodedOutputStream extends ByteOutput { private static final Logger logger = Logger.getLogger(CodedOutputStream.class.getName()); private static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = UnsafeUtil.hasUnsafeArrayOperations(); - private static final long ARRAY_BASE_OFFSET = UnsafeUtil.getArrayBaseOffset(); /** * @deprecated Use {@link #computeFixed32SizeNoTag(int)} instead. */ @Deprecated - public static final int LITTLE_ENDIAN_32_SIZE = FIXED_32_SIZE; + public static final int LITTLE_ENDIAN_32_SIZE = FIXED32_SIZE; /** * The buffer size used in {@link #newInstance(OutputStream)}. @@ -755,7 +754,7 @@ public abstract class CodedOutputStream extends ByteOutput { * {@code fixed32} field. */ public static int computeFixed32SizeNoTag(@SuppressWarnings("unused") final int unused) { - return FIXED_32_SIZE; + return FIXED32_SIZE; } /** @@ -763,7 +762,7 @@ public abstract class CodedOutputStream extends ByteOutput { * {@code sfixed32} field. */ public static int computeSFixed32SizeNoTag(@SuppressWarnings("unused") final int unused) { - return FIXED_32_SIZE; + return FIXED32_SIZE; } /** @@ -813,7 +812,7 @@ public abstract class CodedOutputStream extends ByteOutput { * {@code fixed64} field. */ public static int computeFixed64SizeNoTag(@SuppressWarnings("unused") final long unused) { - return FIXED_64_SIZE; + return FIXED64_SIZE; } /** @@ -821,7 +820,7 @@ public abstract class CodedOutputStream extends ByteOutput { * {@code sfixed64} field. */ public static int computeSFixed64SizeNoTag(@SuppressWarnings("unused") final long unused) { - return FIXED_64_SIZE; + return FIXED64_SIZE; } /** @@ -829,7 +828,7 @@ public abstract class CodedOutputStream extends ByteOutput { * {@code float} field, including tag. */ public static int computeFloatSizeNoTag(@SuppressWarnings("unused") final float unused) { - return FIXED_32_SIZE; + return FIXED32_SIZE; } /** @@ -837,7 +836,7 @@ public abstract class CodedOutputStream extends ByteOutput { * {@code double} field, including tag. */ public static int computeDoubleSizeNoTag(@SuppressWarnings("unused") final double unused) { - return FIXED_64_SIZE; + return FIXED64_SIZE; } /** @@ -1321,15 +1320,12 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public final void writeUInt32NoTag(int value) throws IOException { if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT_SIZE) { - long pos = ARRAY_BASE_OFFSET + position; while (true) { if ((value & ~0x7F) == 0) { - UnsafeUtil.putByte(buffer, pos++, (byte) value); - position++; + UnsafeUtil.putByte(buffer, position++, (byte) value); return; } else { - UnsafeUtil.putByte(buffer, pos++, (byte) ((value & 0x7F) | 0x80)); - position++; + UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80)); value >>>= 7; } } @@ -1367,15 +1363,12 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public final void writeUInt64NoTag(long value) throws IOException { if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT_SIZE) { - long pos = ARRAY_BASE_OFFSET + position; while (true) { if ((value & ~0x7FL) == 0) { - UnsafeUtil.putByte(buffer, pos++, (byte) value); - position++; + UnsafeUtil.putByte(buffer, position++, (byte) value); return; } else { - UnsafeUtil.putByte(buffer, pos++, (byte) (((int) value & 0x7F) | 0x80)); - position++; + UnsafeUtil.putByte(buffer, position++, (byte) (((int) value & 0x7F) | 0x80)); value >>>= 7; } } @@ -1854,7 +1847,7 @@ public abstract class CodedOutputStream extends ByteOutput { } static boolean isSupported() { - return UnsafeUtil.hasUnsafeByteBufferOperations() && UnsafeUtil.hasUnsafeCopyMemory(); + return UnsafeUtil.hasUnsafeByteBufferOperations(); } @Override @@ -2030,7 +2023,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed32NoTag(int value) throws IOException { buffer.putInt(bufferPos(position), value); - position += FIXED_32_SIZE; + position += FIXED32_SIZE; } @Override @@ -2064,7 +2057,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed64NoTag(long value) throws IOException { buffer.putLong(bufferPos(position), value); - position += FIXED_64_SIZE; + position += FIXED64_SIZE; } @Override @@ -2081,8 +2074,7 @@ public abstract class CodedOutputStream extends ByteOutput { String.format("Pos: %d, limit: %d, len: %d", position, limit, length)); } - UnsafeUtil.copyMemory( - value, UnsafeUtil.getArrayBaseOffset() + offset, null, position, length); + UnsafeUtil.copyMemory(value, offset, position, length); position += length; } @@ -2249,19 +2241,17 @@ public abstract class CodedOutputStream extends ByteOutput { */ final void bufferUInt32NoTag(int value) { if (HAS_UNSAFE_ARRAY_OPERATIONS) { - final long originalPos = ARRAY_BASE_OFFSET + position; - long pos = originalPos; + final long originalPos = position; while (true) { if ((value & ~0x7F) == 0) { - UnsafeUtil.putByte(buffer, pos++, (byte) value); + UnsafeUtil.putByte(buffer, position++, (byte) value); break; } else { - UnsafeUtil.putByte(buffer, pos++, (byte) ((value & 0x7F) | 0x80)); + UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80)); value >>>= 7; } } - int delta = (int) (pos - originalPos); - position += delta; + int delta = (int) (position - originalPos); totalBytesWritten += delta; } else { while (true) { @@ -2284,19 +2274,17 @@ public abstract class CodedOutputStream extends ByteOutput { */ final void bufferUInt64NoTag(long value) { if (HAS_UNSAFE_ARRAY_OPERATIONS) { - final long originalPos = ARRAY_BASE_OFFSET + position; - long pos = originalPos; + final long originalPos = position; while (true) { if ((value & ~0x7FL) == 0) { - UnsafeUtil.putByte(buffer, pos++, (byte) value); + UnsafeUtil.putByte(buffer, position++, (byte) value); break; } else { - UnsafeUtil.putByte(buffer, pos++, (byte) (((int) value & 0x7F) | 0x80)); + UnsafeUtil.putByte(buffer, position++, (byte) (((int) value & 0x7F) | 0x80)); value >>>= 7; } } - int delta = (int) (pos - originalPos); - position += delta; + int delta = (int) (position - originalPos); totalBytesWritten += delta; } else { while (true) { @@ -2322,7 +2310,7 @@ public abstract class CodedOutputStream extends ByteOutput { buffer[position++] = (byte) ((value >> 8) & 0xFF); buffer[position++] = (byte) ((value >> 16) & 0xFF); buffer[position++] = (byte) ((value >> 24) & 0xFF); - totalBytesWritten += FIXED_32_SIZE; + totalBytesWritten += FIXED32_SIZE; } /** @@ -2338,7 +2326,7 @@ public abstract class CodedOutputStream extends ByteOutput { buffer[position++] = (byte) ((int) (value >> 40) & 0xFF); buffer[position++] = (byte) ((int) (value >> 48) & 0xFF); buffer[position++] = (byte) ((int) (value >> 56) & 0xFF); - totalBytesWritten += FIXED_64_SIZE; + totalBytesWritten += FIXED64_SIZE; } } @@ -2379,7 +2367,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED_32_SIZE); + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED32_SIZE); bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); bufferFixed32NoTag(value); } @@ -2393,7 +2381,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed64(final int fieldNumber, final long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED_64_SIZE); + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED64_SIZE); bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); bufferFixed64NoTag(value); } @@ -2519,7 +2507,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed32NoTag(final int value) throws IOException { - flushIfNotAvailable(FIXED_32_SIZE); + flushIfNotAvailable(FIXED32_SIZE); bufferFixed32NoTag(value); } @@ -2531,7 +2519,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed64NoTag(final long value) throws IOException { - flushIfNotAvailable(FIXED_64_SIZE); + flushIfNotAvailable(FIXED64_SIZE); bufferFixed64NoTag(value); } @@ -2682,7 +2670,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed32(final int fieldNumber, final int value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED_32_SIZE); + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED32_SIZE); bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED32); bufferFixed32NoTag(value); } @@ -2696,7 +2684,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed64(final int fieldNumber, final long value) throws IOException { - flushIfNotAvailable(MAX_VARINT_SIZE + FIXED_64_SIZE); + flushIfNotAvailable(MAX_VARINT_SIZE + FIXED64_SIZE); bufferTag(fieldNumber, WireFormat.WIRETYPE_FIXED64); bufferFixed64NoTag(value); } @@ -2822,7 +2810,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed32NoTag(final int value) throws IOException { - flushIfNotAvailable(FIXED_32_SIZE); + flushIfNotAvailable(FIXED32_SIZE); bufferFixed32NoTag(value); } @@ -2834,7 +2822,7 @@ public abstract class CodedOutputStream extends ByteOutput { @Override public void writeFixed64NoTag(final long value) throws IOException { - flushIfNotAvailable(FIXED_64_SIZE); + flushIfNotAvailable(FIXED64_SIZE); bufferFixed64NoTag(value); } diff --git a/java/core/src/main/java/com/google/protobuf/DiscardUnknownFieldsParser.java b/java/core/src/main/java/com/google/protobuf/DiscardUnknownFieldsParser.java new file mode 100644 index 00000000..7ae94349 --- /dev/null +++ b/java/core/src/main/java/com/google/protobuf/DiscardUnknownFieldsParser.java @@ -0,0 +1,71 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +/** + * Parsers to discard unknown fields during parsing. + */ +public final class DiscardUnknownFieldsParser { + + /** + * Warps a given {@link Parser} into a new {@link Parser} that discards unknown fields during + * parsing. + * + * <p>Usage example: + * <pre>{@code + * private final static Parser<Foo> FOO_PARSER = DiscardUnknownFieldsParser.wrap(Foo.parser()); + * Foo parseFooDiscardUnknown(ByteBuffer input) throws IOException { + * return FOO_PARSER.parseFrom(input); + * } + * }</pre> + * + * <p>Like all other implementations of {@code Parser}, this parser is stateless and thread-safe. + * + * @param parser The delegated parser that parses messages. + * @return a {@link Parser} that will discard unknown fields during parsing. + */ + public static final <T extends Message> Parser<T> wrap(final Parser<T> parser) { + return new AbstractParser<T>() { + @Override + public T parsePartialFrom(CodedInputStream input, ExtensionRegistryLite extensionRegistry) + throws InvalidProtocolBufferException { + try { + input.discardUnknownFields(); + return parser.parsePartialFrom(input, extensionRegistry); + } finally { + input.unsetDiscardUnknownFields(); + } + } + }; + } + + private DiscardUnknownFieldsParser() {} +} diff --git a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java index 2631db74..ba532021 100644 --- a/java/core/src/main/java/com/google/protobuf/DynamicMessage.java +++ b/java/core/src/main/java/com/google/protobuf/DynamicMessage.java @@ -590,9 +590,8 @@ public final class DynamicMessage extends AbstractMessage { @Override public Builder setUnknownFields(UnknownFieldSet unknownFields) { - if (getDescriptorForType().getFile().getSyntax() - == Descriptors.FileDescriptor.Syntax.PROTO3) { - // Proto3 discards unknown fields. + if (getDescriptorForType().getFile().getSyntax() == Descriptors.FileDescriptor.Syntax.PROTO3 + && CodedInputStream.getProto3DiscardUnknownFieldsDefault()) { return this; } this.unknownFields = unknownFields; @@ -601,9 +600,8 @@ public final class DynamicMessage extends AbstractMessage { @Override public Builder mergeUnknownFields(UnknownFieldSet unknownFields) { - if (getDescriptorForType().getFile().getSyntax() - == Descriptors.FileDescriptor.Syntax.PROTO3) { - // Proto3 discards unknown fields. + if (getDescriptorForType().getFile().getSyntax() == Descriptors.FileDescriptor.Syntax.PROTO3 + && CodedInputStream.getProto3DiscardUnknownFieldsDefault()) { return this; } this.unknownFields = diff --git a/java/core/src/main/java/com/google/protobuf/Extension.java b/java/core/src/main/java/com/google/protobuf/Extension.java index 08ec5b45..5df12e64 100644 --- a/java/core/src/main/java/com/google/protobuf/Extension.java +++ b/java/core/src/main/java/com/google/protobuf/Extension.java @@ -58,10 +58,7 @@ public abstract class Extension<ContainingType extends MessageLite, Type> PROTO1, } - protected ExtensionType getExtensionType() { - // TODO(liujisi): make this abstract after we fix proto1. - return ExtensionType.IMMUTABLE; - } + protected abstract ExtensionType getExtensionType(); /** * Type of a message extension. @@ -70,7 +67,7 @@ public abstract class Extension<ContainingType extends MessageLite, Type> PROTO1, PROTO2, } - + /** * If the extension is a message extension (i.e., getLiteType() == MESSAGE), * returns the type of the message, otherwise undefined. diff --git a/java/core/src/main/java/com/google/protobuf/ExtensionRegistryFactory.java b/java/core/src/main/java/com/google/protobuf/ExtensionRegistryFactory.java index 23174e24..89f7ab9b 100644 --- a/java/core/src/main/java/com/google/protobuf/ExtensionRegistryFactory.java +++ b/java/core/src/main/java/com/google/protobuf/ExtensionRegistryFactory.java @@ -34,7 +34,7 @@ import static com.google.protobuf.ExtensionRegistryLite.EMPTY_REGISTRY_LITE; /** * A factory object to create instances of {@link ExtensionRegistryLite}. - * + * * <p> * This factory detects (via reflection) if the full (non-Lite) protocol buffer libraries * are available, and if so, the instances returned are actually {@link ExtensionRegistry}. @@ -82,6 +82,7 @@ final class ExtensionRegistryFactory { return EMPTY_REGISTRY_LITE; } + static boolean isFullRegistry(ExtensionRegistryLite registry) { return EXTENSION_REGISTRY_CLASS != null && EXTENSION_REGISTRY_CLASS.isAssignableFrom(registry.getClass()); @@ -90,6 +91,6 @@ final class ExtensionRegistryFactory { private static final ExtensionRegistryLite invokeSubclassFactory(String methodName) throws Exception { return (ExtensionRegistryLite) EXTENSION_REGISTRY_CLASS - .getMethod(methodName).invoke(null); + .getDeclaredMethod(methodName).invoke(null); } } diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java index cf3486e1..7116ae1c 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -107,11 +107,12 @@ public abstract class GeneratedMessageLite< @SuppressWarnings("unchecked") // Guaranteed by runtime @Override public int hashCode() { - if (memoizedHashCode == 0) { - HashCodeVisitor visitor = new HashCodeVisitor(); - visit(visitor, (MessageType) this); - memoizedHashCode = visitor.hashCode; + if (memoizedHashCode != 0) { + return memoizedHashCode; } + HashCodeVisitor visitor = new HashCodeVisitor(); + visit(visitor, (MessageType) this); + memoizedHashCode = visitor.hashCode; return memoizedHashCode; } @@ -331,7 +332,7 @@ public abstract class GeneratedMessageLite< if (isBuilt) { MessageType newInstance = (MessageType) instance.dynamicMethod(MethodToInvoke.NEW_MUTABLE_INSTANCE); - newInstance.visit(MergeFromVisitor.INSTANCE, instance); + mergeFromInstance(newInstance, instance); instance = newInstance; isBuilt = false; } @@ -386,10 +387,14 @@ public abstract class GeneratedMessageLite< /** All subclasses implement this. */ public BuilderType mergeFrom(MessageType message) { copyOnWrite(); - instance.visit(MergeFromVisitor.INSTANCE, message); + mergeFromInstance(instance, message); return (BuilderType) this; } + private void mergeFromInstance(MessageType dest, MessageType src) { + dest.visit(MergeFromVisitor.INSTANCE, src); + } + @Override public MessageType getDefaultInstanceForType() { return defaultInstance; @@ -1713,7 +1718,6 @@ public abstract class GeneratedMessageLite< Object visitOneofLong(boolean minePresent, Object mine, Object other); Object visitOneofString(boolean minePresent, Object mine, Object other); Object visitOneofByteString(boolean minePresent, Object mine, Object other); - Object visitOneofLazyMessage(boolean minePresent, Object mine, Object other); Object visitOneofMessage(boolean minePresent, Object mine, Object other); void visitOneofNotSet(boolean minePresent); @@ -1721,7 +1725,6 @@ public abstract class GeneratedMessageLite< * Message fields use null sentinals. */ <T extends MessageLite> T visitMessage(T mine, T other); - LazyFieldLite visitLazyMessage(LazyFieldLite mine, LazyFieldLite other); <T> ProtobufList<T> visitList(ProtobufList<T> mine, ProtobufList<T> other); BooleanList visitBooleanList(BooleanList mine, BooleanList other); @@ -1865,14 +1868,6 @@ public abstract class GeneratedMessageLite< } @Override - public Object visitOneofLazyMessage(boolean minePresent, Object mine, Object other) { - if (minePresent && mine.equals(other)) { - return mine; - } - throw NOT_EQUALS; - } - - @Override public Object visitOneofMessage(boolean minePresent, Object mine, Object other) { if (minePresent && ((GeneratedMessageLite<?, ?>) mine).equals(this, (MessageLite) other)) { return mine; @@ -1903,21 +1898,6 @@ public abstract class GeneratedMessageLite< } @Override - public LazyFieldLite visitLazyMessage( - LazyFieldLite mine, LazyFieldLite other) { - if (mine == null && other == null) { - return null; - } - if (mine == null || other == null) { - throw NOT_EQUALS; - } - if (mine.equals(other)) { - return mine; - } - throw NOT_EQUALS; - } - - @Override public <T> ProtobufList<T> visitList(ProtobufList<T> mine, ProtobufList<T> other) { if (!mine.equals(other)) { throw NOT_EQUALS; @@ -2094,12 +2074,6 @@ public abstract class GeneratedMessageLite< } @Override - public Object visitOneofLazyMessage(boolean minePresent, Object mine, Object other) { - hashCode = (53 * hashCode) + mine.hashCode(); - return mine; - } - - @Override public Object visitOneofMessage(boolean minePresent, Object mine, Object other) { return visitMessage((MessageLite) mine, (MessageLite) other); } @@ -2128,18 +2102,6 @@ public abstract class GeneratedMessageLite< } @Override - public LazyFieldLite visitLazyMessage(LazyFieldLite mine, LazyFieldLite other) { - final int protoHash; - if (mine != null) { - protoHash = mine.hashCode(); - } else { - protoHash = 37; - } - hashCode = (53 * hashCode) + protoHash; - return mine; - } - - @Override public <T> ProtobufList<T> visitList(ProtobufList<T> mine, ProtobufList<T> other) { hashCode = (53 * hashCode) + mine.hashCode(); return mine; @@ -2282,13 +2244,6 @@ public abstract class GeneratedMessageLite< } @Override - public Object visitOneofLazyMessage(boolean minePresent, Object mine, Object other) { - LazyFieldLite lazy = minePresent ? (LazyFieldLite) mine : new LazyFieldLite(); - lazy.merge((LazyFieldLite) other); - return lazy; - } - - @Override public Object visitOneofMessage(boolean minePresent, Object mine, Object other) { if (minePresent) { return visitMessage((MessageLite) mine, (MessageLite) other); @@ -2312,17 +2267,6 @@ public abstract class GeneratedMessageLite< } @Override - public LazyFieldLite visitLazyMessage(LazyFieldLite mine, LazyFieldLite other) { - if (other != null) { - if (mine == null) { - mine = new LazyFieldLite(); - } - mine.merge(other); - } - return mine; - } - - @Override public <T> ProtobufList<T> visitList(ProtobufList<T> mine, ProtobufList<T> other) { int size = mine.size(); int otherSize = other.size(); diff --git a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java index b949cd17..592869a1 100644 --- a/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java +++ b/java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import static com.google.protobuf.Internal.checkNotNull; + import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; @@ -47,7 +49,6 @@ import com.google.protobuf.Descriptors.OneofDescriptor; // to be able to use GeneratedMessage.GeneratedExtension. The GeneratedExtension definition in // this file is also excluded from opensource to avoid conflict. import com.google.protobuf.GeneratedMessage.GeneratedExtension; - import java.io.IOException; import java.io.InputStream; import java.io.ObjectStreamException; @@ -277,13 +278,30 @@ public abstract class GeneratedMessageV3 extends AbstractMessage /** * Called by subclasses to parse an unknown field. + * * @return {@code true} unless the tag is an end-group tag. */ protected boolean parseUnknownField( CodedInputStream input, UnknownFieldSet.Builder unknownFields, ExtensionRegistryLite extensionRegistry, - int tag) throws IOException { + int tag) + throws IOException { + if (input.shouldDiscardUnknownFields()) { + return input.skipField(tag); + } + return unknownFields.mergeFieldFrom(tag, input); + } + + protected boolean parseUnknownFieldProto3( + CodedInputStream input, + UnknownFieldSet.Builder unknownFields, + ExtensionRegistryLite extensionRegistry, + int tag) + throws IOException { + if (input.shouldDiscardUnknownFieldsProto3()) { + return input.skipField(tag); + } return unknownFields.mergeFieldFrom(tag, input); } @@ -619,15 +637,22 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return (BuilderType) this; } + protected BuilderType setUnknownFieldsProto3(final UnknownFieldSet unknownFields) { + if (CodedInputStream.getProto3DiscardUnknownFieldsDefault()) { + return (BuilderType) this; + } + this.unknownFields = unknownFields; + onChanged(); + return (BuilderType) this; + } + @Override public BuilderType mergeUnknownFields( final UnknownFieldSet unknownFields) { - this.unknownFields = + return setUnknownFields( UnknownFieldSet.newBuilder(this.unknownFields) .mergeFrom(unknownFields) - .build(); - onChanged(); - return (BuilderType) this; + .build()); } @Override @@ -666,18 +691,6 @@ public abstract class GeneratedMessageV3 extends AbstractMessage } /** - * Called by subclasses to parse an unknown field. - * @return {@code true} unless the tag is an end-group tag. - */ - protected boolean parseUnknownField( - final CodedInputStream input, - final UnknownFieldSet.Builder unknownFields, - final ExtensionRegistryLite extensionRegistry, - final int tag) throws IOException { - return unknownFields.mergeFieldFrom(tag, input); - } - - /** * Implementation of {@link BuilderParent} for giving to our children. This * small inner class makes it so we don't publicly expose the BuilderParent * methods. @@ -987,8 +1000,23 @@ public abstract class GeneratedMessageV3 extends AbstractMessage ExtensionRegistryLite extensionRegistry, int tag) throws IOException { return MessageReflection.mergeFieldFrom( - input, unknownFields, extensionRegistry, getDescriptorForType(), - new MessageReflection.ExtensionAdapter(extensions), tag); + input, input.shouldDiscardUnknownFields() ? null : unknownFields, extensionRegistry, + getDescriptorForType(), new MessageReflection.ExtensionAdapter(extensions), tag); + } + + @Override + protected boolean parseUnknownFieldProto3( + CodedInputStream input, + UnknownFieldSet.Builder unknownFields, + ExtensionRegistryLite extensionRegistry, + int tag) throws IOException { + return MessageReflection.mergeFieldFrom( + input, + input.shouldDiscardUnknownFieldsProto3() ? null : unknownFields, + extensionRegistry, + getDescriptorForType(), + new MessageReflection.ExtensionAdapter(extensions), + tag); } @@ -1458,21 +1486,6 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return super.isInitialized() && extensionsAreInitialized(); } - /** - * Called by subclasses to parse an unknown field or an extension. - * @return {@code true} unless the tag is an end-group tag. - */ - @Override - protected boolean parseUnknownField( - final CodedInputStream input, - final UnknownFieldSet.Builder unknownFields, - final ExtensionRegistryLite extensionRegistry, - final int tag) throws IOException { - return MessageReflection.mergeFieldFrom( - input, unknownFields, extensionRegistry, getDescriptorForType(), - new MessageReflection.BuilderAdapter(this), tag); - } - // --------------------------------------------------------------- // Reflection @@ -2277,7 +2290,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage public Object getRepeatedRaw(Builder builder, int index) { return getRepeated(builder, index); } - + @Override public void setRepeated(Builder builder, int index, Object value) { getMutableMapField(builder).getMutableList().set(index, coerceType((Message) value)); @@ -2678,7 +2691,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return (Extension<MessageType, T>) extension; } - + protected static int computeStringSize(final int fieldNumber, final Object value) { if (value instanceof String) { return CodedOutputStream.computeStringSize(fieldNumber, (String) value); @@ -2686,7 +2699,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return CodedOutputStream.computeBytesSize(fieldNumber, (ByteString) value); } } - + protected static int computeStringSizeNoTag(final Object value) { if (value instanceof String) { return CodedOutputStream.computeStringSizeNoTag((String) value); @@ -2694,7 +2707,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage return CodedOutputStream.computeBytesSizeNoTag((ByteString) value); } } - + protected static void writeString( CodedOutputStream output, final int fieldNumber, final Object value) throws IOException { if (value instanceof String) { @@ -2703,7 +2716,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage output.writeBytes(fieldNumber, (ByteString) value); } } - + protected static void writeStringNoTag( CodedOutputStream output, final Object value) throws IOException { if (value instanceof String) { diff --git a/java/core/src/main/java/com/google/protobuf/Internal.java b/java/core/src/main/java/com/google/protobuf/Internal.java index 36bdece3..848cad03 100644 --- a/java/core/src/main/java/com/google/protobuf/Internal.java +++ b/java/core/src/main/java/com/google/protobuf/Internal.java @@ -414,9 +414,8 @@ public final class Internal { } } - /** - * An empty byte array constant used in generated code. - */ + + /** An empty byte array constant used in generated code. */ public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; /** diff --git a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java index 55e33d21..510c6aac 100644 --- a/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java +++ b/java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java @@ -50,6 +50,10 @@ public class InvalidProtocolBufferException extends IOException { super(e.getMessage(), e); } + public InvalidProtocolBufferException(final String description, IOException e) { + super(description, e); + } + /** * Attaches an unfinished message to the exception to support best-effort * parsing in {@code Parser} interface. diff --git a/java/core/src/main/java/com/google/protobuf/MessageReflection.java b/java/core/src/main/java/com/google/protobuf/MessageReflection.java index 3d73efb3..69ad7ddf 100644 --- a/java/core/src/main/java/com/google/protobuf/MessageReflection.java +++ b/java/core/src/main/java/com/google/protobuf/MessageReflection.java @@ -31,7 +31,6 @@ package com.google.protobuf; import com.google.protobuf.Descriptors.FieldDescriptor; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -714,12 +713,14 @@ class MessageReflection { } /** - * Parses a single field into MergeTarget. The target can be Message.Builder, - * FieldSet or MutableMessage. + * Parses a single field into MergeTarget. The target can be Message.Builder, FieldSet or + * MutableMessage. * - * Package-private because it is used by GeneratedMessage.ExtendableMessage. + * <p>Package-private because it is used by GeneratedMessage.ExtendableMessage. * * @param tag The tag, which should have already been read. + * @param unknownFields If not null, unknown fields will be merged to this {@link + * UnknownFieldSet}, otherwise unknown fields will be discarded. * @return {@code true} unless the tag is an end-group tag. */ static boolean mergeFieldFrom( @@ -728,7 +729,8 @@ class MessageReflection { ExtensionRegistryLite extensionRegistry, Descriptors.Descriptor type, MergeTarget target, - int tag) throws IOException { + int tag) + throws IOException { if (type.getOptions().getMessageSetWireFormat() && tag == WireFormat.MESSAGE_SET_ITEM_TAG) { mergeMessageSetExtensionFromCodedStream( @@ -792,7 +794,11 @@ class MessageReflection { } if (unknown) { // Unknown field or wrong wire type. Skip. - return unknownFields.mergeFieldFrom(tag, input); + if (unknownFields != null) { + return unknownFields.mergeFieldFrom(tag, input); + } else { + return input.skipField(tag); + } } if (packed) { @@ -844,7 +850,9 @@ class MessageReflection { // If the number isn't recognized as a valid value for this enum, // drop it. if (value == null) { - unknownFields.mergeVarintField(fieldNumber, rawValue); + if (unknownFields != null) { + unknownFields.mergeVarintField(fieldNumber, rawValue); + } return true; } } @@ -947,7 +955,7 @@ class MessageReflection { mergeMessageSetExtensionFromBytes( rawBytes, extension, extensionRegistry, target); } else { // We don't know how to parse this. Ignore it. - if (rawBytes != null) { + if (rawBytes != null && unknownFields != null) { unknownFields.mergeField(typeId, UnknownFieldSet.Field.newBuilder() .addLengthDelimited(rawBytes).build()); } diff --git a/java/core/src/main/java/com/google/protobuf/TextFormat.java b/java/core/src/main/java/com/google/protobuf/TextFormat.java index 53dead80..64094d09 100644 --- a/java/core/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/core/src/main/java/com/google/protobuf/TextFormat.java @@ -34,7 +34,6 @@ import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.EnumDescriptor; import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; - import java.io.IOException; import java.math.BigInteger; import java.nio.CharBuffer; @@ -56,14 +55,7 @@ import java.util.regex.Pattern; public final class TextFormat { private TextFormat() {} - private static final Logger logger = - Logger.getLogger(TextFormat.class.getName()); - - private static final Printer DEFAULT_PRINTER = new Printer(); - private static final Printer SINGLE_LINE_PRINTER = - (new Printer()).setSingleLineMode(true); - private static final Printer UNICODE_PRINTER = - (new Printer()).setEscapeNonAscii(false); + private static final Logger logger = Logger.getLogger(TextFormat.class.getName()); /** * Outputs a textual representation of the Protocol Message supplied into @@ -73,14 +65,14 @@ public final class TextFormat { public static void print( final MessageOrBuilder message, final Appendable output) throws IOException { - DEFAULT_PRINTER.print(message, new TextGenerator(output)); + Printer.DEFAULT.print(message, multiLineOutput(output)); } /** Outputs a textual representation of {@code fields} to {@code output}. */ public static void print(final UnknownFieldSet fields, final Appendable output) throws IOException { - DEFAULT_PRINTER.printUnknownFields(fields, new TextGenerator(output)); + Printer.DEFAULT.printUnknownFields(fields, multiLineOutput(output)); } /** @@ -90,7 +82,7 @@ public final class TextFormat { public static void printUnicode( final MessageOrBuilder message, final Appendable output) throws IOException { - UNICODE_PRINTER.print(message, new TextGenerator(output)); + Printer.UNICODE.print(message, multiLineOutput(output)); } /** @@ -100,7 +92,7 @@ public final class TextFormat { public static void printUnicode(final UnknownFieldSet fields, final Appendable output) throws IOException { - UNICODE_PRINTER.printUnknownFields(fields, new TextGenerator(output)); + Printer.UNICODE.printUnknownFields(fields, multiLineOutput(output)); } /** @@ -109,10 +101,9 @@ public final class TextFormat { */ public static String shortDebugString(final MessageOrBuilder message) { try { - final StringBuilder sb = new StringBuilder(); - SINGLE_LINE_PRINTER.print(message, new TextGenerator(sb)); - // Single line mode currently might have an extra space at the end. - return sb.toString().trim(); + final StringBuilder text = new StringBuilder(); + Printer.DEFAULT.print(message, singleLineOutput(text)); + return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); } @@ -125,11 +116,11 @@ public final class TextFormat { public static String shortDebugString(final FieldDescriptor field, final Object value) { try { - final StringBuilder sb = new StringBuilder(); - SINGLE_LINE_PRINTER.printField(field, value, new TextGenerator(sb)); - return sb.toString().trim(); + final StringBuilder text = new StringBuilder(); + Printer.DEFAULT.printField(field, value, singleLineOutput(text)); + return text.toString(); } catch (IOException e) { - throw new IllegalStateException(e); + throw new IllegalStateException(e); } } @@ -139,10 +130,9 @@ public final class TextFormat { */ public static String shortDebugString(final UnknownFieldSet fields) { try { - final StringBuilder sb = new StringBuilder(); - SINGLE_LINE_PRINTER.printUnknownFields(fields, new TextGenerator(sb)); - // Single line mode currently might have an extra space at the end. - return sb.toString().trim(); + final StringBuilder text = new StringBuilder(); + Printer.DEFAULT.printUnknownFields(fields, singleLineOutput(text)); + return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); } @@ -183,7 +173,7 @@ public final class TextFormat { public static String printToUnicodeString(final MessageOrBuilder message) { try { final StringBuilder text = new StringBuilder(); - UNICODE_PRINTER.print(message, new TextGenerator(text)); + Printer.UNICODE.print(message, multiLineOutput(text)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -197,7 +187,7 @@ public final class TextFormat { public static String printToUnicodeString(final UnknownFieldSet fields) { try { final StringBuilder text = new StringBuilder(); - UNICODE_PRINTER.printUnknownFields(fields, new TextGenerator(text)); + Printer.UNICODE.printUnknownFields(fields, multiLineOutput(text)); return text.toString(); } catch (IOException e) { throw new IllegalStateException(e); @@ -208,7 +198,7 @@ public final class TextFormat { final Object value, final Appendable output) throws IOException { - DEFAULT_PRINTER.printField(field, value, new TextGenerator(output)); + Printer.DEFAULT.printField(field, value, multiLineOutput(output)); } public static String printFieldToString(final FieldDescriptor field, @@ -223,6 +213,23 @@ public final class TextFormat { } /** + * Outputs a unicode textual representation of the value of given field value. + * + * <p>Same as {@code printFieldValue()}, except that non-ASCII characters in string type fields + * are not escaped in backslash+octals. + * + * @param field the descriptor of the field + * @param value the value of the field + * @param output the output to which to append the formatted value + * @throws ClassCastException if the value is not appropriate for the given field descriptor + * @throws IOException if there is an exception writing to the output + */ + public static void printUnicodeFieldValue( + final FieldDescriptor field, final Object value, final Appendable output) throws IOException { + Printer.UNICODE.printFieldValue(field, value, multiLineOutput(output)); + } + + /** * Outputs a textual representation of the value of given field value. * * @param field the descriptor of the field @@ -236,7 +243,7 @@ public final class TextFormat { final Object value, final Appendable output) throws IOException { - DEFAULT_PRINTER.printFieldValue(field, value, new TextGenerator(output)); + Printer.DEFAULT.printFieldValue(field, value, multiLineOutput(output)); } /** @@ -253,7 +260,7 @@ public final class TextFormat { final Object value, final Appendable output) throws IOException { - printUnknownFieldValue(tag, value, new TextGenerator(output)); + printUnknownFieldValue(tag, value, multiLineOutput(output)); } private static void printUnknownFieldValue(final int tag, @@ -277,7 +284,7 @@ public final class TextFormat { generator.print("\""); break; case WireFormat.WIRETYPE_START_GROUP: - DEFAULT_PRINTER.printUnknownFields((UnknownFieldSet) value, generator); + Printer.DEFAULT.printUnknownFields((UnknownFieldSet) value, generator); break; default: throw new IllegalArgumentException("Bad tag: " + tag); @@ -286,24 +293,16 @@ public final class TextFormat { /** Helper class for converting protobufs to text. */ private static final class Printer { - /** Whether to omit newlines from the output. */ - boolean singleLineMode = false; + // Printer instance which escapes non-ASCII characters. + static final Printer DEFAULT = new Printer(true); + // Printer instance which emits Unicode (it still escapes newlines and quotes in strings). + static final Printer UNICODE = new Printer(false); /** Whether to escape non ASCII characters with backslash and octal. */ - boolean escapeNonAscii = true; - - private Printer() {} + private final boolean escapeNonAscii; - /** Setter of singleLineMode */ - private Printer setSingleLineMode(boolean singleLineMode) { - this.singleLineMode = singleLineMode; - return this; - } - - /** Setter of escapeNonAscii */ - private Printer setEscapeNonAscii(boolean escapeNonAscii) { + private Printer(boolean escapeNonAscii) { this.escapeNonAscii = escapeNonAscii; - return this; } private void print( @@ -355,12 +354,9 @@ public final class TextFormat { } if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - if (singleLineMode) { - generator.print(" { "); - } else { - generator.print(" {\n"); - generator.indent(); - } + generator.print(" {"); + generator.eol(); + generator.indent(); } else { generator.print(": "); } @@ -368,19 +364,10 @@ public final class TextFormat { printFieldValue(field, value, generator); if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) { - if (singleLineMode) { - generator.print("} "); - } else { - generator.outdent(); - generator.print("}\n"); - } - } else { - if (singleLineMode) { - generator.print(" "); - } else { - generator.print("\n"); - } + generator.outdent(); + generator.print("}"); } + generator.eol(); } private void printFieldValue(final FieldDescriptor field, @@ -469,19 +456,13 @@ public final class TextFormat { field.getLengthDelimitedList(), generator); for (final UnknownFieldSet value : field.getGroupList()) { generator.print(entry.getKey().toString()); - if (singleLineMode) { - generator.print(" { "); - } else { - generator.print(" {\n"); - generator.indent(); - } + generator.print(" {"); + generator.eol(); + generator.indent(); printUnknownFields(value, generator); - if (singleLineMode) { - generator.print("} "); - } else { - generator.outdent(); - generator.print("}\n"); - } + generator.outdent(); + generator.print("}"); + generator.eol(); } } } @@ -495,7 +476,7 @@ public final class TextFormat { generator.print(String.valueOf(number)); generator.print(": "); printUnknownFieldValue(wireType, value, generator); - generator.print(singleLineMode ? " " : "\n"); + generator.eol(); } } } @@ -521,16 +502,29 @@ public final class TextFormat { } } - /** + private static TextGenerator multiLineOutput(Appendable output) { + return new TextGenerator(output, false); + } + + private static TextGenerator singleLineOutput(Appendable output) { + return new TextGenerator(output, true); + } + + /** * An inner class for writing text to the output stream. */ private static final class TextGenerator { private final Appendable output; private final StringBuilder indent = new StringBuilder(); - private boolean atStartOfLine = true; + private final boolean singleLineMode; + // While technically we are "at the start of a line" at the very beginning of the output, all + // we would do in response to this is emit the (zero length) indentation, so it has no effect. + // Setting it false here does however suppress an unwanted leading space in single-line mode. + private boolean atStartOfLine = false; - private TextGenerator(final Appendable output) { + private TextGenerator(final Appendable output, boolean singleLineMode) { this.output = output; + this.singleLineMode = singleLineMode; } /** @@ -552,35 +546,31 @@ public final class TextFormat { throw new IllegalArgumentException( " Outdent() without matching Indent()."); } - indent.delete(length - 2, length); + indent.setLength(length - 2); } /** - * Print text to the output stream. + * Print text to the output stream. Bare newlines are never expected to be passed to this + * method; to indicate the end of a line, call "eol()". */ public void print(final CharSequence text) throws IOException { - final int size = text.length(); - int pos = 0; - - for (int i = 0; i < size; i++) { - if (text.charAt(i) == '\n') { - write(text.subSequence(pos, i + 1)); - pos = i + 1; - atStartOfLine = true; - } + if (atStartOfLine) { + atStartOfLine = false; + output.append(singleLineMode ? " " : indent); } - write(text.subSequence(pos, size)); + output.append(text); } - private void write(final CharSequence data) throws IOException { - if (data.length() == 0) { - return; - } - if (atStartOfLine) { - atStartOfLine = false; - output.append(indent); + /** + * Signifies reaching the "end of the current line" in the output. In single-line mode, this + * does not result in a newline being emitted, but ensures that a separating space is written + * before the next output. + */ + public void eol() throws IOException { + if (!singleLineMode) { + output.append("\n"); } - output.append(data); + atStartOfLine = true; } } @@ -1469,9 +1459,15 @@ public final class TextFormat { extensionRegistry, name.toString()); if (extension == null) { - unknownFields.add((tokenizer.getPreviousLine() + 1) + ":" + - (tokenizer.getPreviousColumn() + 1) + ":\t" + - type.getFullName() + ".[" + name + "]"); + unknownFields.add( + (tokenizer.getPreviousLine() + 1) + + ":" + + (tokenizer.getPreviousColumn() + 1) + + ":\t" + + type.getFullName() + + ".[" + + name + + "]"); } else { if (extension.descriptor.getContainingType() != type) { throw tokenizer.parseExceptionPreviousToken( @@ -1506,9 +1502,14 @@ public final class TextFormat { } if (field == null) { - unknownFields.add((tokenizer.getPreviousLine() + 1) + ":" + - (tokenizer.getPreviousColumn() + 1) + ":\t" + - type.getFullName() + "." + name); + unknownFields.add( + (tokenizer.getPreviousLine() + 1) + + ":" + + (tokenizer.getPreviousColumn() + 1) + + ":\t" + + type.getFullName() + + "." + + name); } } diff --git a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java index d9381135..37d64633 100644 --- a/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java +++ b/java/core/src/main/java/com/google/protobuf/UnknownFieldSet.java @@ -715,7 +715,7 @@ public final class UnknownFieldSet implements MessageLite { * @see UnknownFieldSet */ public static final class Field { - Field() {} + private Field() {} /** Construct a new {@link Builder}. */ public static Builder newBuilder() { diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java index ca80d946..acc03a7c 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java @@ -47,8 +47,29 @@ final class UnsafeUtil { private static final boolean HAS_UNSAFE_BYTEBUFFER_OPERATIONS = supportsUnsafeByteBufferOperations(); private static final boolean HAS_UNSAFE_ARRAY_OPERATIONS = supportsUnsafeArrayOperations(); - private static final boolean HAS_UNSAFE_COPY_MEMORY = supportsUnsafeCopyMemory(); - private static final long ARRAY_BASE_OFFSET = byteArrayBaseOffset(); + + private static final long BYTE_ARRAY_BASE_OFFSET = arrayBaseOffset(byte[].class); + // Micro-optimization: we can assume a scale of 1 and skip the multiply + // private static final long BYTE_ARRAY_INDEX_SCALE = 1; + + private static final long BOOLEAN_ARRAY_BASE_OFFSET = arrayBaseOffset(boolean[].class); + private static final long BOOLEAN_ARRAY_INDEX_SCALE = arrayIndexScale(boolean[].class); + + private static final long INT_ARRAY_BASE_OFFSET = arrayBaseOffset(int[].class); + private static final long INT_ARRAY_INDEX_SCALE = arrayIndexScale(int[].class); + + private static final long LONG_ARRAY_BASE_OFFSET = arrayBaseOffset(long[].class); + private static final long LONG_ARRAY_INDEX_SCALE = arrayIndexScale(long[].class); + + private static final long FLOAT_ARRAY_BASE_OFFSET = arrayBaseOffset(float[].class); + private static final long FLOAT_ARRAY_INDEX_SCALE = arrayIndexScale(float[].class); + + private static final long DOUBLE_ARRAY_BASE_OFFSET = arrayBaseOffset(double[].class); + private static final long DOUBLE_ARRAY_INDEX_SCALE = arrayIndexScale(double[].class); + + private static final long OBJECT_ARRAY_BASE_OFFSET = arrayBaseOffset(Object[].class); + private static final long OBJECT_ARRAY_INDEX_SCALE = arrayIndexScale(Object[].class); + private static final long BUFFER_ADDRESS_OFFSET = fieldOffset(bufferAddressField()); private UnsafeUtil() {} @@ -57,10 +78,6 @@ final class UnsafeUtil { return HAS_UNSAFE_ARRAY_OPERATIONS; } - static boolean hasUnsafeCopyMemory() { - return HAS_UNSAFE_COPY_MEMORY; - } - static boolean hasUnsafeByteBufferOperations() { return HAS_UNSAFE_BYTEBUFFER_OPERATIONS; } @@ -69,8 +86,12 @@ final class UnsafeUtil { return MEMORY_ACCESSOR.objectFieldOffset(field); } - static long getArrayBaseOffset() { - return ARRAY_BASE_OFFSET; + private static int arrayBaseOffset(Class<?> clazz) { + return HAS_UNSAFE_ARRAY_OPERATIONS ? MEMORY_ACCESSOR.arrayBaseOffset(clazz) : -1; + } + + private static int arrayIndexScale(Class<?> clazz) { + return HAS_UNSAFE_ARRAY_OPERATIONS ? MEMORY_ACCESSOR.arrayIndexScale(clazz) : -1; } static byte getByte(Object target, long offset) { @@ -129,9 +150,82 @@ final class UnsafeUtil { MEMORY_ACCESSOR.putObject(target, offset, value); } - static void copyMemory( - Object src, long srcOffset, Object target, long targetOffset, long length) { - MEMORY_ACCESSOR.copyMemory(src, srcOffset, target, targetOffset, length); + static byte getByte(byte[] target, long index) { + return MEMORY_ACCESSOR.getByte(target, BYTE_ARRAY_BASE_OFFSET + index); + } + + static void putByte(byte[] target, long index, byte value) { + MEMORY_ACCESSOR.putByte(target, BYTE_ARRAY_BASE_OFFSET + index, value); + } + + static int getInt(int[] target, long index) { + return MEMORY_ACCESSOR.getInt(target, INT_ARRAY_BASE_OFFSET + (index * INT_ARRAY_INDEX_SCALE)); + } + + static void putInt(int[] target, long index, int value) { + MEMORY_ACCESSOR.putInt(target, INT_ARRAY_BASE_OFFSET + (index * INT_ARRAY_INDEX_SCALE), value); + } + + static long getLong(long[] target, long index) { + return MEMORY_ACCESSOR.getLong( + target, LONG_ARRAY_BASE_OFFSET + (index * LONG_ARRAY_INDEX_SCALE)); + } + + static void putLong(long[] target, long index, long value) { + MEMORY_ACCESSOR.putLong( + target, LONG_ARRAY_BASE_OFFSET + (index * LONG_ARRAY_INDEX_SCALE), value); + } + + static boolean getBoolean(boolean[] target, long index) { + return MEMORY_ACCESSOR.getBoolean( + target, BOOLEAN_ARRAY_BASE_OFFSET + (index * BOOLEAN_ARRAY_INDEX_SCALE)); + } + + static void putBoolean(boolean[] target, long index, boolean value) { + MEMORY_ACCESSOR.putBoolean( + target, BOOLEAN_ARRAY_BASE_OFFSET + (index * BOOLEAN_ARRAY_INDEX_SCALE), value); + } + + static float getFloat(float[] target, long index) { + return MEMORY_ACCESSOR.getFloat( + target, FLOAT_ARRAY_BASE_OFFSET + (index * FLOAT_ARRAY_INDEX_SCALE)); + } + + static void putFloat(float[] target, long index, float value) { + MEMORY_ACCESSOR.putFloat( + target, FLOAT_ARRAY_BASE_OFFSET + (index * FLOAT_ARRAY_INDEX_SCALE), value); + } + + static double getDouble(double[] target, long index) { + return MEMORY_ACCESSOR.getDouble( + target, DOUBLE_ARRAY_BASE_OFFSET + (index * DOUBLE_ARRAY_INDEX_SCALE)); + } + + static void putDouble(double[] target, long index, double value) { + MEMORY_ACCESSOR.putDouble( + target, DOUBLE_ARRAY_BASE_OFFSET + (index * DOUBLE_ARRAY_INDEX_SCALE), value); + } + + static Object getObject(Object[] target, long index) { + return MEMORY_ACCESSOR.getObject( + target, OBJECT_ARRAY_BASE_OFFSET + (index * OBJECT_ARRAY_INDEX_SCALE)); + } + + static void putObject(Object[] target, long index, Object value) { + MEMORY_ACCESSOR.putObject( + target, OBJECT_ARRAY_BASE_OFFSET + (index * OBJECT_ARRAY_INDEX_SCALE), value); + } + + static void copyMemory(byte[] src, long srcIndex, long targetOffset, long length) { + MEMORY_ACCESSOR.copyMemory(src, srcIndex, targetOffset, length); + } + + static void copyMemory(long srcOffset, byte[] target, long targetIndex, long length) { + MEMORY_ACCESSOR.copyMemory(srcOffset, target, targetIndex, length); + } + + static void copyMemory(byte[] src, long srcIndex, byte[] target, long targetIndex, long length) { + System.arraycopy(src, (int) srcIndex, target, (int) targetIndex, (int) length); } static byte getByte(long address) { @@ -221,6 +315,7 @@ final class UnsafeUtil { Class<?> clazz = UNSAFE.getClass(); clazz.getMethod("objectFieldOffset", Field.class); clazz.getMethod("arrayBaseOffset", Class.class); + clazz.getMethod("arrayIndexScale", Class.class); clazz.getMethod("getInt", Object.class, long.class); clazz.getMethod("putInt", Object.class, long.class, int.class); clazz.getMethod("getLong", Object.class, long.class); @@ -245,27 +340,6 @@ final class UnsafeUtil { return false; } - /** - * Indicates whether or not unsafe copyMemory(object, long, object, long, long) operations are - * supported on this platform. - */ - private static boolean supportsUnsafeCopyMemory() { - if (UNSAFE == null) { - return false; - } - try { - Class<?> clazz = UNSAFE.getClass(); - clazz.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class); - - return true; - } catch (Throwable e) { - logger.log( - Level.WARNING, - "copyMemory is missing from platform - proto runtime falling back to safer methods."); - } - return false; - } - private static boolean supportsUnsafeByteBufferOperations() { if (UNSAFE == null) { return false; @@ -283,6 +357,7 @@ final class UnsafeUtil { clazz.getMethod("getLong", long.class); clazz.getMethod("putLong", long.class, long.class); clazz.getMethod("copyMemory", long.class, long.class, long.class); + clazz.getMethod("copyMemory", Object.class, long.class, Object.class, long.class, long.class); return true; } catch (Throwable e) { logger.log( @@ -308,13 +383,6 @@ final class UnsafeUtil { } /** - * Get the base offset for byte arrays, or {@code -1} if {@code sun.misc.Unsafe} is not available. - */ - private static int byteArrayBaseOffset() { - return HAS_UNSAFE_ARRAY_OPERATIONS ? MEMORY_ACCESSOR.arrayBaseOffset(byte[].class) : -1; - } - - /** * Returns the offset of the provided field, or {@code -1} if {@code sun.misc.Unsafe} is not * available. */ @@ -394,6 +462,10 @@ final class UnsafeUtil { return unsafe.arrayBaseOffset(clazz); } + public final int arrayIndexScale(Class<?> clazz) { + return unsafe.arrayIndexScale(clazz); + } + public abstract byte getByte(long address); public abstract void putByte(long address, byte value); @@ -408,10 +480,11 @@ final class UnsafeUtil { public abstract void copyMemory(long srcAddress, long targetAddress, long length); - public abstract void copyMemory( - Object src, long srcOffset, Object target, long targetOffset, long length); - public abstract Object getStaticObject(Field field); + + public abstract void copyMemory(long srcOffset, byte[] target, long targetIndex, long length); + + public abstract void copyMemory(byte[] src, long srcIndex, long targetOffset, long length); } private static final class JvmMemoryAccessor extends MemoryAccessor { @@ -491,15 +564,19 @@ final class UnsafeUtil { } @Override - public void copyMemory( - Object src, long srcOffset, Object target, long targetOffset, long length) { - unsafe.copyMemory(src, srcOffset, target, targetOffset, length); - } - - @Override public void copyMemory(long srcAddress, long targetAddress, long length) { unsafe.copyMemory(srcAddress, targetAddress, length); } + + @Override + public void copyMemory(long srcOffset, byte[] target, long targetIndex, long length) { + unsafe.copyMemory(null, srcOffset, target, BYTE_ARRAY_BASE_OFFSET + targetIndex, length); + } + + @Override + public void copyMemory(byte[] src, long srcIndex, long targetOffset, long length) { + unsafe.copyMemory(src, BYTE_ARRAY_BASE_OFFSET + srcIndex, null, targetOffset, length); + } @Override public Object getStaticObject(Field field) { diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index d98e914d..1b136144 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -31,7 +31,6 @@ package com.google.protobuf; import static com.google.protobuf.UnsafeUtil.addressOffset; -import static com.google.protobuf.UnsafeUtil.getArrayBaseOffset; import static com.google.protobuf.UnsafeUtil.hasUnsafeArrayOperations; import static com.google.protobuf.UnsafeUtil.hasUnsafeByteBufferOperations; import static java.lang.Character.MAX_SURROGATE; @@ -1001,8 +1000,8 @@ final class Utf8 { throw new ArrayIndexOutOfBoundsException( String.format("Array length=%d, index=%d, limit=%d", bytes.length, index, limit)); } - long offset = getArrayBaseOffset() + index; - final long offsetLimit = getArrayBaseOffset() + limit; + long offset = index; + final long offsetLimit = limit; if (state != COMPLETE) { // The previous decoding operation was incomplete (or malformed). // We look for a well-formed sequence consisting of bytes from @@ -1187,7 +1186,7 @@ final class Utf8 { @Override int encodeUtf8(final CharSequence in, final byte[] out, final int offset, final int length) { - long outIx = getArrayBaseOffset() + offset; + long outIx = offset; final long outLimit = outIx + length; final int inLimit = in.length(); if (inLimit > length || out.length - length < offset) { @@ -1204,7 +1203,7 @@ final class Utf8 { } if (inIx == inLimit) { // We're done, it was ASCII encoded. - return (int) (outIx - getArrayBaseOffset()); + return (int) outIx; } for (char c; inIx < inLimit; ++inIx) { @@ -1243,7 +1242,7 @@ final class Utf8 { } // All bytes have been encoded. - return (int) (outIx - getArrayBaseOffset()); + return (int) outIx; } @Override @@ -1321,31 +1320,17 @@ final class Utf8 { */ private static int unsafeEstimateConsecutiveAscii( byte[] bytes, long offset, final int maxChars) { - int remaining = maxChars; - if (remaining < UNSAFE_COUNT_ASCII_THRESHOLD) { + if (maxChars < UNSAFE_COUNT_ASCII_THRESHOLD) { // Don't bother with small strings. return 0; } - // Read bytes until 8-byte aligned so that we can read longs in the loop below. - // Byte arrays are already either 8 or 16-byte aligned, so we just need to make sure that - // the index (relative to the start of the array) is also 8-byte aligned. We do this by - // ANDing the index with 7 to determine the number of bytes that need to be read before - // we're 8-byte aligned. - final int unaligned = 8 - ((int) offset & 7); - for (int j = unaligned; j > 0; j--) { + for (int i = 0; i < maxChars; i++) { if (UnsafeUtil.getByte(bytes, offset++) < 0) { - return unaligned - j; + return i; } } - - // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII). - // To speed things up further, we're reading longs instead of bytes so we use a mask to - // determine if any byte in the current long is non-ASCII. - remaining -= unaligned; - for (; remaining >= 8 && (UnsafeUtil.getLong(bytes, offset) & ASCII_MASK_LONG) == 0; - offset += 8, remaining -= 8) {} - return maxChars - remaining; + return maxChars; } /** diff --git a/java/core/src/main/java/com/google/protobuf/WireFormat.java b/java/core/src/main/java/com/google/protobuf/WireFormat.java index 0b0cdb7d..8b837ee5 100644 --- a/java/core/src/main/java/com/google/protobuf/WireFormat.java +++ b/java/core/src/main/java/com/google/protobuf/WireFormat.java @@ -47,8 +47,10 @@ public final class WireFormat { // Do not allow instantiation. private WireFormat() {} - static final int FIXED_32_SIZE = 4; - static final int FIXED_64_SIZE = 8; + static final int FIXED32_SIZE = 4; + static final int FIXED64_SIZE = 8; + static final int MAX_VARINT32_SIZE = 5; + static final int MAX_VARINT64_SIZE = 10; static final int MAX_VARINT_SIZE = 10; public static final int WIRETYPE_VARINT = 0; diff --git a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java index e440c7db..da2c067e 100644 --- a/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java +++ b/java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java @@ -41,6 +41,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import junit.framework.TestCase; /** @@ -613,6 +614,82 @@ public class CodedInputStreamTest extends TestCase { checkSizeLimitExceeded(expected); } } + + public void testRefillBufferWithCorrectSize() throws Exception { + // NOTE: refillBuffer only applies to the stream-backed CIS. + byte[] bytes = "123456789".getBytes("UTF-8"); + ByteArrayOutputStream rawOutput = new ByteArrayOutputStream(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput, bytes.length); + + int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(tag); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.writeRawVarint32(tag); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.writeRawByte(4); + output.flush(); + + // Input is two string with length 9 and one raw byte. + byte[] rawInput = rawOutput.toByteArray(); + for (int inputStreamBufferLength = 8; + inputStreamBufferLength <= rawInput.length + 1; inputStreamBufferLength++) { + CodedInputStream input = CodedInputStream.newInstance( + new ByteArrayInputStream(rawInput), inputStreamBufferLength); + input.setSizeLimit(rawInput.length - 1); + input.readString(); + input.readString(); + try { + input.readRawByte(); // Hits limit. + fail("Should have thrown an exception!"); + } catch (InvalidProtocolBufferException expected) { + checkSizeLimitExceeded(expected); + } + } + } + + public void testIsAtEnd() throws Exception { + CodedInputStream input = CodedInputStream.newInstance( + new ByteArrayInputStream(new byte[5])); + try { + for (int i = 0; i < 5; i++) { + assertEquals(false, input.isAtEnd()); + input.readRawByte(); + } + assertEquals(true, input.isAtEnd()); + } catch (Exception e) { + fail("Catch exception in the testIsAtEnd"); + } + } + + public void testCurrentLimitExceeded() throws Exception { + byte[] bytes = "123456789".getBytes("UTF-8"); + ByteArrayOutputStream rawOutput = new ByteArrayOutputStream(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput, bytes.length); + + int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); + output.writeRawVarint32(tag); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.flush(); + + byte[] rawInput = rawOutput.toByteArray(); + CodedInputStream input = CodedInputStream.newInstance( + new ByteArrayInputStream(rawInput)); + // The length of the whole rawInput + input.setSizeLimit(11); + // Some number that is smaller than the rawInput's length + // but larger than 2 + input.pushLimit(5); + try { + input.readString(); + fail("Should have thrown an exception"); + } catch (InvalidProtocolBufferException expected) { + assertEquals(expected.getMessage(), + InvalidProtocolBufferException.truncatedMessage().getMessage()); + } + } public void testSizeLimitMultipleMessages() throws Exception { // NOTE: Size limit only applies to the stream-backed CIS. @@ -807,6 +884,52 @@ public class CodedInputStreamTest extends TestCase { } } + public void testReadLargeByteStringFromInputStream() throws Exception { + byte[] bytes = new byte[1024 * 1024]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) (i & 0xFF); + } + ByteString.Output rawOutput = ByteString.newOutput(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.flush(); + byte[] data = rawOutput.toByteString().toByteArray(); + + CodedInputStream input = CodedInputStream.newInstance( + new ByteArrayInputStream(data) { + @Override + public synchronized int available() { + return 0; + } + }); + ByteString result = input.readBytes(); + assertEquals(ByteString.copyFrom(bytes), result); + } + + public void testReadLargeByteArrayFromInputStream() throws Exception { + byte[] bytes = new byte[1024 * 1024]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) (i & 0xFF); + } + ByteString.Output rawOutput = ByteString.newOutput(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); + output.writeRawVarint32(bytes.length); + output.writeRawBytes(bytes); + output.flush(); + byte[] data = rawOutput.toByteString().toByteArray(); + + CodedInputStream input = CodedInputStream.newInstance( + new ByteArrayInputStream(data) { + @Override + public synchronized int available() { + return 0; + } + }); + byte[] result = input.readByteArray(); + assertTrue(Arrays.equals(bytes, result)); + } + public void testReadByteBuffer() throws Exception { ByteString.Output rawOutput = ByteString.newOutput(); CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); diff --git a/java/core/src/test/java/com/google/protobuf/DiscardUnknownFieldsTest.java b/java/core/src/test/java/com/google/protobuf/DiscardUnknownFieldsTest.java new file mode 100644 index 00000000..0f09a51b --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/DiscardUnknownFieldsTest.java @@ -0,0 +1,157 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import static org.junit.Assert.assertEquals; + +import protobuf_unittest.UnittestProto; +import proto3_unittest.UnittestProto3; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for discard or preserve unknown fields. */ +@RunWith(JUnit4.class) +public class DiscardUnknownFieldsTest { + @Test + public void testProto2() throws Exception { + testProto2Message( + UnittestProto.TestEmptyMessage.getDefaultInstance()); + testProto2Message( + UnittestProto.TestEmptyMessageWithExtensions.getDefaultInstance()); + testProto2Message( + DynamicMessage.getDefaultInstance(UnittestProto.TestEmptyMessage.getDescriptor())); + testProto2Message( + DynamicMessage.getDefaultInstance( + UnittestProto.TestEmptyMessageWithExtensions.getDescriptor())); + } + + @Test + public void testProto3() throws Exception { + testProto3Message(UnittestProto3.TestEmptyMessage.getDefaultInstance()); + testProto3Message( + DynamicMessage.getDefaultInstance(UnittestProto3.TestEmptyMessage.getDescriptor())); + } + + private static void testProto2Message(Message message) throws Exception { + assertUnknownFieldsDefaultPreserved(message); + assertUnknownFieldsExplicitlyDiscarded(message); + assertReuseCodedInputStreamPreserve(message); + assertUnknownFieldsInUnknownFieldSetArePreserve(message); + } + + private static void testProto3Message(Message message) throws Exception { + CodedInputStream.setProto3KeepUnknownsByDefaultForTest(); + assertUnknownFieldsDefaultPreserved(message); + assertUnknownFieldsExplicitlyDiscarded(message); + assertReuseCodedInputStreamPreserve(message); + assertUnknownFieldsInUnknownFieldSetArePreserve(message); + CodedInputStream.setProto3DiscardUnknownsByDefaultForTest(); + assertUnknownFieldsDefaultDiscarded(message); + assertUnknownFieldsExplicitlyDiscarded(message); + assertUnknownFieldsInUnknownFieldSetAreDiscarded(message); + } + + private static void assertReuseCodedInputStreamPreserve(Message message) throws Exception { + final int messageSize = payload.size(); + byte[] copied = new byte[messageSize * 2]; + payload.copyTo(copied, 0); + payload.copyTo(copied, messageSize); + CodedInputStream input = CodedInputStream.newInstance(copied); + { + // Use DiscardUnknownFieldsParser to parse the first payload. + int oldLimit = input.pushLimit(messageSize); + Message parsed = DiscardUnknownFieldsParser.wrap(message.getParserForType()).parseFrom(input); + assertEquals(message.getClass().getName(), 0, parsed.getSerializedSize()); + input.popLimit(oldLimit); + } + { + // Use the normal parser to parse the remaining payload should have unknown fields preserved. + Message parsed = message.getParserForType().parseFrom(input); + assertEquals(message.getClass().getName(), payload, parsed.toByteString()); + } + } + + /** + * {@link Message.Builder#setUnknownFields(UnknownFieldSet)} and {@link + * Message.Builder#mergeUnknownFields(UnknownFieldSet)} should preserve the unknown fields. + */ + private static void assertUnknownFieldsInUnknownFieldSetArePreserve(Message message) + throws Exception { + UnknownFieldSet unknownFields = UnknownFieldSet.newBuilder().mergeFrom(payload).build(); + Message built = message.newBuilderForType().setUnknownFields(unknownFields).build(); + assertEquals(message.getClass().getName(), payload, built.toByteString()); + + } + /** + * {@link Message.Builder#setUnknownFields(UnknownFieldSet)} and {@link + * Message.Builder#mergeUnknownFields(UnknownFieldSet)} should discard the unknown fields. + */ + private static void assertUnknownFieldsInUnknownFieldSetAreDiscarded(Message message) + throws Exception { + UnknownFieldSet unknownFields = UnknownFieldSet.newBuilder().mergeFrom(payload).build(); + Message built = message.newBuilderForType().setUnknownFields(unknownFields).build(); + assertEquals(message.getClass().getName(), 0, built.getSerializedSize()); + } + + private static void assertUnknownFieldsDefaultPreserved(MessageLite message) throws Exception { + { + MessageLite parsed = message.getParserForType().parseFrom(payload); + assertEquals(message.getClass().getName(), payload, parsed.toByteString()); + } + + { + MessageLite parsed = message.newBuilderForType().mergeFrom(payload).build(); + assertEquals(message.getClass().getName(), payload, parsed.toByteString()); + } + } + + private static void assertUnknownFieldsDefaultDiscarded(MessageLite message) throws Exception { + { + MessageLite parsed = message.getParserForType().parseFrom(payload); + assertEquals(message.getClass().getName(), 0, parsed.getSerializedSize()); + } + + { + MessageLite parsed = message.newBuilderForType().mergeFrom(payload).build(); + assertEquals(message.getClass().getName(), 0, parsed.getSerializedSize()); + } + } + + private static void assertUnknownFieldsExplicitlyDiscarded(Message message) throws Exception { + Message parsed = + DiscardUnknownFieldsParser.wrap(message.getParserForType()).parseFrom(payload); + assertEquals(message.getClass().getName(), 0, parsed.getSerializedSize()); + } + + private static final ByteString payload = + TestUtilLite.getAllLiteSetBuilder().build().toByteString(); +} diff --git a/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java b/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java index 4a42c897..ff686a0c 100644 --- a/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java +++ b/java/core/src/test/java/com/google/protobuf/FieldPresenceTest.java @@ -108,7 +108,7 @@ public class FieldPresenceTest extends TestCase { assertFalse(TestAllTypes.newBuilder().build().hasOptionalNestedMessage()); assertFalse(TestAllTypes.newBuilder().hasOptionalNestedMessage()); - // oneof fields don't have hasFoo() methods (even for message types). + // oneof fields don't have hasFoo() methods for non-message types. assertHasMethodRemoved( UnittestProto.TestAllTypes.class, TestAllTypes.class, @@ -121,10 +121,8 @@ public class FieldPresenceTest extends TestCase { UnittestProto.TestAllTypes.class, TestAllTypes.class, "OneofBytes"); - assertHasMethodRemoved( - UnittestProto.TestAllTypes.class, - TestAllTypes.class, - "OneofNestedMessage"); + assertFalse(TestAllTypes.newBuilder().build().hasOneofNestedMessage()); + assertFalse(TestAllTypes.newBuilder().hasOneofNestedMessage()); assertHasMethodRemoved( UnittestProto.TestAllTypes.Builder.class, @@ -138,10 +136,6 @@ public class FieldPresenceTest extends TestCase { UnittestProto.TestAllTypes.Builder.class, TestAllTypes.Builder.class, "OneofBytes"); - assertHasMethodRemoved( - UnittestProto.TestAllTypes.Builder.class, - TestAllTypes.Builder.class, - "OneofNestedMessage"); } public void testOneofEquals() throws Exception { diff --git a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java index 3eece26a..a4311d17 100644 --- a/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/GeneratedMessageTest.java @@ -923,15 +923,9 @@ public class GeneratedMessageTest extends TestCase { } public void testEnumValues() { - assertEquals( - TestAllTypes.NestedEnum.BAR.getNumber(), - TestAllTypes.NestedEnum.BAR_VALUE); - assertEquals( - TestAllTypes.NestedEnum.BAZ.getNumber(), - TestAllTypes.NestedEnum.BAZ_VALUE); - assertEquals( - TestAllTypes.NestedEnum.FOO.getNumber(), - TestAllTypes.NestedEnum.FOO_VALUE); + assertEquals(TestAllTypes.NestedEnum.BAR_VALUE, TestAllTypes.NestedEnum.BAR.getNumber()); + assertEquals(TestAllTypes.NestedEnum.BAZ_VALUE, TestAllTypes.NestedEnum.BAZ.getNumber()); + assertEquals(TestAllTypes.NestedEnum.FOO_VALUE, TestAllTypes.NestedEnum.FOO.getNumber()); } public void testNonNestedExtensionInitialization() { @@ -1319,51 +1313,51 @@ public class GeneratedMessageTest extends TestCase { assertFalse(builder.clearFooInt().hasFooInt()); TestOneof2 message2 = builder.build(); assertFalse(message2.hasFooInt()); - assertEquals(message2.getFooInt(), 0); + assertEquals(0, message2.getFooInt()); } // Enum { TestOneof2.Builder builder = TestOneof2.newBuilder(); - assertEquals(builder.getFooEnum(), TestOneof2.NestedEnum.FOO); + assertEquals(TestOneof2.NestedEnum.FOO, builder.getFooEnum()); assertTrue(builder.setFooEnum(TestOneof2.NestedEnum.BAR).hasFooEnum()); - assertEquals(builder.getFooEnum(), TestOneof2.NestedEnum.BAR); + assertEquals(TestOneof2.NestedEnum.BAR, builder.getFooEnum()); TestOneof2 message = builder.buildPartial(); assertTrue(message.hasFooEnum()); - assertEquals(message.getFooEnum(), TestOneof2.NestedEnum.BAR); + assertEquals(TestOneof2.NestedEnum.BAR, message.getFooEnum()); assertFalse(builder.clearFooEnum().hasFooEnum()); TestOneof2 message2 = builder.build(); assertFalse(message2.hasFooEnum()); - assertEquals(message2.getFooEnum(), TestOneof2.NestedEnum.FOO); + assertEquals(TestOneof2.NestedEnum.FOO, message2.getFooEnum()); } // String { TestOneof2.Builder builder = TestOneof2.newBuilder(); - assertEquals(builder.getFooString(), ""); + assertEquals("", builder.getFooString()); builder.setFooString("foo"); assertTrue(builder.hasFooString()); - assertEquals(builder.getFooString(), "foo"); + assertEquals("foo", builder.getFooString()); TestOneof2 message = builder.buildPartial(); assertTrue(message.hasFooString()); - assertEquals(message.getFooString(), "foo"); + assertEquals("foo", message.getFooString()); assertEquals(message.getFooStringBytes(), TestUtil.toBytes("foo")); assertFalse(builder.clearFooString().hasFooString()); TestOneof2 message2 = builder.buildPartial(); assertFalse(message2.hasFooString()); - assertEquals(message2.getFooString(), ""); + assertEquals("", message2.getFooString()); assertEquals(message2.getFooStringBytes(), TestUtil.toBytes("")); // Get method should not change the oneof value. builder.setFooInt(123); - assertEquals(builder.getFooString(), ""); + assertEquals("", builder.getFooString()); assertEquals(builder.getFooStringBytes(), TestUtil.toBytes("")); assertEquals(123, builder.getFooInt()); message = builder.build(); - assertEquals(message.getFooString(), ""); + assertEquals("", message.getFooString()); assertEquals(message.getFooStringBytes(), TestUtil.toBytes("")); assertEquals(123, message.getFooInt()); } @@ -1371,38 +1365,38 @@ public class GeneratedMessageTest extends TestCase { // Cord { TestOneof2.Builder builder = TestOneof2.newBuilder(); - assertEquals(builder.getFooCord(), ""); + assertEquals("", builder.getFooCord()); builder.setFooCord("foo"); assertTrue(builder.hasFooCord()); - assertEquals(builder.getFooCord(), "foo"); + assertEquals("foo", builder.getFooCord()); TestOneof2 message = builder.buildPartial(); assertTrue(message.hasFooCord()); - assertEquals(message.getFooCord(), "foo"); + assertEquals("foo", message.getFooCord()); assertEquals(message.getFooCordBytes(), TestUtil.toBytes("foo")); assertFalse(builder.clearFooCord().hasFooCord()); TestOneof2 message2 = builder.build(); assertFalse(message2.hasFooCord()); - assertEquals(message2.getFooCord(), ""); + assertEquals("", message2.getFooCord()); assertEquals(message2.getFooCordBytes(), TestUtil.toBytes("")); } // StringPiece { TestOneof2.Builder builder = TestOneof2.newBuilder(); - assertEquals(builder.getFooStringPiece(), ""); + assertEquals("", builder.getFooStringPiece()); builder.setFooStringPiece("foo"); assertTrue(builder.hasFooStringPiece()); - assertEquals(builder.getFooStringPiece(), "foo"); + assertEquals("foo", builder.getFooStringPiece()); TestOneof2 message = builder.buildPartial(); assertTrue(message.hasFooStringPiece()); - assertEquals(message.getFooStringPiece(), "foo"); + assertEquals("foo", message.getFooStringPiece()); assertEquals(message.getFooStringPieceBytes(), TestUtil.toBytes("foo")); assertFalse(builder.clearFooStringPiece().hasFooStringPiece()); TestOneof2 message2 = builder.build(); assertFalse(message2.hasFooStringPiece()); - assertEquals(message2.getFooStringPiece(), ""); + assertEquals("", message2.getFooStringPiece()); assertEquals(message2.getFooStringPieceBytes(), TestUtil.toBytes("")); } @@ -1410,20 +1404,20 @@ public class GeneratedMessageTest extends TestCase { { // set TestOneof2.Builder builder = TestOneof2.newBuilder(); - assertEquals(builder.getFooMessage().getQuxInt(), 0); + assertEquals(0, builder.getFooMessage().getQuxInt()); builder.setFooMessage( TestOneof2.NestedMessage.newBuilder().setQuxInt(234).build()); assertTrue(builder.hasFooMessage()); - assertEquals(builder.getFooMessage().getQuxInt(), 234); + assertEquals(234, builder.getFooMessage().getQuxInt()); TestOneof2 message = builder.buildPartial(); assertTrue(message.hasFooMessage()); - assertEquals(message.getFooMessage().getQuxInt(), 234); + assertEquals(234, message.getFooMessage().getQuxInt()); // clear assertFalse(builder.clearFooMessage().hasFooString()); message = builder.build(); assertFalse(message.hasFooMessage()); - assertEquals(message.getFooMessage().getQuxInt(), 0); + assertEquals(0, message.getFooMessage().getQuxInt()); // nested builder builder = TestOneof2.newBuilder(); @@ -1432,10 +1426,10 @@ public class GeneratedMessageTest extends TestCase { assertFalse(builder.hasFooMessage()); builder.getFooMessageBuilder().setQuxInt(123); assertTrue(builder.hasFooMessage()); - assertEquals(builder.getFooMessage().getQuxInt(), 123); + assertEquals(123, builder.getFooMessage().getQuxInt()); message = builder.build(); assertTrue(message.hasFooMessage()); - assertEquals(message.getFooMessage().getQuxInt(), 123); + assertEquals(123, message.getFooMessage().getQuxInt()); } // LazyMessage is tested in LazyMessageLiteTest.java @@ -1448,7 +1442,7 @@ public class GeneratedMessageTest extends TestCase { TestOneof2 message = builder.setFooInt(123).build(); TestOneof2 message2 = TestOneof2.newBuilder().mergeFrom(message).build(); assertTrue(message2.hasFooInt()); - assertEquals(message2.getFooInt(), 123); + assertEquals(123, message2.getFooInt()); } // String @@ -1457,7 +1451,7 @@ public class GeneratedMessageTest extends TestCase { TestOneof2 message = builder.setFooString("foo").build(); TestOneof2 message2 = TestOneof2.newBuilder().mergeFrom(message).build(); assertTrue(message2.hasFooString()); - assertEquals(message2.getFooString(), "foo"); + assertEquals("foo", message2.getFooString()); } // Enum @@ -1466,7 +1460,7 @@ public class GeneratedMessageTest extends TestCase { TestOneof2 message = builder.setFooEnum(TestOneof2.NestedEnum.BAR).build(); TestOneof2 message2 = TestOneof2.newBuilder().mergeFrom(message).build(); assertTrue(message2.hasFooEnum()); - assertEquals(message2.getFooEnum(), TestOneof2.NestedEnum.BAR); + assertEquals(TestOneof2.NestedEnum.BAR, message2.getFooEnum()); } // Message @@ -1476,7 +1470,7 @@ public class GeneratedMessageTest extends TestCase { TestOneof2.NestedMessage.newBuilder().setQuxInt(234).build()).build(); TestOneof2 message2 = TestOneof2.newBuilder().mergeFrom(message).build(); assertTrue(message2.hasFooMessage()); - assertEquals(message2.getFooMessage().getQuxInt(), 234); + assertEquals(234, message2.getFooMessage().getQuxInt()); } } @@ -1488,7 +1482,7 @@ public class GeneratedMessageTest extends TestCase { ByteString serialized = message.toByteString(); TestOneof2 message2 = TestOneof2.parseFrom(serialized); assertTrue(message2.hasFooInt()); - assertEquals(message2.getFooInt(), 123); + assertEquals(123, message2.getFooInt()); } // String @@ -1498,7 +1492,7 @@ public class GeneratedMessageTest extends TestCase { ByteString serialized = message.toByteString(); TestOneof2 message2 = TestOneof2.parseFrom(serialized); assertTrue(message2.hasFooString()); - assertEquals(message2.getFooString(), "foo"); + assertEquals("foo", message2.getFooString()); } // Enum @@ -1508,7 +1502,7 @@ public class GeneratedMessageTest extends TestCase { ByteString serialized = message.toByteString(); TestOneof2 message2 = TestOneof2.parseFrom(serialized); assertTrue(message2.hasFooEnum()); - assertEquals(message2.getFooEnum(), TestOneof2.NestedEnum.BAR); + assertEquals(TestOneof2.NestedEnum.BAR, message2.getFooEnum()); } // Message @@ -1519,7 +1513,7 @@ public class GeneratedMessageTest extends TestCase { ByteString serialized = message.toByteString(); TestOneof2 message2 = TestOneof2.parseFrom(serialized); assertTrue(message2.hasFooMessage()); - assertEquals(message2.getFooMessage().getQuxInt(), 234); + assertEquals(234, message2.getFooMessage().getQuxInt()); } } diff --git a/java/core/src/test/java/com/google/protobuf/LiteTest.java b/java/core/src/test/java/com/google/protobuf/LiteTest.java index 98c637a9..ba8bcb1c 100644 --- a/java/core/src/test/java/com/google/protobuf/LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/LiteTest.java @@ -33,6 +33,7 @@ package com.google.protobuf; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import com.google.protobuf.FieldPresenceTestProto.TestAllTypes; import com.google.protobuf.UnittestImportLite.ImportEnumLite; import com.google.protobuf.UnittestImportPublicLite.PublicImportMessageLite; import com.google.protobuf.UnittestLite.ForeignEnumLite; @@ -52,7 +53,12 @@ import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.BarPrime; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.Foo; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestOneofEquals; import protobuf_unittest.lite_equals_and_hash.LiteEqualsAndHash.TestRecursiveOneof; +import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import junit.framework.TestCase; /** @@ -155,6 +161,31 @@ public class LiteTest extends TestCase { // expected. } } + + public void testMemoization() throws Exception { + TestAllExtensionsLite message = TestUtilLite.getAllLiteExtensionsSet(); + + // Test serialized size is memoized + message.memoizedSerializedSize = -1; + int size = message.getSerializedSize(); + assertTrue(size > 0); + assertEquals(size, message.memoizedSerializedSize); + + // Test hashCode is memoized + assertEquals(0, message.memoizedHashCode); + int hashCode = message.hashCode(); + assertTrue(hashCode != 0); + assertEquals(hashCode, message.memoizedHashCode); + + // Test isInitialized is memoized + Field memo = message.getClass().getDeclaredField("memoizedIsInitialized"); + memo.setAccessible(true); + memo.set(message, (byte) -1); + boolean initialized = message.isInitialized(); + assertTrue(initialized); + // We have to cast to Byte first. Casting to byte causes a type error + assertEquals(1, ((Byte) memo.get(message)).intValue()); + } public void testSanityCopyOnWrite() throws InvalidProtocolBufferException { // Since builders are implemented as a thin wrapper around a message @@ -2378,4 +2409,187 @@ public class LiteTest extends TestCase { expected.getUnfinishedMessage()); } } + + // Make sure we haven't screwed up the code generation for packing fields by default. + public void testPackedSerialization() throws Exception { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + builder.addRepeatedInt32(4321); + builder.addRepeatedNestedEnum(TestAllTypes.NestedEnum.BAZ); + TestAllTypes message = builder.build(); + + CodedInputStream in = CodedInputStream.newInstance(message.toByteArray()); + + while (!in.isAtEnd()) { + int tag = in.readTag(); + assertEquals(WireFormat.WIRETYPE_LENGTH_DELIMITED, WireFormat.getTagWireType(tag)); + in.skipField(tag); + } + } + + public void testAddAllIteratesOnce() { + TestAllTypesLite message = + TestAllTypesLite.newBuilder() + .addAllRepeatedBool(new OneTimeIterableList(false)) + .addAllRepeatedInt32(new OneTimeIterableList(0)) + .addAllRepeatedInt64(new OneTimeIterableList(0L)) + .addAllRepeatedFloat(new OneTimeIterableList(0f)) + .addAllRepeatedDouble(new OneTimeIterableList(0d)) + .addAllRepeatedBytes(new OneTimeIterableList(ByteString.EMPTY)) + .addAllRepeatedString(new OneTimeIterableList("")) + .addAllRepeatedNestedMessage( + new OneTimeIterableList(NestedMessage.getDefaultInstance())) + .addAllRepeatedBool(new OneTimeIterable(false)) + .addAllRepeatedInt32(new OneTimeIterable(0)) + .addAllRepeatedInt64(new OneTimeIterable(0L)) + .addAllRepeatedFloat(new OneTimeIterable(0f)) + .addAllRepeatedDouble(new OneTimeIterable(0d)) + .addAllRepeatedBytes(new OneTimeIterable(ByteString.EMPTY)) + .addAllRepeatedString(new OneTimeIterable("")) + .addAllRepeatedNestedMessage(new OneTimeIterable(NestedMessage.getDefaultInstance())) + .build(); + } + + public void testAddAllIteratesOnce_throwsOnNull() { + TestAllTypesLite.Builder builder = TestAllTypesLite.newBuilder(); + try { + builder.addAllRepeatedBool(new OneTimeIterableList(true, false, (Boolean) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 2 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedBoolCount()); + } + + try { + builder.addAllRepeatedBool(new OneTimeIterable(true, false, (Boolean) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 2 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedBoolCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedBool(new OneTimeIterableList((Boolean) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedBoolCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedInt32(new OneTimeIterableList((Integer) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedInt32Count()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedInt64(new OneTimeIterableList((Long) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedInt64Count()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedFloat(new OneTimeIterableList((Float) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedFloatCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedDouble(new OneTimeIterableList((Double) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedDoubleCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedBytes(new OneTimeIterableList((ByteString) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedBytesCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedString(new OneTimeIterableList("", "", (String) null, "")); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 2 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedStringCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedString(new OneTimeIterable("", "", (String) null, "")); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 2 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedStringCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedString(new OneTimeIterableList((String) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedStringCount()); + } + + try { + builder = TestAllTypesLite.newBuilder(); + builder.addAllRepeatedNestedMessage(new OneTimeIterableList((NestedMessage) null)); + fail(); + } catch (NullPointerException expected) { + assertEquals("Element at index 0 is null.", expected.getMessage()); + assertEquals(0, builder.getRepeatedNestedMessageCount()); + } + } + + private static final class OneTimeIterableList<T> extends ArrayList<T> { + private boolean wasIterated = false; + + OneTimeIterableList(T... contents) { + addAll(Arrays.asList(contents)); + } + + @Override + public Iterator<T> iterator() { + if (wasIterated) { + fail(); + } + wasIterated = true; + return super.iterator(); + } + } + + private static final class OneTimeIterable<T> implements Iterable<T> { + private final List<T> list; + private boolean wasIterated = false; + + OneTimeIterable(T... contents) { + list = Arrays.asList(contents); + } + + @Override + public Iterator<T> iterator() { + if (wasIterated) { + fail(); + } + wasIterated = true; + return list.iterator(); + } + } } diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java b/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java index 0a14f584..da9195f9 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java @@ -408,12 +408,12 @@ public final class MapForProto2LiteTest extends TestCase { TestMap map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToInt32Field(5, bytes) .build()); - assertEquals(map.getInt32ToInt32FieldOrDefault(5, -1), 0); + assertEquals(0, map.getInt32ToInt32FieldOrDefault(5, -1)); map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToStringField(stringKey, 5) .build()); - assertEquals(map.getInt32ToStringFieldOrDefault(0, null), ""); + assertEquals("", map.getInt32ToStringFieldOrDefault(0, null)); map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToBytesField(stringKey, 5) @@ -423,7 +423,7 @@ public final class MapForProto2LiteTest extends TestCase { map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToEnumField(stringKey, bytes) .build()); - assertEquals(map.getInt32ToEnumFieldOrDefault(0, null), TestMap.EnumValue.FOO); + assertEquals(TestMap.EnumValue.FOO, map.getInt32ToEnumFieldOrDefault(0, null)); try { tryParseTestMap(BizarroTestMap.newBuilder() @@ -439,7 +439,7 @@ public final class MapForProto2LiteTest extends TestCase { map = tryParseTestMap(BizarroTestMap.newBuilder() .putStringToInt32Field(stringKey, bytes) .build()); - assertEquals(map.getStringToInt32FieldOrDefault(stringKey, -1), 0); + assertEquals(0, map.getStringToInt32FieldOrDefault(stringKey, -1)); } public void testMergeFrom() throws Exception { diff --git a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java index 453d3928..37827f76 100644 --- a/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java +++ b/java/core/src/test/java/com/google/protobuf/MapForProto2Test.java @@ -546,12 +546,12 @@ public class MapForProto2Test extends TestCase { TestMap map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToInt32Field(5, bytes) .build()); - assertEquals(map.getInt32ToInt32FieldOrDefault(5, -1), 0); + assertEquals(0, map.getInt32ToInt32FieldOrDefault(5, -1)); map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToStringField(stringKey, 5) .build()); - assertEquals(map.getInt32ToStringFieldOrDefault(0, null), ""); + assertEquals("", map.getInt32ToStringFieldOrDefault(0, null)); map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToBytesField(stringKey, 5) @@ -561,7 +561,7 @@ public class MapForProto2Test extends TestCase { map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToEnumField(stringKey, bytes) .build()); - assertEquals(map.getInt32ToEnumFieldOrDefault(0, null), TestMap.EnumValue.FOO); + assertEquals(TestMap.EnumValue.FOO, map.getInt32ToEnumFieldOrDefault(0, null)); try { tryParseTestMap(BizarroTestMap.newBuilder() @@ -577,7 +577,7 @@ public class MapForProto2Test extends TestCase { map = tryParseTestMap(BizarroTestMap.newBuilder() .putStringToInt32Field(stringKey, bytes) .build()); - assertEquals(map.getStringToInt32FieldOrDefault(stringKey, -1), 0); + assertEquals(0, map.getStringToInt32FieldOrDefault(stringKey, -1)); } public void testMergeFrom() throws Exception { diff --git a/java/core/src/test/java/com/google/protobuf/MapTest.java b/java/core/src/test/java/com/google/protobuf/MapTest.java index 45019537..64ae4435 100644 --- a/java/core/src/test/java/com/google/protobuf/MapTest.java +++ b/java/core/src/test/java/com/google/protobuf/MapTest.java @@ -576,12 +576,12 @@ public class MapTest extends TestCase { TestMap map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToInt32Field(5, bytes) .build()); - assertEquals(map.getInt32ToInt32FieldOrDefault(5, -1), 0); + assertEquals(0, map.getInt32ToInt32FieldOrDefault(5, -1)); map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToStringField(stringKey, 5) .build()); - assertEquals(map.getInt32ToStringFieldOrDefault(0, null), ""); + assertEquals("", map.getInt32ToStringFieldOrDefault(0, null)); map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToBytesField(stringKey, 5) @@ -591,7 +591,7 @@ public class MapTest extends TestCase { map = tryParseTestMap(BizarroTestMap.newBuilder() .putInt32ToEnumField(stringKey, bytes) .build()); - assertEquals(map.getInt32ToEnumFieldOrDefault(0, null), TestMap.EnumValue.FOO); + assertEquals(TestMap.EnumValue.FOO, map.getInt32ToEnumFieldOrDefault(0, null)); try { tryParseTestMap(BizarroTestMap.newBuilder() @@ -607,7 +607,7 @@ public class MapTest extends TestCase { map = tryParseTestMap(BizarroTestMap.newBuilder() .putStringToInt32Field(stringKey, bytes) .build()); - assertEquals(map.getStringToInt32FieldOrDefault(stringKey, -1), 0); + assertEquals(0, map.getStringToInt32FieldOrDefault(stringKey, -1)); } public void testMergeFrom() throws Exception { diff --git a/java/core/src/test/java/com/google/protobuf/MessageTest.java b/java/core/src/test/java/com/google/protobuf/MessageTest.java index 75b79a34..9d55d0dd 100644 --- a/java/core/src/test/java/com/google/protobuf/MessageTest.java +++ b/java/core/src/test/java/com/google/protobuf/MessageTest.java @@ -321,8 +321,10 @@ public class MessageTest extends TestCase { assertTrue(result.getField(result.getDescriptorForType() .findFieldByName("repeated_foreign_message")) instanceof List<?>); - assertEquals(result.getRepeatedFieldCount(result.getDescriptorForType() - .findFieldByName("repeated_foreign_message")), 0); + assertEquals( + 0, + result.getRepeatedFieldCount( + result.getDescriptorForType().findFieldByName("repeated_foreign_message"))); } /** Test reading repeated message from DynamicMessage. */ @@ -345,7 +347,9 @@ public class MessageTest extends TestCase { assertTrue(result.getField(result.getDescriptorForType() .findFieldByName("repeated_foreign_message")) instanceof List<?>); - assertEquals(result.getRepeatedFieldCount(result.getDescriptorForType() - .findFieldByName("repeated_foreign_message")), 2); + assertEquals( + 2, + result.getRepeatedFieldCount( + result.getDescriptorForType().findFieldByName("repeated_foreign_message"))); } } diff --git a/java/core/src/test/java/com/google/protobuf/TestBadIdentifiers.java b/java/core/src/test/java/com/google/protobuf/TestBadIdentifiers.java index 2c60fe0e..4af55429 100644 --- a/java/core/src/test/java/com/google/protobuf/TestBadIdentifiers.java +++ b/java/core/src/test/java/com/google/protobuf/TestBadIdentifiers.java @@ -92,5 +92,31 @@ public class TestBadIdentifiers extends TestCase { assertEquals(0L, message.getExtension( TestBadIdentifiersProto.TestConflictingFieldNames.int64FieldList).longValue()); + assertEquals("", message.getFieldName32()); + assertEquals("", message.getFieldName33()); + assertEquals(0, message.get2Conflict34()); + assertEquals(0, message.get2Conflict35()); + + } + + public void testNumberFields() throws Exception { + TestBadIdentifiersProto.TestLeadingNumberFields message = + TestBadIdentifiersProto.TestLeadingNumberFields.getDefaultInstance(); + // Make sure generated accessors are properly named. + assertFalse(message.has30DayImpressions()); + assertEquals(0, message.get30DayImpressions()); + assertEquals(0, message.get60DayImpressionsCount()); + assertEquals(0, message.get60DayImpressionsList().size()); + + assertFalse(message.has2Underscores()); + assertEquals("", message.get2Underscores()); + assertEquals(0, message.get2RepeatedUnderscoresCount()); + assertEquals(0, message.get2RepeatedUnderscoresList().size()); + + assertFalse(message.has32()); + assertEquals(0, message.get32()); + assertEquals(0, message.get64Count()); + assertEquals(0, message.get64List().size()); + } } diff --git a/java/core/src/test/java/com/google/protobuf/TestBadIdentifiersLite.java b/java/core/src/test/java/com/google/protobuf/TestBadIdentifiersLite.java new file mode 100644 index 00000000..37f94c03 --- /dev/null +++ b/java/core/src/test/java/com/google/protobuf/TestBadIdentifiersLite.java @@ -0,0 +1,83 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import junit.framework.TestCase; + +/** + * Tests that proto2 api generation doesn't cause compile errors when compiling protocol buffers + * that have names that would otherwise conflict if not fully qualified (like @Deprecated + * and @Override). + * + * <p>Forked from {@link TestBadIdentifiers}. + * + * @author jonp@google.com (Jon Perlow) + */ +public final class TestBadIdentifiersLite extends TestCase { + + public void testCompilation() { + // If this compiles, it means the generation was correct. + TestBadIdentifiersProto.Deprecated.newBuilder(); + TestBadIdentifiersProto.Override.newBuilder(); + } + + public void testConflictingFieldNames() throws Exception { + TestBadIdentifiersProto.TestConflictingFieldNames message = + TestBadIdentifiersProto.TestConflictingFieldNames.getDefaultInstance(); + // Make sure generated accessors are properly named. + assertEquals(0, message.getInt32Field1Count()); + assertEquals(0, message.getEnumField2Count()); + assertEquals(0, message.getStringField3Count()); + assertEquals(0, message.getBytesField4Count()); + assertEquals(0, message.getMessageField5Count()); + + assertEquals(0, message.getInt32FieldCount11()); + assertEquals(0, message.getEnumFieldCount12().getNumber()); + assertEquals("", message.getStringFieldCount13()); + assertEquals(ByteString.EMPTY, message.getBytesFieldCount14()); + assertEquals(0, message.getMessageFieldCount15().getSerializedSize()); + + assertEquals(0, message.getInt32Field21Count()); + assertEquals(0, message.getEnumField22Count()); + assertEquals(0, message.getStringField23Count()); + assertEquals(0, message.getBytesField24Count()); + assertEquals(0, message.getMessageField25Count()); + + assertEquals(0, message.getInt32Field1List().size()); + assertEquals(0, message.getInt32FieldList31()); + + assertEquals(0, message.getInt64FieldCount()); + assertEquals(0L, message.getExtension( + TestBadIdentifiersProto.TestConflictingFieldNames.int64FieldCount).longValue()); + assertEquals(0L, message.getExtension( + TestBadIdentifiersProto.TestConflictingFieldNames.int64FieldList).longValue()); + } +} diff --git a/java/core/src/test/java/com/google/protobuf/UnknownEnumValueTest.java b/java/core/src/test/java/com/google/protobuf/UnknownEnumValueTest.java index 8f45976f..88cbbf86 100644 --- a/java/core/src/test/java/com/google/protobuf/UnknownEnumValueTest.java +++ b/java/core/src/test/java/com/google/protobuf/UnknownEnumValueTest.java @@ -36,7 +36,6 @@ import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.FieldPresenceTestProto.TestAllTypes; import com.google.protobuf.TextFormat.ParseException; - import junit.framework.TestCase; /** @@ -151,18 +150,15 @@ public class UnknownEnumValueTest extends TestCase { assertEquals(4321, unknown4321.getNumber()); assertEquals(5432, unknown5432.getNumber()); assertEquals(6543, unknown6543.getNumber()); - + // Unknown EnumValueDescriptor will map to UNRECOGNIZED. assertEquals( - TestAllTypes.NestedEnum.valueOf(unknown4321), - TestAllTypes.NestedEnum.UNRECOGNIZED); + TestAllTypes.NestedEnum.UNRECOGNIZED, TestAllTypes.NestedEnum.valueOf(unknown4321)); assertEquals( - TestAllTypes.NestedEnum.valueOf(unknown5432), - TestAllTypes.NestedEnum.UNRECOGNIZED); + TestAllTypes.NestedEnum.UNRECOGNIZED, TestAllTypes.NestedEnum.valueOf(unknown5432)); assertEquals( - TestAllTypes.NestedEnum.valueOf(unknown6543), - TestAllTypes.NestedEnum.UNRECOGNIZED); - + TestAllTypes.NestedEnum.UNRECOGNIZED, TestAllTypes.NestedEnum.valueOf(unknown6543)); + // Setters also accept unknown EnumValueDescriptor. builder.setField(optionalNestedEnumField, unknown6543); builder.setRepeatedField(repeatedNestedEnumField, 0, unknown4321); diff --git a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto index d2c77936..ff5bf3ae 100644 --- a/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto +++ b/java/core/src/test/proto/com/google/protobuf/test_bad_identifiers.proto @@ -148,6 +148,12 @@ message TestConflictingFieldNames { // the method getInt32FieldList(). required int32 int32_field_list = 31; // NO_PROTO3 + // These field pairs have the same Java converted name + optional string field_name = 32; // NO_PROTO3 + optional string field__name = 33; // NO_PROTO3 + optional int32 _2conflict = 34; // NO_PROTO3 + optional int32 __2conflict = 35; + extensions 1000 to max; // NO_PROTO3 repeated int64 int64_field = 41; @@ -166,3 +172,14 @@ message TestMapField { map<int32, int32> map_field = 1; } + +message TestLeadingNumberFields { + optional int32 _30day_impressions = 1; + repeated string _60day_impressions = 2; + + optional string __2_underscores = 3; + repeated string __2repeated_underscores = 4; + + optional int32 _32 = 32; + repeated int64 _64 = 64; +} diff --git a/java/pom.xml b/java/pom.xml index 39ed31ab..dd3ba3b8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -11,7 +11,7 @@ <groupId>com.google.protobuf</groupId> <artifactId>protobuf-parent</artifactId> - <version>3.3.2</version> + <version>3.4.0</version> <packaging>pom</packaging> <name>Protocol Buffers [Parent]</name> diff --git a/java/util/pom.xml b/java/util/pom.xml index 62394ed7..61abd146 100644 --- a/java/util/pom.xml +++ b/java/util/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-parent</artifactId> - <version>3.3.2</version> + <version>3.4.0</version> </parent> <artifactId>protobuf-java-util</artifactId> diff --git a/java/util/src/main/java/com/google/protobuf/util/Durations.java b/java/util/src/main/java/com/google/protobuf/util/Durations.java index 46b21828..fb7f4343 100644 --- a/java/util/src/main/java/com/google/protobuf/util/Durations.java +++ b/java/util/src/main/java/com/google/protobuf/util/Durations.java @@ -30,7 +30,6 @@ package com.google.protobuf.util; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.math.IntMath.checkedAdd; import static com.google.common.math.IntMath.checkedSubtract; import static com.google.common.math.LongMath.checkedAdd; @@ -84,6 +83,17 @@ public final class Durations { } /** + * Compares two durations. The value returned is identical to what would be returned by: + * {@code Durations.comparator().compare(x, y)}. + * + * @return the value {@code 0} if {@code x == y}; a value less than {@code 0} if {@code x < y}; + * and a value greater than {@code 0} if {@code x > y} + */ + public static int compare(Duration x, Duration y) { + return COMPARATOR.compare(x, y); + } + + /** * Returns true if the given {@link Duration} is valid. The {@code seconds} value must be in the * range [-315,576,000,000, +315,576,000,000]. The {@code nanos} value must be in the range * [-999,999,999, +999,999,999]. @@ -124,14 +134,13 @@ public final class Durations { public static Duration checkValid(Duration duration) { long seconds = duration.getSeconds(); int nanos = duration.getNanos(); - checkArgument( - isValid(seconds, nanos), - "Duration is not valid. See proto definition for valid values. " + if (!isValid(seconds, nanos)) { + throw new IllegalArgumentException(String.format( + "Duration is not valid. See proto definition for valid values. " + "Seconds (%s) must be in range [-315,576,000,000, +315,576,000,000]. " + "Nanos (%s) must be in range [-999,999,999, +999,999,999]. " - + "Nanos must have the same sign as seconds", - seconds, - nanos); + + "Nanos must have the same sign as seconds", seconds, nanos)); + } return duration; } diff --git a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java index 21d11b2c..b2f849c4 100644 --- a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java +++ b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java @@ -311,16 +311,19 @@ public class FieldMaskUtil { return replacePrimitiveFields; } - public void setReplaceMessageFields(boolean value) { + public MergeOptions setReplaceMessageFields(boolean value) { replaceMessageFields = value; + return this; } - public void setReplaceRepeatedFields(boolean value) { + public MergeOptions setReplaceRepeatedFields(boolean value) { replaceRepeatedFields = value; + return this; } - public void setReplacePrimitiveFields(boolean value) { + public MergeOptions setReplacePrimitiveFields(boolean value) { replacePrimitiveFields = value; + return this; } } diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index e1c2d73d..a603d96a 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -1686,7 +1686,11 @@ public class JsonFormat { } private ByteString parseBytes(JsonElement json) throws InvalidProtocolBufferException { - return ByteString.copyFrom(BaseEncoding.base64().decode(json.getAsString())); + try { + return ByteString.copyFrom(BaseEncoding.base64().decode(json.getAsString())); + } catch (IllegalArgumentException e) { + return ByteString.copyFrom(BaseEncoding.base64Url().decode(json.getAsString())); + } } private EnumValueDescriptor parseEnum(EnumDescriptor enumDescriptor, JsonElement json) diff --git a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java index d0bac417..7a1f2014 100644 --- a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java +++ b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java @@ -30,7 +30,6 @@ package com.google.protobuf.util; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.math.IntMath.checkedAdd; import static com.google.common.math.IntMath.checkedSubtract; import static com.google.common.math.LongMath.checkedAdd; @@ -115,6 +114,17 @@ public final class Timestamps { } /** + * Compares two timestamps. The value returned is identical to what would be returned by: + * {@code Timestamps.comparator().compare(x, y)}. + * + * @return the value {@code 0} if {@code x == y}; a value less than {@code 0} if {@code x < y}; + * and a value greater than {@code 0} if {@code x > y} + */ + public static int compare(Timestamp x, Timestamp y) { + return COMPARATOR.compare(x, y); + } + + /** * Returns true if the given {@link Timestamp} is valid. The {@code seconds} value must be in the * range [-62,135,596,800, +253,402,300,799] (i.e., between 0001-01-01T00:00:00Z and * 9999-12-31T23:59:59Z). The {@code nanos} value must be in the range [0, +999,999,999]. @@ -149,13 +159,12 @@ public final class Timestamps { public static Timestamp checkValid(Timestamp timestamp) { long seconds = timestamp.getSeconds(); int nanos = timestamp.getNanos(); - checkArgument( - isValid(seconds, nanos), - "Timestamp is not valid. See proto definition for valid values. " + if (!isValid(seconds, nanos)) { + throw new IllegalArgumentException(String.format( + "Timestamp is not valid. See proto definition for valid values. " + "Seconds (%s) must be in range [-62,135,596,800, +253,402,300,799]. " - + "Nanos (%s) must be in range [0, +999,999,999].", - seconds, - nanos); + + "Nanos (%s) must be in range [0, +999,999,999].", seconds, nanos)); + } return timestamp; } diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index 460b81f6..e4c5387a 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -73,7 +73,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Locale; -import java.util.Map; import java.util.Set; import junit.framework.TestCase; @@ -1144,7 +1143,8 @@ public class JsonFormatTest extends TestCase { } public void testParserAcceptBase64Variants() throws Exception { - assertAccepts("optionalBytes", "AQI"); + assertAccepts("optionalBytes", "AQI"); // No padding + assertAccepts("optionalBytes", "-_w"); // base64Url, no padding } public void testParserRejectInvalidEnumValue() throws Exception { |