Integrate changes from internal code.

protoc
* Enum values may now have custom options, using syntax similar to field
  options.
* Fixed bug where .proto files which use custom options but don't actually
  define them (i.e. they import another .proto file defining the options)
  had to explicitly import descriptor.proto.
* Adjacent string literals in .proto files will now be concatenated, like in
  C.

C++
* Generated message classes now have a Swap() method which efficiently swaps
  the contents of two objects.
* All message classes now have a SpaceUsed() method which returns an estimate
  of the number of bytes of allocated memory currently owned by the object.
  This is particularly useful when you are reusing a single message object
  to improve performance but want to make sure it doesn't bloat up too large.
* New method Message::SerializeAsString() returns a string containing the
  serialized data.  May be more convenient than calling
  SerializeToString(string*).
* In debug mode, log error messages when string-type fields are found to
  contain bytes that are not valid UTF-8.
* Fixed bug where a message with multiple extension ranges couldn't parse
  extensions.
* Fixed bug where MergeFrom(const Message&) didn't do anything if invoked on
  a message that contained no fields (but possibly contained extensions).
* Fixed ShortDebugString() to not be O(n^2).  Durr.
* Fixed crash in TextFormat parsing if the first token in the input caused a
  tokenization error.

Java
* New overload of mergeFrom() which parses a slice of a byte array instead
  of the whole thing.
* New method ByteString.asReadOnlyByteBuffer() does what it sounds like.
* Improved performance of isInitialized() when optimizing for code size.

Python
* Corrected ListFields() signature in Message base class to match what
  subclasses actually implement.
* Some minor refactoring.

8 files changed, 160 insertions, 10 deletions

diff --git a/java/src/main/java/com/google/protobuf/AbstractMessage.java b/java/src/main/java/com/google/protobuf/AbstractMessage.java
index 51c6e783..2f61859d 100644
--- a/java/src/main/java/com/google/protobuf/AbstractMessage.java
+++ b/java/src/main/java/com/google/protobuf/AbstractMessage.java
@@ -77,6 +77,7 @@ public abstract class AbstractMessage implements Message {
     return true;
   }
 
+  @Override
   public final String toString() {
     return TextFormat.printToString(this);
   }
@@ -199,6 +200,7 @@ public abstract class AbstractMessage implements Message {
   public static abstract class Builder<BuilderType extends Builder>
       implements Message.Builder {
     // The compiler produces an error if this is not declared explicitly.
+    @Override
     public abstract BuilderType clone();
 
     public BuilderType clear() {
@@ -307,8 +309,13 @@ public abstract class AbstractMessage implements Message {
 
     public BuilderType mergeFrom(byte[] data)
         throws InvalidProtocolBufferException {
+      return mergeFrom(data, 0, data.length);
+    }
+
+    public BuilderType mergeFrom(byte[] data, int off, int len)
+        throws InvalidProtocolBufferException {
       try {
-        CodedInputStream input = CodedInputStream.newInstance(data);
+        CodedInputStream input = CodedInputStream.newInstance(data, off, len);
         mergeFrom(input);
         input.checkLastTagWas(0);
         return (BuilderType) this;
@@ -322,10 +329,18 @@ public abstract class AbstractMessage implements Message {
     }
 
     public BuilderType mergeFrom(
-        byte[] data, ExtensionRegistry extensionRegistry)
+        byte[] data,
+        ExtensionRegistry extensionRegistry)
+        throws InvalidProtocolBufferException {
+      return mergeFrom(data, 0, data.length, extensionRegistry);
+    }
+
+    public BuilderType mergeFrom(
+        byte[] data, int off, int len,
+        ExtensionRegistry extensionRegistry)
         throws InvalidProtocolBufferException {
       try {
-        CodedInputStream input = CodedInputStream.newInstance(data);
+        CodedInputStream input = CodedInputStream.newInstance(data, off, len);
         mergeFrom(input, extensionRegistry);
         input.checkLastTagWas(0);
         return (BuilderType) this;
diff --git a/java/src/main/java/com/google/protobuf/ByteString.java b/java/src/main/java/com/google/protobuf/ByteString.java
index 9814dfc1..f376e7a1 100644
--- a/java/src/main/java/com/google/protobuf/ByteString.java
+++ b/java/src/main/java/com/google/protobuf/ByteString.java
@@ -35,6 +35,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.FilterOutputStream;
 import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
 
 /**
  * Immutable array of bytes.
@@ -154,6 +155,15 @@ public final class ByteString {
   }
 
   /**
+   * Constructs a new read-only {@code java.nio.ByteBuffer} with the
+   * same backing byte array.
+   */
+  public ByteBuffer asReadOnlyByteBuffer() {
+    ByteBuffer byteBuffer = ByteBuffer.wrap(this.bytes);
+    return byteBuffer.asReadOnlyBuffer();
+  }
+
+  /**
    * Constructs a new {@code String} by decoding the bytes using the
    * specified charset.
    */
diff --git a/java/src/main/java/com/google/protobuf/CodedInputStream.java b/java/src/main/java/com/google/protobuf/CodedInputStream.java
index c956ed2b..caef068b 100644
--- a/java/src/main/java/com/google/protobuf/CodedInputStream.java
+++ b/java/src/main/java/com/google/protobuf/CodedInputStream.java
@@ -59,7 +59,14 @@ public final class CodedInputStream {
    * Create a new CodedInputStream wrapping the given byte array.
    */
   public static CodedInputStream newInstance(byte[] buf) {
-    return new CodedInputStream(buf);
+    return newInstance(buf, 0, buf.length);
+  }
+
+  /**
+   * Create a new CodedInputStream wrapping the given byte array slice.
+   */
+  public static CodedInputStream newInstance(byte[] buf, int off, int len) {
+    return new CodedInputStream(buf, off, len);
   }
 
   // -----------------------------------------------------------------
@@ -454,7 +461,7 @@ public final class CodedInputStream {
   private byte[] buffer;
   private int bufferSize;
   private int bufferSizeAfterLimit = 0;
-  private int bufferPos = 0;
+  private int bufferPos;
   private InputStream input;
   private int lastTag = 0;
 
@@ -479,15 +486,17 @@ public final class CodedInputStream {
   private static final int DEFAULT_SIZE_LIMIT = 64 << 20;  // 64MB
   private static final int BUFFER_SIZE = 4096;
 
-  private CodedInputStream(byte[] buffer) {
+  private CodedInputStream(byte[] buffer, int off, int len) {
     this.buffer = buffer;
-    this.bufferSize = buffer.length;
+    this.bufferSize = off + len;
+    this.bufferPos = off;
     this.input = null;
   }
 
   private CodedInputStream(InputStream input) {
     this.buffer = new byte[BUFFER_SIZE];
     this.bufferSize = 0;
+    this.bufferPos = 0;
     this.input = input;
   }
 
diff --git a/java/src/main/java/com/google/protobuf/GeneratedMessage.java b/java/src/main/java/com/google/protobuf/GeneratedMessage.java
index bc231a15..b1be8b14 100644
--- a/java/src/main/java/com/google/protobuf/GeneratedMessage.java
+++ b/java/src/main/java/com/google/protobuf/GeneratedMessage.java
@@ -87,6 +87,33 @@ public abstract class GeneratedMessage extends AbstractMessage {
     }
     return result;
   }
+  
+  public boolean isInitialized() {
+    for (FieldDescriptor field : getDescriptorForType().getFields()) {
+      // Check that all required fields are present.
+      if (field.isRequired()) {
+        if (!hasField(field)) {
+          return false;
+        }
+      }
+      // Check that embedded messages are initialized.
+      if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
+        if (field.isRepeated()) {
+          for (Message element : (List<Message>) getField(field)) {
+            if (!element.isInitialized()) {
+              return false;
+            }
+          }
+        } else {
+          if (hasField(field) && !((Message) getField(field)).isInitialized()) {
+            return false;
+          }
+        }
+      }
+    }
+
+    return true;
+  }
 
   public Map<FieldDescriptor, Object> getAllFields() {
     return Collections.unmodifiableMap(getAllFieldsMutable());
@@ -370,6 +397,10 @@ public abstract class GeneratedMessage extends AbstractMessage {
     protected boolean extensionsAreInitialized() {
       return extensions.isInitialized();
     }
+    
+    public boolean isInitialized() {
+      return super.isInitialized() && extensionsAreInitialized();
+    }
 
     /**
      * Used by subclasses to serialize extensions.  Extension ranges may be
diff --git a/java/src/main/java/com/google/protobuf/Message.java b/java/src/main/java/com/google/protobuf/Message.java
index add5dab6..9635387a 100644
--- a/java/src/main/java/com/google/protobuf/Message.java
+++ b/java/src/main/java/com/google/protobuf/Message.java
@@ -400,6 +400,13 @@ public interface Message {
     /**
      * Parse {@code data} as a message of this type and merge it with the
      * message being built.  This is just a small wrapper around
+     * {@link #mergeFrom(CodedInputStream)}.
+     */
+    public Builder mergeFrom(byte[] data, int off, int len) throws InvalidProtocolBufferException;
+
+    /**
+     * Parse {@code data} as a message of this type and merge it with the
+     * message being built.  This is just a small wrapper around
      * {@link #mergeFrom(CodedInputStream,ExtensionRegistry)}.
      */
     Builder mergeFrom(byte[] data,
@@ -407,6 +414,15 @@ public interface Message {
                       throws InvalidProtocolBufferException;
 
     /**
+     * Parse {@code data} as a message of this type and merge it with the
+     * message being built.  This is just a small wrapper around
+     * {@link #mergeFrom(CodedInputStream,ExtensionRegistry)}.
+     */
+    Builder mergeFrom(byte[] data, int off, int len,
+                      ExtensionRegistry extensionRegistry)
+                      throws InvalidProtocolBufferException;
+
+    /**
      * Parse a message of this type from {@code input} and merge it with the
      * message being built.  This is just a small wrapper around
      * {@link #mergeFrom(CodedInputStream)}.  Note that this method always
diff --git a/java/src/test/java/com/google/protobuf/CodedInputStreamTest.java b/java/src/test/java/com/google/protobuf/CodedInputStreamTest.java
index eca02f74..eaaddf44 100644
--- a/java/src/test/java/com/google/protobuf/CodedInputStreamTest.java
+++ b/java/src/test/java/com/google/protobuf/CodedInputStreamTest.java
@@ -412,4 +412,14 @@ public class CodedInputStreamTest extends TestCase {
     String text = input.readString();
     assertEquals(0xfffd, text.charAt(0));
   }
+
+  public void testReadFromSlice() throws Exception {
+    byte[] bytes = bytes(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+    CodedInputStream in = CodedInputStream.newInstance(bytes, 3, 5);
+    for (int i = 3; i < 8; i++) {
+      assertEquals(i, in.readRawByte());
+    }
+    // eof
+    assertEquals(0, in.readTag());
+  }
 }
diff --git a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java
index 958943da..95c269a2 100644
--- a/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java
+++ b/java/src/test/java/com/google/protobuf/GeneratedMessageTest.java
@@ -30,8 +30,9 @@
 
 package com.google.protobuf;
 
-import protobuf_unittest.UnittestOptimizeFor.TestRequiredOptimizedForSize;
+import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize;
 import protobuf_unittest.UnittestOptimizeFor.TestOptionalOptimizedForSize;
+import protobuf_unittest.UnittestOptimizeFor.TestRequiredOptimizedForSize;
 import protobuf_unittest.UnittestProto;
 import protobuf_unittest.UnittestProto.ForeignMessage;
 import protobuf_unittest.UnittestProto.ForeignEnum;
@@ -260,8 +261,10 @@ public class GeneratedMessageTest extends TestCase {
         MultipleFilesTestProto.extensionWithOuter));
   }
 
-  public void testOptionalFieldWithRequiredSubfieldsOptimizedForSize() throws Exception {
-    TestOptionalOptimizedForSize message = TestOptionalOptimizedForSize.getDefaultInstance();
+  public void testOptionalFieldWithRequiredSubfieldsOptimizedForSize()
+    throws Exception {
+    TestOptionalOptimizedForSize message =
+        TestOptionalOptimizedForSize.getDefaultInstance();
     assertTrue(message.isInitialized());
     
     message = TestOptionalOptimizedForSize.newBuilder().setO(
@@ -274,4 +277,19 @@ public class GeneratedMessageTest extends TestCase {
         ).buildPartial();
     assertTrue(message.isInitialized());
   }
+
+  public void testUninitializedExtensionInOptimizedForSize()
+      throws Exception {
+    TestOptimizedForSize.Builder builder = TestOptimizedForSize.newBuilder();
+    builder.setExtension(TestOptimizedForSize.testExtension2,
+        TestRequiredOptimizedForSize.newBuilder().buildPartial());
+    assertFalse(builder.isInitialized());
+    assertFalse(builder.buildPartial().isInitialized());
+
+    builder = TestOptimizedForSize.newBuilder();
+    builder.setExtension(TestOptimizedForSize.testExtension2,
+        TestRequiredOptimizedForSize.newBuilder().setX(10).buildPartial());
+    assertTrue(builder.isInitialized());
+    assertTrue(builder.buildPartial().isInitialized());
+  }
 }
diff --git a/java/src/test/java/com/google/protobuf/WireFormatTest.java b/java/src/test/java/com/google/protobuf/WireFormatTest.java
index d7295c20..3fb54fcf 100644
--- a/java/src/test/java/com/google/protobuf/WireFormatTest.java
+++ b/java/src/test/java/com/google/protobuf/WireFormatTest.java
@@ -135,6 +135,47 @@ public class WireFormatTest extends TestCase {
     assertFieldsInOrder(dynamic_data);
   }
 
+  private ExtensionRegistry getTestFieldOrderingsRegistry() {
+    ExtensionRegistry result = ExtensionRegistry.newInstance();
+    result.add(UnittestProto.myExtensionInt);
+    result.add(UnittestProto.myExtensionString);
+    return result;
+  }
+
+  public void testParseMultipleExtensionRanges() throws Exception {
+    // Make sure we can parse a message that contains multiple extensions
+    // ranges.
+    TestFieldOrderings source =
+      TestFieldOrderings.newBuilder()
+        .setMyInt(1)
+        .setMyString("foo")
+        .setMyFloat(1.0F)
+        .setExtension(UnittestProto.myExtensionInt, 23)
+        .setExtension(UnittestProto.myExtensionString, "bar")
+        .build();
+    TestFieldOrderings dest =
+      TestFieldOrderings.parseFrom(source.toByteString(),
+                                   getTestFieldOrderingsRegistry());
+    assertEquals(source, dest);
+  }
+
+  public void testParseMultipleExtensionRangesDynamic() throws Exception {
+    // Same as above except with DynamicMessage.
+    Descriptors.Descriptor descriptor = TestFieldOrderings.getDescriptor();
+    DynamicMessage source =
+      DynamicMessage.newBuilder(TestFieldOrderings.getDescriptor())
+        .setField(descriptor.findFieldByName("my_int"), 1L)
+        .setField(descriptor.findFieldByName("my_string"), "foo")
+        .setField(descriptor.findFieldByName("my_float"), 1.0F)
+        .setField(UnittestProto.myExtensionInt.getDescriptor(), 23)
+        .setField(UnittestProto.myExtensionString.getDescriptor(), "bar")
+        .build();
+    DynamicMessage dest =
+      DynamicMessage.parseFrom(descriptor, source.toByteString(),
+                               getTestFieldOrderingsRegistry());
+    assertEquals(source, dest);
+  }
+
   private static final int UNKNOWN_TYPE_ID = 1550055;
   private static final int TYPE_ID_1 =
     TestMessageSetExtension1.getDescriptor().getExtensions().get(0).getNumber();