Down-integrate from internal code base.

11 files changed, 147 insertions, 21 deletions

diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc
index 53449755..df88205c 100644
--- a/src/google/protobuf/io/coded_stream.cc
+++ b/src/google/protobuf/io/coded_stream.cc
@@ -40,8 +40,10 @@
 
 #include <google/protobuf/io/coded_stream_inl.h>
 #include <algorithm>
+#include <utility>
 #include <limits.h>
 #include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/arena.h>
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/stl_util.h>
 
@@ -149,6 +151,19 @@ void CodedInputStream::PopLimit(Limit limit) {
   legitimate_message_end_ = false;
 }
 
+std::pair<CodedInputStream::Limit, int>
+CodedInputStream::IncrementRecursionDepthAndPushLimit(int byte_limit) {
+  return make_pair(PushLimit(byte_limit), --recursion_budget_);
+}
+
+bool CodedInputStream::DecrementRecursionDepthAndPopLimit(Limit limit) {
+  bool result = ConsumedEntireMessage();
+  PopLimit(limit);
+  GOOGLE_DCHECK_LT(recursion_budget_, recursion_limit_);
+  ++recursion_budget_;
+  return result;
+}
+
 int CodedInputStream::BytesUntilLimit() const {
   if (current_limit_ == INT_MAX) return -1;
   int current_position = CurrentPosition();
diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h
index 775c6067..b9c30fa3 100644
--- a/src/google/protobuf/io/coded_stream.h
+++ b/src/google/protobuf/io/coded_stream.h
@@ -110,6 +110,7 @@
 #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__
 
 #include <string>
+#include <utility>
 #ifdef _MSC_VER
   #if defined(_M_IX86) && \
       !defined(PROTOBUF_DISABLE_LITTLE_ENDIAN_OPT_FOR_TEST)
@@ -129,8 +130,8 @@
 #endif
 #include <google/protobuf/stubs/common.h>
 
-
 namespace google {
+
 namespace protobuf {
 
 class DescriptorPool;
@@ -388,6 +389,23 @@ class LIBPROTOBUF_EXPORT CodedInputStream {
   // Decrements the recursion depth.
   void DecrementRecursionDepth();
 
+  // Shorthand for make_pair(PushLimit(byte_limit), --recursion_budget_).
+  // Using this can reduce code size and complexity in some cases.  The caller
+  // is expected to check that the second part of the result is non-negative (to
+  // bail out if the depth of recursion is too high) and, if all is well, to
+  // later pass the first part of the result to PopLimit() or similar.
+  std::pair<CodedInputStream::Limit, int> IncrementRecursionDepthAndPushLimit(
+      int byte_limit);
+
+  // Helper that is equivalent to: {
+  //  bool result = ConsumedEntireMessage();
+  //  PopLimit(limit);
+  //  DecrementRecursionDepth();
+  //  return result; }
+  // Using this can reduce code size and complexity in some cases.
+  // Do not use unless the current recursion depth is greater than zero.
+  bool DecrementRecursionDepthAndPopLimit(Limit limit);
+
   // Extension Registry ----------------------------------------------
   // ADVANCED USAGE:  99.9% of people can ignore this section.
   //
@@ -470,9 +488,9 @@ class LIBPROTOBUF_EXPORT CodedInputStream {
  private:
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(CodedInputStream);
 
-  ZeroCopyInputStream* input_;
   const uint8* buffer_;
   const uint8* buffer_end_;     // pointer to the end of the buffer.
+  ZeroCopyInputStream* input_;
   int total_bytes_read_;  // total bytes read from input_, including
                           // the current buffer
 
@@ -513,9 +531,10 @@ class LIBPROTOBUF_EXPORT CodedInputStream {
   // If -2: Internal: Limit has been reached, print full size when destructing.
   int total_bytes_warning_threshold_;
 
-  // Current recursion depth, controlled by IncrementRecursionDepth() and
-  // DecrementRecursionDepth().
-  int recursion_depth_;
+  // Current recursion budget, controlled by IncrementRecursionDepth() and
+  // similar.  Starts at recursion_limit_ and goes down: if this reaches
+  // -1 we are over budget.
+  int recursion_budget_;
   // Recursion depth limit, set by SetRecursionLimit().
   int recursion_limit_;
 
@@ -1132,16 +1151,17 @@ inline void CodedOutputStream::Advance(int amount) {
 }
 
 inline void CodedInputStream::SetRecursionLimit(int limit) {
+  recursion_budget_ += limit - recursion_limit_;
   recursion_limit_ = limit;
 }
 
 inline bool CodedInputStream::IncrementRecursionDepth() {
-  ++recursion_depth_;
-  return recursion_depth_ <= recursion_limit_;
+  --recursion_budget_;
+  return recursion_budget_ >= 0;
 }
 
 inline void CodedInputStream::DecrementRecursionDepth() {
-  if (recursion_depth_ > 0) --recursion_depth_;
+  if (recursion_budget_ < recursion_limit_) ++recursion_budget_;
 }
 
 inline void CodedInputStream::SetExtensionRegistry(const DescriptorPool* pool,
@@ -1163,9 +1183,9 @@ inline int CodedInputStream::BufferSize() const {
 }
 
 inline CodedInputStream::CodedInputStream(ZeroCopyInputStream* input)
-  : input_(input),
-    buffer_(NULL),
+  : buffer_(NULL),
     buffer_end_(NULL),
+    input_(input),
     total_bytes_read_(0),
     overflow_bytes_(0),
     last_tag_(0),
@@ -1175,7 +1195,7 @@ inline CodedInputStream::CodedInputStream(ZeroCopyInputStream* input)
     buffer_size_after_limit_(0),
     total_bytes_limit_(kDefaultTotalBytesLimit),
     total_bytes_warning_threshold_(kDefaultTotalBytesWarningThreshold),
-    recursion_depth_(0),
+    recursion_budget_(default_recursion_limit_),
     recursion_limit_(default_recursion_limit_),
     extension_pool_(NULL),
     extension_factory_(NULL) {
@@ -1184,9 +1204,9 @@ inline CodedInputStream::CodedInputStream(ZeroCopyInputStream* input)
 }
 
 inline CodedInputStream::CodedInputStream(const uint8* buffer, int size)
-  : input_(NULL),
-    buffer_(buffer),
+  : buffer_(buffer),
     buffer_end_(buffer + size),
+    input_(NULL),
     total_bytes_read_(size),
     overflow_bytes_(0),
     last_tag_(0),
@@ -1196,7 +1216,7 @@ inline CodedInputStream::CodedInputStream(const uint8* buffer, int size)
     buffer_size_after_limit_(0),
     total_bytes_limit_(kDefaultTotalBytesLimit),
     total_bytes_warning_threshold_(kDefaultTotalBytesWarningThreshold),
-    recursion_depth_(0),
+    recursion_budget_(default_recursion_limit_),
     recursion_limit_(default_recursion_limit_),
     extension_pool_(NULL),
     extension_factory_(NULL) {
diff --git a/src/google/protobuf/io/coded_stream_inl.h b/src/google/protobuf/io/coded_stream_inl.h
index 88c14cab..cd8d1746 100644
--- a/src/google/protobuf/io/coded_stream_inl.h
+++ b/src/google/protobuf/io/coded_stream_inl.h
@@ -36,6 +36,7 @@
 #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_INL_H__
 #define GOOGLE_PROTOBUF_IO_CODED_STREAM_INL_H__
 
+#include <google/protobuf/stubs/common.h>
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <string>
@@ -51,10 +52,12 @@ inline bool CodedInputStream::InternalReadStringInline(string* buffer,
 
   if (BufferSize() >= size) {
     STLStringResizeUninitialized(buffer, size);
-    // When buffer is empty, string_as_array(buffer) will return NULL but memcpy
-    // requires non-NULL pointers even when size is 0. Hench this check.
-    if (size > 0) {
-      memcpy(mutable_string_data(buffer), buffer_, size);
+    std::pair<char*, bool> z = as_string_data(buffer);
+    if (z.second) {
+      // Oddly enough, memcpy() requires its first two args to be non-NULL even
+      // if we copy 0 bytes.  So, we have ensured that z.first is non-NULL here.
+      GOOGLE_DCHECK(z.first != NULL);
+      memcpy(z.first, buffer_, size);
       Advance(size);
     }
     return true;
diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc
index f4cb5ea1..bbe5e399 100644
--- a/src/google/protobuf/io/coded_stream_unittest.cc
+++ b/src/google/protobuf/io/coded_stream_unittest.cc
@@ -144,6 +144,7 @@ uint8 CodedStreamTest::buffer_[CodedStreamTest::kBufferSize];
 // checks.
 const int kBlockSizes[] = {1, 2, 3, 5, 7, 13, 32, 1024};
 
+
 // -------------------------------------------------------------------
 // Varint tests.
 
diff --git a/src/google/protobuf/io/gzip_stream.cc b/src/google/protobuf/io/gzip_stream.cc
index ee286961..e6037863 100644
--- a/src/google/protobuf/io/gzip_stream.cc
+++ b/src/google/protobuf/io/gzip_stream.cc
@@ -48,7 +48,7 @@ static const int kDefaultBufferSize = 65536;
 
 GzipInputStream::GzipInputStream(
     ZeroCopyInputStream* sub_stream, Format format, int buffer_size)
-    : format_(format), sub_stream_(sub_stream), zerror_(Z_OK) {
+    : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
   zcontext_.zalloc = Z_NULL;
   zcontext_.zfree = Z_NULL;
   zcontext_.opaque = Z_NULL;
@@ -134,6 +134,7 @@ bool GzipInputStream::Next(const void** data, int* size) {
     if (zcontext_.next_out != NULL) {
       // sub_stream_ may have concatenated streams to follow
       zerror_ = inflateEnd(&zcontext_);
+      byte_count_ += zcontext_.total_out;
       if (zerror_ != Z_OK) {
         return false;
       }
@@ -178,8 +179,12 @@ bool GzipInputStream::Skip(int count) {
   return ok;
 }
 int64 GzipInputStream::ByteCount() const {
-  return zcontext_.total_out +
-    (((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_));
+  int64 ret = byte_count_ + zcontext_.total_out;
+  if (zcontext_.next_out != NULL && output_position_ != NULL) {
+    ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
+           reinterpret_cast<uintptr_t>(output_position_);
+  }
+  return ret;
 }
 
 // =========================================================================
diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h
index c7ccc260..82445000 100644
--- a/src/google/protobuf/io/gzip_stream.h
+++ b/src/google/protobuf/io/gzip_stream.h
@@ -99,6 +99,7 @@ class LIBPROTOBUF_EXPORT GzipInputStream : public ZeroCopyInputStream {
   void* output_buffer_;
   void* output_position_;
   size_t output_buffer_length_;
+  int64 byte_count_;
 
   int Inflate(int flush);
   void DoNextOutput(const void** data, int* size);
diff --git a/src/google/protobuf/io/printer.cc b/src/google/protobuf/io/printer.cc
index c8df4177..e621ba1d 100644
--- a/src/google/protobuf/io/printer.cc
+++ b/src/google/protobuf/io/printer.cc
@@ -142,6 +142,19 @@ void Printer::Print(const char* text,
   Print(vars, text);
 }
 
+void Printer::Print(const char* text,
+                    const char* variable1, const string& value1,
+                    const char* variable2, const string& value2,
+                    const char* variable3, const string& value3,
+                    const char* variable4, const string& value4) {
+  map<string, string> vars;
+  vars[variable1] = value1;
+  vars[variable2] = value2;
+  vars[variable3] = value3;
+  vars[variable4] = value4;
+  Print(vars, text);
+}
+
 void Printer::Indent() {
   indent_ += "  ";
 }
diff --git a/src/google/protobuf/io/printer.h b/src/google/protobuf/io/printer.h
index f06cbf2f..92ce3409 100644
--- a/src/google/protobuf/io/printer.h
+++ b/src/google/protobuf/io/printer.h
@@ -86,6 +86,11 @@ class LIBPROTOBUF_EXPORT Printer {
   void Print(const char* text, const char* variable1, const string& value1,
                                const char* variable2, const string& value2,
                                const char* variable3, const string& value3);
+  // Like the first Print(), except the substitutions are given as parameters.
+  void Print(const char* text, const char* variable1, const string& value1,
+                               const char* variable2, const string& value2,
+                               const char* variable3, const string& value3,
+                               const char* variable4, const string& value4);
   // TODO(kenton):  Overloaded versions with more variables?  Three seems
   //   to be enough.
 
diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
index 58aff0e2..97b73b88 100644
--- a/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
+++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
@@ -37,6 +37,7 @@
 #include <algorithm>
 #include <limits>
 
+#include <google/protobuf/stubs/casts.h>
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/stl_util.h>
 
diff --git a/src/google/protobuf/io/zero_copy_stream_impl_lite.h b/src/google/protobuf/io/zero_copy_stream_impl_lite.h
index e18da72c..a517161d 100644
--- a/src/google/protobuf/io/zero_copy_stream_impl_lite.h
+++ b/src/google/protobuf/io/zero_copy_stream_impl_lite.h
@@ -334,6 +334,18 @@ class LIBPROTOBUF_EXPORT CopyingOutputStreamAdaptor : public ZeroCopyOutputStrea
 
 // ===================================================================
 
+// mutable_string_data() and as_string_data() are workarounds to improve
+// the performance of writing new data to an existing string.  Unfortunately
+// the methods provided by the string class are suboptimal, and using memcpy()
+// is mildly annoying because it requires its pointer args to be non-NULL even
+// if we ask it to copy 0 bytes.  Furthermore, string_as_array() has the
+// property that it always returns NULL if its arg is the empty string, exactly
+// what we want to avoid if we're using it in conjunction with memcpy()!
+// With C++11, the desired memcpy() boils down to memcpy(..., &(*s)[0], size),
+// where s is a string*.  Without C++11, &(*s)[0] is not guaranteed to be safe,
+// so we use string_as_array(), and live with the extra logic that tests whether
+// *s is empty.
+
 // Return a pointer to mutable characters underlying the given string.  The
 // return value is valid until the next time the string is resized.  We
 // trust the caller to treat the return value as an array of length s->size().
@@ -347,6 +359,19 @@ inline char* mutable_string_data(string* s) {
 #endif
 }
 
+// as_string_data(s) is equivalent to
+//  ({ char* p = mutable_string_data(s); make_pair(p, p != NULL); })
+// Sometimes it's faster: in some scenarios p cannot be NULL, and then the
+// code can avoid that check.
+inline std::pair<char*, bool> as_string_data(string* s) {
+  char *p = mutable_string_data(s);
+#ifdef LANG_CXX11
+  return make_pair(p, true);
+#else
+  return make_pair(p, p != NULL);
+#endif
+}
+
 }  // namespace io
 }  // namespace protobuf
 
diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc
index bf978cc8..dd3d1285 100644
--- a/src/google/protobuf/io/zero_copy_stream_unittest.cc
+++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc
@@ -656,6 +656,43 @@ TEST_F(IoTest, TwoSessionWriteGzip) {
   delete [] temp_buffer;
   delete [] buffer;
 }
+
+TEST_F(IoTest, GzipInputByteCountAfterClosed) {
+  string golden = "abcdefghijklmnopqrstuvwxyz";
+  string compressed = Compress(golden, GzipOutputStream::Options());
+
+  for (int i = 0; i < kBlockSizeCount; i++) {
+    ArrayInputStream arr_input(compressed.data(), compressed.size(),
+                               kBlockSizes[i]);
+    GzipInputStream gz_input(&arr_input);
+    const void* buffer;
+    int size;
+    while (gz_input.Next(&buffer, &size)) {
+      EXPECT_LE(gz_input.ByteCount(), golden.size());
+    }
+    EXPECT_EQ(golden.size(), gz_input.ByteCount());
+  }
+}
+
+TEST_F(IoTest, GzipInputByteCountAfterClosedConcatenatedStreams) {
+  string golden1 = "abcdefghijklmnopqrstuvwxyz";
+  string golden2 = "the quick brown fox jumps over the lazy dog";
+  const size_t total_size = golden1.size() + golden2.size();
+  string compressed = Compress(golden1, GzipOutputStream::Options()) +
+                      Compress(golden2, GzipOutputStream::Options());
+
+  for (int i = 0; i < kBlockSizeCount; i++) {
+    ArrayInputStream arr_input(compressed.data(), compressed.size(),
+                               kBlockSizes[i]);
+    GzipInputStream gz_input(&arr_input);
+    const void* buffer;
+    int size;
+    while (gz_input.Next(&buffer, &size)) {
+      EXPECT_LE(gz_input.ByteCount(), total_size);
+    }
+    EXPECT_EQ(total_size, gz_input.ByteCount());
+  }
+}
 #endif
 
 // There is no string input, only string output.  Also, it doesn't support