From d36c0c538a545fac5d9db6ba65c525246d4efa95 Mon Sep 17 00:00:00 2001 From: Feng Xiao Date: Wed, 29 Mar 2017 14:32:48 -0700 Subject: Down-integrate from google3. --- src/google/protobuf/io/coded_stream.cc | 46 +++++-------------------- src/google/protobuf/io/coded_stream.h | 39 +++++++++++---------- src/google/protobuf/io/coded_stream_unittest.cc | 33 +----------------- src/google/protobuf/io/gzip_stream.h | 2 +- 4 files changed, 31 insertions(+), 89 deletions(-) (limited to 'src/google/protobuf/io') diff --git a/src/google/protobuf/io/coded_stream.cc b/src/google/protobuf/io/coded_stream.cc index 3c2e0fbd..df4250e5 100644 --- a/src/google/protobuf/io/coded_stream.cc +++ b/src/google/protobuf/io/coded_stream.cc @@ -76,10 +76,6 @@ CodedInputStream::~CodedInputStream() { if (input_ != NULL) { BackUpInputToCurrentPosition(); } - - if (total_bytes_warning_threshold_ == -2) { - GOOGLE_LOG(WARNING) << "The total number of bytes read was " << total_bytes_read_; - } } // Static. @@ -123,21 +119,15 @@ CodedInputStream::Limit CodedInputStream::PushLimit(int byte_limit) { Limit old_limit = current_limit_; // security: byte_limit is possibly evil, so check for negative values - // and overflow. - if (byte_limit >= 0 && - byte_limit <= INT_MAX - current_position) { + // and overflow. Also check that the new requested limit is before the + // previous limit; otherwise we continue to enforce the previous limit. + if GOOGLE_PREDICT_TRUE(byte_limit >= 0 && + byte_limit <= INT_MAX - current_position && + byte_limit < current_limit_ - current_position) { current_limit_ = current_position + byte_limit; - } else { - // Negative or overflow. - current_limit_ = INT_MAX; + RecomputeBufferLimits(); } - // We need to enforce all limits, not just the new one, so if the previous - // limit was before the new requested limit, we continue to enforce the - // previous limit. - current_limit_ = std::min(current_limit_, old_limit); - - RecomputeBufferLimits(); return old_limit; } @@ -185,16 +175,12 @@ int CodedInputStream::BytesUntilLimit() const { void CodedInputStream::SetTotalBytesLimit( int total_bytes_limit, int warning_threshold) { + (void) warning_threshold; + // Make sure the limit isn't already past, since this could confuse other // code. int current_position = CurrentPosition(); total_bytes_limit_ = std::max(current_position, total_bytes_limit); - if (warning_threshold >= 0) { - total_bytes_warning_threshold_ = warning_threshold; - } else { - // warning_threshold is negative - total_bytes_warning_threshold_ = -1; - } RecomputeBufferLimits(); } @@ -605,20 +591,6 @@ bool CodedInputStream::Refresh() { return false; } - if (total_bytes_warning_threshold_ >= 0 && - total_bytes_read_ >= total_bytes_warning_threshold_) { - GOOGLE_LOG(INFO) << "Reading dangerously large protocol message. If the " - "message turns out to be larger than " - << total_bytes_limit_ << " bytes, parsing will be halted " - "for security reasons. To increase the limit (or to " - "disable these warnings), see " - "CodedInputStream::SetTotalBytesLimit() in " - "google/protobuf/io/coded_stream.h."; - - // Don't warn again for this stream, and print total size at the end. - total_bytes_warning_threshold_ = -2; - } - const void* void_buffer; int buffer_size; if (NextNonEmpty(input_, &void_buffer, &buffer_size)) { @@ -655,7 +627,7 @@ bool CodedInputStream::Refresh() { // CodedOutputStream ================================================= -bool CodedOutputStream::default_serialization_deterministic_ = false; +google::protobuf::internal::AtomicWord CodedOutputStream::default_serialization_deterministic_ = 0; CodedOutputStream::CodedOutputStream(ZeroCopyOutputStream* output) : output_(output), diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index b71b4a98..20d86143 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -131,7 +131,9 @@ #define PROTOBUF_LITTLE_ENDIAN 1 #endif #endif +#include #include +#include namespace google { @@ -371,11 +373,10 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // maximum message length should be limited to the shortest length that // will not harm usability. The theoretical shortest message that could // cause integer overflows is 512MB. The default limit is 64MB. Apps - // should set shorter limits if possible. If warning_threshold is not -1, - // a warning will be printed to stderr after warning_threshold bytes are - // read. For backwards compatibility all negative values get squashed to -1, - // as other negative values might have special internal meanings. - // An error will always be printed to stderr if the limit is reached. + // should set shorter limits if possible. For backwards compatibility all + // negative values get squashed to -1, as other negative values might have + // special internal meanings. An error will always be printed to stderr if + // the limit is reached. // // This is unrelated to PushLimit()/PopLimit(). // @@ -568,12 +569,6 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // current_limit_. Set using SetTotalBytesLimit(). int total_bytes_limit_; - // If positive/0: Limit for bytes read after which a warning due to size - // should be logged. - // If -1: Printing of warning disabled. Can be set by client. - // If -2: Internal: Limit has been reached, print full size when destructing. - int total_bytes_warning_threshold_; - // Current recursion budget, controlled by IncrementRecursionDepth() and // similar. Starts at recursion_limit_ and goes down: if this reaches // -1 we are over budget. @@ -581,6 +576,8 @@ class LIBPROTOBUF_EXPORT CodedInputStream { // Recursion depth limit, set by SetRecursionLimit(). int recursion_limit_; + bool disable_strict_correctness_enforcement_; + // See SetExtensionRegistry(). const DescriptorPool* extension_pool_; MessageFactory* extension_factory_; @@ -642,8 +639,6 @@ class LIBPROTOBUF_EXPORT CodedInputStream { static const int kDefaultTotalBytesLimit = INT_MAX; - static const int kDefaultTotalBytesWarningThreshold = 32 << 20; // 32MB - static int default_recursion_limit_; // 100 by default. }; @@ -870,7 +865,7 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { } static bool IsDefaultSerializationDeterministic() { - return default_serialization_deterministic_; + return google::protobuf::internal::Acquire_Load(&default_serialization_deterministic_); } private: @@ -885,7 +880,8 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { // See SetSerializationDeterministic() regarding these three fields. bool serialization_deterministic_is_overridden_; bool serialization_deterministic_override_; - static bool default_serialization_deterministic_; + // Conceptually, default_serialization_deterministic_ is an atomic bool. + static google::protobuf::internal::AtomicWord default_serialization_deterministic_; // Advance the buffer by a given number of bytes. void Advance(int amount); @@ -904,10 +900,15 @@ class LIBPROTOBUF_EXPORT CodedOutputStream { void WriteVarint64SlowPath(uint64 value); // See above. Other projects may use "friend" to allow them to call this. - // Requires: no protocol buffer serialization in progress. + // After SetDefaultSerializationDeterministic() completes, all protocol + // buffer serializations will be deterministic by default. Thread safe. + // However, the meaning of "after" is subtle here: to be safe, each thread + // that wants deterministic serialization by default needs to call + // SetDefaultSerializationDeterministic() or ensure on its own that another + // thread has done so. friend void ::google::protobuf::internal::MapTestForceDeterministic(); static void SetDefaultSerializationDeterministic() { - default_serialization_deterministic_ = true; + google::protobuf::internal::Release_Store(&default_serialization_deterministic_, 1); } }; @@ -1395,9 +1396,9 @@ inline CodedInputStream::CodedInputStream(ZeroCopyInputStream* input) current_limit_(kint32max), buffer_size_after_limit_(0), total_bytes_limit_(kDefaultTotalBytesLimit), - total_bytes_warning_threshold_(kDefaultTotalBytesWarningThreshold), recursion_budget_(default_recursion_limit_), recursion_limit_(default_recursion_limit_), + disable_strict_correctness_enforcement_(true), extension_pool_(NULL), extension_factory_(NULL) { // Eagerly Refresh() so buffer space is immediately available. @@ -1416,9 +1417,9 @@ inline CodedInputStream::CodedInputStream(const uint8* buffer, int size) current_limit_(size), buffer_size_after_limit_(0), total_bytes_limit_(kDefaultTotalBytesLimit), - total_bytes_warning_threshold_(kDefaultTotalBytesWarningThreshold), recursion_budget_(default_recursion_limit_), recursion_limit_(default_recursion_limit_), + disable_strict_correctness_enforcement_(true), extension_pool_(NULL), extension_factory_(NULL) { // Note that setting current_limit_ == size is important to prevent some diff --git a/src/google/protobuf/io/coded_stream_unittest.cc b/src/google/protobuf/io/coded_stream_unittest.cc index 31574d5b..96f91ae9 100644 --- a/src/google/protobuf/io/coded_stream_unittest.cc +++ b/src/google/protobuf/io/coded_stream_unittest.cc @@ -63,6 +63,7 @@ namespace protobuf { namespace io { namespace { + // =================================================================== // Data-Driven Test Infrastructure @@ -1296,35 +1297,6 @@ void CodedStreamTest::SetupTotalBytesLimitWarningTest( *out_warnings = scoped_log.GetMessages(WARNING); } -TEST_F(CodedStreamTest, TotalBytesLimitWarning) { - std::vector errors; - std::vector warnings; - SetupTotalBytesLimitWarningTest(10240, 1024, &errors, &warnings); - - EXPECT_EQ(0, errors.size()); - - EXPECT_EQ(1, warnings.size()); - EXPECT_PRED_FORMAT2(testing::IsSubstring, - "The total number of bytes read was 2048", - warnings[0]); -} - -TEST_F(CodedStreamTest, TotalBytesLimitWarningDisabled) { - std::vector errors; - std::vector warnings; - - // Test with -1 - SetupTotalBytesLimitWarningTest(10240, -1, &errors, &warnings); - EXPECT_EQ(0, errors.size()); - EXPECT_EQ(0, warnings.size()); - - // Test again with -2, expecting the same result - SetupTotalBytesLimitWarningTest(10240, -2, &errors, &warnings); - EXPECT_EQ(0, errors.size()); - EXPECT_EQ(0, warnings.size()); -} - - TEST_F(CodedStreamTest, RecursionLimit) { ArrayInputStream input(buffer_, sizeof(buffer_)); CodedInputStream coded_input(&input); @@ -1425,9 +1397,6 @@ TEST_F(CodedStreamTest, InputOver2G) { EXPECT_EQ(0, errors.size()); } -// =================================================================== - - } // namespace } // namespace io } // namespace protobuf diff --git a/src/google/protobuf/io/gzip_stream.h b/src/google/protobuf/io/gzip_stream.h index df1a446e..15b02fe3 100644 --- a/src/google/protobuf/io/gzip_stream.h +++ b/src/google/protobuf/io/gzip_stream.h @@ -118,7 +118,7 @@ class LIBPROTOBUF_EXPORT GzipOutputStream : public ZeroCopyOutputStream { ZLIB = 2, }; - struct LIBPROTOBUF_EXPORT Options { + struct Options { // Defaults to GZIP. Format format; -- cgit v1.2.3