From cb3e84b78edf0725c3ee3e3e00469e1a25fac8b8 Mon Sep 17 00:00:00 2001 From: Byron Yi Date: Thu, 16 Mar 2017 20:01:22 +0800 Subject: migrate delimited messages functions to util package --- src/google/protobuf/util/delimited_message_util.cc | 79 ++++++++++++++++++++++ src/google/protobuf/util/delimited_message_util.h | 66 ++++++++++++++++++ .../protobuf/util/delimited_message_util_test.cc | 57 ++++++++++++++++ 3 files changed, 202 insertions(+) create mode 100644 src/google/protobuf/util/delimited_message_util.cc create mode 100644 src/google/protobuf/util/delimited_message_util.h create mode 100644 src/google/protobuf/util/delimited_message_util_test.cc (limited to 'src/google/protobuf/util') diff --git a/src/google/protobuf/util/delimited_message_util.cc b/src/google/protobuf/util/delimited_message_util.cc new file mode 100644 index 00000000..36e0118d --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.cc @@ -0,0 +1,79 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include + +namespace google { +namespace protobuf { +namespace util { + +bool SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor) { + io::FileOutputStream output(file_descriptor); + return SerializeDelimitedToZeroCopyStream(message, &output); +} + +bool SerializeDelimitedToOstream(const Message* message, ostream* output) { + { + io::OstreamOutputStream zero_copy_output(output); + if (!SerializeDelimitedToZeroCopyStream(message, &zero_copy_output)) return false; + } + return output->good(); +} + +bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof) { + google::protobuf::io::CodedInputStream coded_input(input); + return ParseDelimitedFromCodedStream(message, &coded_input, clean_eof); +} + +bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof) { + if (clean_eof != NULL) *clean_eof = false; + int start = input->CurrentPosition(); + + // Read the size. + uint32 size; + if (!input->ReadVarint32(&size)) { + if (clean_eof != NULL) *clean_eof = input->CurrentPosition() == start; + return false; + } + + // Tell the stream not to read beyond that size. + google::protobuf::io::CodedInputStream::Limit limit = input->PushLimit(size); + + // Parse the message. + if (!message->MergeFromCodedStream(input)) return false; + if (!input->ConsumedEntireMessage()) return false; + + // Release the limit. + input->PopLimit(limit); + + return true; +} + +bool SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output) { + google::protobuf::io::CodedOutputStream coded_output(output); + return SerializeDelimitedToCodedStream(message, &coded_output); +} + +bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output) { + // Write the size. + int size = message->ByteSize(); + output->WriteVarint32(size); + + // Write the content. + uint8* buffer = output->GetDirectBufferForNBytesAndAdvance(size); + if (buffer != NULL) { + // Optimization: The message fits in one buffer, so use the faster + // direct-to-array serialization path. + message->SerializeWithCachedSizesToArray(buffer); + } else { + // Slightly-slower path when the message is multiple buffers. + message->SerializeWithCachedSizes(output); + if (output->HadError()) return false; + } + + return true; +} + +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/delimited_message_util.h b/src/google/protobuf/util/delimited_message_util.h new file mode 100644 index 00000000..970f4e92 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.h @@ -0,0 +1,66 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#ifndef GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ +#define GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ + +#include + +#include +#include +#include + +namespace google { +namespace protobuf { +namespace util { + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +// +// Note that if you want to *read* a delimited message from a file descriptor +// or istream, you will need to construct an io::FileInputStream or +// io::OstreamInputStream (implementations of io::ZeroCopyStream) and use the +// MessageLite method ParseDelimitedFromZeroCopyStream(). You must then +// continue to use the same ZeroCopyInputStream to read all further data from +// the stream until EOF. This is because these ZeroCopyInputStream +// implementations are buffered: they read a big chunk of data at a time, +// then parse it. As a result, they may read past the end of the delimited +// message. There is no way for them to push the extra data back into the +// underlying source, so instead you must keep using the same stream object. +bool SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor); + +bool SerializeDelimitedToOstream(const Message* message, ostream* output); + +// Read a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally parsing consumes the entire input. A delimited message +// is a varint encoding the message size followed by a message of exactly +// that size. +// +// If |clean_eof| is not NULL, then it will be set to indicate whether the +// stream ended cleanly. That is, if the stream ends without this method +// having read any data at all from it, then *clean_eof will be set true, +// otherwise it will be set false. Note that these methods return false +// on EOF, but they also return false on other errors, so |clean_eof| is +// needed to distinguish a clean end from errors. +bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof); + +bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof); + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +bool SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output); + +bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output); + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ diff --git a/src/google/protobuf/util/delimited_message_util_test.cc b/src/google/protobuf/util/delimited_message_util_test.cc new file mode 100644 index 00000000..0954773b --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util_test.cc @@ -0,0 +1,57 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include + +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace util { + +TEST(MessageTest, DelimitedMessages) { + stringstream stream; + + { + protobuf_unittest::TestAllTypes message1; + TestUtil::SetAllFields(&message1); + EXPECT_TRUE(SerializeDelimitedToOstream(&message1, &stream)); + + protobuf_unittest::TestPackedTypes message2; + TestUtil::SetPackedFields(&message2); + EXPECT_TRUE(SerializeDelimitedToOstream(&message2, &stream)); + } + + { + bool clean_eof; + io::IstreamInputStream zstream(&stream); + + protobuf_unittest::TestAllTypes message1; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message1, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectAllFieldsSet(message1); + + protobuf_unittest::TestPackedTypes message2; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectPackedFieldsSet(message2); + + clean_eof = false; + EXPECT_FALSE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_TRUE(clean_eof); + } +} + +} // namespace util +} // namespace protobuf +} // namespace google -- cgit v1.2.3 From 7008a88e2b4c6c13bfbc321ab218d3425546e109 Mon Sep 17 00:00:00 2001 From: Byron Yi Date: Thu, 16 Mar 2017 20:21:05 +0800 Subject: add LIBPROTOBUF_EXPORT to make msvc happy --- src/google/protobuf/util/delimited_message_util.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/google/protobuf/util') diff --git a/src/google/protobuf/util/delimited_message_util.h b/src/google/protobuf/util/delimited_message_util.h index 970f4e92..0ff91db1 100644 --- a/src/google/protobuf/util/delimited_message_util.h +++ b/src/google/protobuf/util/delimited_message_util.h @@ -30,9 +30,9 @@ namespace util { // then parse it. As a result, they may read past the end of the delimited // message. There is no way for them to push the extra data back into the // underlying source, so instead you must keep using the same stream object. -bool SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor); -bool SerializeDelimitedToOstream(const Message* message, ostream* output); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToOstream(const Message* message, ostream* output); // Read a single size-delimited message from the given stream. Delimited // format allows a single file or stream to contain multiple messages, @@ -46,18 +46,18 @@ bool SerializeDelimitedToOstream(const Message* message, ostream* output); // otherwise it will be set false. Note that these methods return false // on EOF, but they also return false on other errors, so |clean_eof| is // needed to distinguish a clean end from errors. -bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof); +bool LIBPROTOBUF_EXPORT ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof); -bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof); +bool LIBPROTOBUF_EXPORT ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof); // Write a single size-delimited message from the given stream. Delimited // format allows a single file or stream to contain multiple messages, // whereas normally writing multiple non-delimited messages to the same // stream would cause them to be merged. A delimited message is a varint // encoding the message size followed by a message of exactly that size. -bool SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output); -bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output); } // namespace util } // namespace protobuf -- cgit v1.2.3 From c415a1445b722295f12246f6081831338119c090 Mon Sep 17 00:00:00 2001 From: Byron Yi Date: Fri, 17 Mar 2017 10:00:59 +0800 Subject: fix several issues --- src/google/protobuf/util/delimited_message_util.cc | 14 +++++++------- src/google/protobuf/util/delimited_message_util.h | 12 ++++++------ src/google/protobuf/util/delimited_message_util_test.cc | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'src/google/protobuf/util') diff --git a/src/google/protobuf/util/delimited_message_util.cc b/src/google/protobuf/util/delimited_message_util.cc index 36e0118d..16378782 100644 --- a/src/google/protobuf/util/delimited_message_util.cc +++ b/src/google/protobuf/util/delimited_message_util.cc @@ -7,12 +7,12 @@ namespace google { namespace protobuf { namespace util { -bool SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor) { +bool SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor) { io::FileOutputStream output(file_descriptor); return SerializeDelimitedToZeroCopyStream(message, &output); } -bool SerializeDelimitedToOstream(const Message* message, ostream* output) { +bool SerializeDelimitedToOstream(const MessageLite& message, ostream* output) { { io::OstreamOutputStream zero_copy_output(output); if (!SerializeDelimitedToZeroCopyStream(message, &zero_copy_output)) return false; @@ -49,14 +49,14 @@ bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* i return true; } -bool SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output) { +bool SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output) { google::protobuf::io::CodedOutputStream coded_output(output); return SerializeDelimitedToCodedStream(message, &coded_output); } -bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output) { +bool SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output) { // Write the size. - int size = message->ByteSize(); + int size = message.ByteSize(); output->WriteVarint32(size); // Write the content. @@ -64,10 +64,10 @@ bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutput if (buffer != NULL) { // Optimization: The message fits in one buffer, so use the faster // direct-to-array serialization path. - message->SerializeWithCachedSizesToArray(buffer); + message.SerializeWithCachedSizesToArray(buffer); } else { // Slightly-slower path when the message is multiple buffers. - message->SerializeWithCachedSizes(output); + message.SerializeWithCachedSizes(output); if (output->HadError()) return false; } diff --git a/src/google/protobuf/util/delimited_message_util.h b/src/google/protobuf/util/delimited_message_util.h index 0ff91db1..302d4781 100644 --- a/src/google/protobuf/util/delimited_message_util.h +++ b/src/google/protobuf/util/delimited_message_util.h @@ -6,7 +6,7 @@ #include -#include +#include #include #include @@ -23,16 +23,16 @@ namespace util { // Note that if you want to *read* a delimited message from a file descriptor // or istream, you will need to construct an io::FileInputStream or // io::OstreamInputStream (implementations of io::ZeroCopyStream) and use the -// MessageLite method ParseDelimitedFromZeroCopyStream(). You must then +// utility function ParseDelimitedFromZeroCopyStream(). You must then // continue to use the same ZeroCopyInputStream to read all further data from // the stream until EOF. This is because these ZeroCopyInputStream // implementations are buffered: they read a big chunk of data at a time, // then parse it. As a result, they may read past the end of the delimited // message. There is no way for them to push the extra data back into the // underlying source, so instead you must keep using the same stream object. -bool LIBPROTOBUF_EXPORT SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor); -bool LIBPROTOBUF_EXPORT SerializeDelimitedToOstream(const Message* message, ostream* output); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToOstream(const MessageLite& message, ostream* output); // Read a single size-delimited message from the given stream. Delimited // format allows a single file or stream to contain multiple messages, @@ -55,9 +55,9 @@ bool LIBPROTOBUF_EXPORT ParseDelimitedFromCodedStream(MessageLite* message, io:: // whereas normally writing multiple non-delimited messages to the same // stream would cause them to be merged. A delimited message is a varint // encoding the message size followed by a message of exactly that size. -bool LIBPROTOBUF_EXPORT SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output); -bool LIBPROTOBUF_EXPORT SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output); +bool LIBPROTOBUF_EXPORT SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output); } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/delimited_message_util_test.cc b/src/google/protobuf/util/delimited_message_util_test.cc index 0954773b..a2e36a3a 100644 --- a/src/google/protobuf/util/delimited_message_util_test.cc +++ b/src/google/protobuf/util/delimited_message_util_test.cc @@ -20,11 +20,11 @@ TEST(MessageTest, DelimitedMessages) { { protobuf_unittest::TestAllTypes message1; TestUtil::SetAllFields(&message1); - EXPECT_TRUE(SerializeDelimitedToOstream(&message1, &stream)); + EXPECT_TRUE(SerializeDelimitedToOstream(message1, &stream)); protobuf_unittest::TestPackedTypes message2; TestUtil::SetPackedFields(&message2); - EXPECT_TRUE(SerializeDelimitedToOstream(&message2, &stream)); + EXPECT_TRUE(SerializeDelimitedToOstream(message2, &stream)); } { -- cgit v1.2.3 From e8e6aa28645029eeaa436c41487990b2ec8a3c93 Mon Sep 17 00:00:00 2001 From: Bairen Yi Date: Sat, 18 Mar 2017 00:32:43 +0800 Subject: Update delimited_message_util_test.cc --- src/google/protobuf/util/delimited_message_util_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/google/protobuf/util') diff --git a/src/google/protobuf/util/delimited_message_util_test.cc b/src/google/protobuf/util/delimited_message_util_test.cc index a2e36a3a..b261d43c 100644 --- a/src/google/protobuf/util/delimited_message_util_test.cc +++ b/src/google/protobuf/util/delimited_message_util_test.cc @@ -14,7 +14,7 @@ namespace google { namespace protobuf { namespace util { -TEST(MessageTest, DelimitedMessages) { +TEST(DelimitedMessageUtilTest, DelimitedMessages) { stringstream stream; { -- cgit v1.2.3