diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Makefile.am | 3 | ||||
| -rw-r--r-- | src/google/protobuf/util/delimited_message_util.cc | 79 | ||||
| -rw-r--r-- | src/google/protobuf/util/delimited_message_util.h | 66 | ||||
| -rw-r--r-- | src/google/protobuf/util/delimited_message_util_test.cc | 57 |
4 files changed, 205 insertions, 0 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 6a91044d..bcd81576 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -169,6 +169,7 @@ nobase_include_HEADERS = \ google/protobuf/compiler/python/python_generator.h \ google/protobuf/compiler/ruby/ruby_generator.h \ google/protobuf/util/type_resolver.h \ + google/protobuf/util/delimited_message_util.h \ google/protobuf/util/field_comparator.h \ google/protobuf/util/field_mask_util.h \ google/protobuf/util/json_util.h \ @@ -267,6 +268,7 @@ libprotobuf_la_SOURCES = \ google/protobuf/io/zero_copy_stream_impl.cc \ google/protobuf/compiler/importer.cc \ google/protobuf/compiler/parser.cc \ + google/protobuf/util/delimited_message_util.cc \ google/protobuf/util/field_comparator.cc \ google/protobuf/util/field_mask_util.cc \ google/protobuf/util/internal/constants.h \ @@ -800,6 +802,7 @@ protobuf_test_SOURCES = \ google/protobuf/compiler/ruby/ruby_generator_unittest.cc \ google/protobuf/compiler/csharp/csharp_bootstrap_unittest.cc \ google/protobuf/compiler/csharp/csharp_generator_unittest.cc \ + google/protobuf/util/delimited_message_util_test.cc \ google/protobuf/util/field_comparator_test.cc \ google/protobuf/util/field_mask_util_test.cc \ google/protobuf/util/internal/default_value_objectwriter_test.cc \ diff --git a/src/google/protobuf/util/delimited_message_util.cc b/src/google/protobuf/util/delimited_message_util.cc new file mode 100644 index 00000000..36e0118d --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.cc @@ -0,0 +1,79 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include <google/protobuf/util/delimited_message_util.h> + +namespace google { +namespace protobuf { +namespace util { + +bool SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor) { + io::FileOutputStream output(file_descriptor); + return SerializeDelimitedToZeroCopyStream(message, &output); +} + +bool SerializeDelimitedToOstream(const Message* message, ostream* output) { + { + io::OstreamOutputStream zero_copy_output(output); + if (!SerializeDelimitedToZeroCopyStream(message, &zero_copy_output)) return false; + } + return output->good(); +} + +bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof) { + google::protobuf::io::CodedInputStream coded_input(input); + return ParseDelimitedFromCodedStream(message, &coded_input, clean_eof); +} + +bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof) { + if (clean_eof != NULL) *clean_eof = false; + int start = input->CurrentPosition(); + + // Read the size. + uint32 size; + if (!input->ReadVarint32(&size)) { + if (clean_eof != NULL) *clean_eof = input->CurrentPosition() == start; + return false; + } + + // Tell the stream not to read beyond that size. + google::protobuf::io::CodedInputStream::Limit limit = input->PushLimit(size); + + // Parse the message. + if (!message->MergeFromCodedStream(input)) return false; + if (!input->ConsumedEntireMessage()) return false; + + // Release the limit. + input->PopLimit(limit); + + return true; +} + +bool SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output) { + google::protobuf::io::CodedOutputStream coded_output(output); + return SerializeDelimitedToCodedStream(message, &coded_output); +} + +bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output) { + // Write the size. + int size = message->ByteSize(); + output->WriteVarint32(size); + + // Write the content. + uint8* buffer = output->GetDirectBufferForNBytesAndAdvance(size); + if (buffer != NULL) { + // Optimization: The message fits in one buffer, so use the faster + // direct-to-array serialization path. + message->SerializeWithCachedSizesToArray(buffer); + } else { + // Slightly-slower path when the message is multiple buffers. + message->SerializeWithCachedSizes(output); + if (output->HadError()) return false; + } + + return true; +} + +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/delimited_message_util.h b/src/google/protobuf/util/delimited_message_util.h new file mode 100644 index 00000000..970f4e92 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.h @@ -0,0 +1,66 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#ifndef GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ +#define GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ + +#include <ostream> + +#include <google/protobuf/message.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/io/zero_copy_stream_impl.h> + +namespace google { +namespace protobuf { +namespace util { + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +// +// Note that if you want to *read* a delimited message from a file descriptor +// or istream, you will need to construct an io::FileInputStream or +// io::OstreamInputStream (implementations of io::ZeroCopyStream) and use the +// MessageLite method ParseDelimitedFromZeroCopyStream(). You must then +// continue to use the same ZeroCopyInputStream to read all further data from +// the stream until EOF. This is because these ZeroCopyInputStream +// implementations are buffered: they read a big chunk of data at a time, +// then parse it. As a result, they may read past the end of the delimited +// message. There is no way for them to push the extra data back into the +// underlying source, so instead you must keep using the same stream object. +bool SerializeDelimitedToFileDescriptor(const Message* message, int file_descriptor); + +bool SerializeDelimitedToOstream(const Message* message, ostream* output); + +// Read a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally parsing consumes the entire input. A delimited message +// is a varint encoding the message size followed by a message of exactly +// that size. +// +// If |clean_eof| is not NULL, then it will be set to indicate whether the +// stream ended cleanly. That is, if the stream ends without this method +// having read any data at all from it, then *clean_eof will be set true, +// otherwise it will be set false. Note that these methods return false +// on EOF, but they also return false on other errors, so |clean_eof| is +// needed to distinguish a clean end from errors. +bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof); + +bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof); + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +bool SerializeDelimitedToZeroCopyStream(const MessageLite* message, io::ZeroCopyOutputStream* output); + +bool SerializeDelimitedToCodedStream(const MessageLite* message, io::CodedOutputStream* output); + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ diff --git a/src/google/protobuf/util/delimited_message_util_test.cc b/src/google/protobuf/util/delimited_message_util_test.cc new file mode 100644 index 00000000..0954773b --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util_test.cc @@ -0,0 +1,57 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include <google/protobuf/util/delimited_message_util.h> + +#include <sstream> + +#include <google/protobuf/test_util.h> +#include <google/protobuf/unittest.pb.h> +#include <google/protobuf/testing/googletest.h> +#include <gtest/gtest.h> + +namespace google { +namespace protobuf { +namespace util { + +TEST(MessageTest, DelimitedMessages) { + stringstream stream; + + { + protobuf_unittest::TestAllTypes message1; + TestUtil::SetAllFields(&message1); + EXPECT_TRUE(SerializeDelimitedToOstream(&message1, &stream)); + + protobuf_unittest::TestPackedTypes message2; + TestUtil::SetPackedFields(&message2); + EXPECT_TRUE(SerializeDelimitedToOstream(&message2, &stream)); + } + + { + bool clean_eof; + io::IstreamInputStream zstream(&stream); + + protobuf_unittest::TestAllTypes message1; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message1, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectAllFieldsSet(message1); + + protobuf_unittest::TestPackedTypes message2; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectPackedFieldsSet(message2); + + clean_eof = false; + EXPECT_FALSE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_TRUE(clean_eof); + } +} + +} // namespace util +} // namespace protobuf +} // namespace google |