From 47b7d2c7cadf74ceec90fc5042232819cd0dd557 Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Wed, 3 Jan 2018 09:57:58 +0000 Subject: Add DiscardUnknownFields support for C# By default, unknown fields are preserved when parsing. To discard them, use a parser configured to do so: var parser = MyMessage.Parser.WithDiscardUnknownFields(true); --- .../Google.Protobuf.Test/UnknownFieldSetTest.cs | 48 ++++++++++ csharp/src/Google.Protobuf/CodedInputStream.cs | 5 + csharp/src/Google.Protobuf/MessageExtensions.cs | 102 ++++++++++++--------- csharp/src/Google.Protobuf/MessageParser.cs | 66 ++++++++++--- csharp/src/Google.Protobuf/UnknownFieldSet.cs | 8 +- 5 files changed, 173 insertions(+), 56 deletions(-) (limited to 'csharp') diff --git a/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs b/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs index 1edd6fba..ddf62321 100644 --- a/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs +++ b/csharp/src/Google.Protobuf.Test/UnknownFieldSetTest.cs @@ -31,6 +31,7 @@ #endregion using System; +using System.IO; using Google.Protobuf.TestProtos; using NUnit.Framework; @@ -124,5 +125,52 @@ namespace Google.Protobuf Assert.AreEqual(message.CalculateSize(), otherEmptyMessage.CalculateSize()); Assert.AreEqual(message.ToByteArray(), otherEmptyMessage.ToByteArray()); } + + [Test] + public void TestDiscardUnknownFields() + { + var message = SampleMessages.CreateFullTestAllTypes(); + var goldenEmptyMessage = new TestEmptyMessage(); + byte[] data = message.ToByteArray(); + int fullSize = message.CalculateSize(); + + Action assertEmpty = msg => + { + Assert.AreEqual(0, msg.CalculateSize()); + Assert.AreEqual(goldenEmptyMessage, msg); + }; + + Action assertFull = msg => Assert.AreEqual(fullSize, msg.CalculateSize()); + + // Test the behavior of the parsers with and without discarding, both generic and non-generic. + MessageParser retainingParser1 = TestEmptyMessage.Parser; + MessageParser retainingParser2 = retainingParser1; + MessageParser discardingParser1 = retainingParser1.WithDiscardUnknownFields(true); + MessageParser discardingParser2 = retainingParser2.WithDiscardUnknownFields(true); + + // Test parse from byte[] + assertFull(retainingParser1.ParseFrom(data)); + assertFull(retainingParser2.ParseFrom(data)); + assertEmpty(discardingParser1.ParseFrom(data)); + assertEmpty(discardingParser2.ParseFrom(data)); + + // Test parse from byte[] with offset + assertFull(retainingParser1.ParseFrom(data, 0, data.Length)); + assertFull(retainingParser2.ParseFrom(data, 0, data.Length)); + assertEmpty(discardingParser1.ParseFrom(data, 0, data.Length)); + assertEmpty(discardingParser2.ParseFrom(data, 0, data.Length)); + + // Test parse from CodedInputStream + assertFull(retainingParser1.ParseFrom(new CodedInputStream(data))); + assertFull(retainingParser2.ParseFrom(new CodedInputStream(data))); + assertEmpty(discardingParser1.ParseFrom(new CodedInputStream(data))); + assertEmpty(discardingParser2.ParseFrom(new CodedInputStream(data))); + + // Test parse from Stream + assertFull(retainingParser1.ParseFrom(new MemoryStream(data))); + assertFull(retainingParser2.ParseFrom(new MemoryStream(data))); + assertEmpty(discardingParser1.ParseFrom(new MemoryStream(data))); + assertEmpty(discardingParser2.ParseFrom(new MemoryStream(data))); + } } } diff --git a/csharp/src/Google.Protobuf/CodedInputStream.cs b/csharp/src/Google.Protobuf/CodedInputStream.cs index 24d436c0..6bee238f 100644 --- a/csharp/src/Google.Protobuf/CodedInputStream.cs +++ b/csharp/src/Google.Protobuf/CodedInputStream.cs @@ -267,6 +267,11 @@ namespace Google.Protobuf /// public int RecursionLimit { get { return recursionLimit; } } + /// + /// Internal-only property; when set to true, unknown fields will be discarded while parsing. + /// + internal bool DiscardUnknownFields { get; set; } + /// /// Disposes of this instance, potentially closing any underlying stream. /// diff --git a/csharp/src/Google.Protobuf/MessageExtensions.cs b/csharp/src/Google.Protobuf/MessageExtensions.cs index 9dbc49d6..62181eb9 100644 --- a/csharp/src/Google.Protobuf/MessageExtensions.cs +++ b/csharp/src/Google.Protobuf/MessageExtensions.cs @@ -44,14 +44,8 @@ namespace Google.Protobuf /// /// The message to merge the data into. /// The data to merge, which must be protobuf-encoded binary data. - public static void MergeFrom(this IMessage message, byte[] data) - { - ProtoPreconditions.CheckNotNull(message, "message"); - ProtoPreconditions.CheckNotNull(data, "data"); - CodedInputStream input = new CodedInputStream(data); - message.MergeFrom(input); - input.CheckReadEndOfStreamTag(); - } + public static void MergeFrom(this IMessage message, byte[] data) => + MergeFrom(message, data, false); /// /// Merges data from the given byte array slice into an existing message. @@ -60,42 +54,24 @@ namespace Google.Protobuf /// The data containing the slice to merge, which must be protobuf-encoded binary data. /// The offset of the slice to merge. /// The length of the slice to merge. - public static void MergeFrom(this IMessage message, byte[] data, int offset, int length) - { - ProtoPreconditions.CheckNotNull(message, "message"); - ProtoPreconditions.CheckNotNull(data, "data"); - CodedInputStream input = new CodedInputStream(data, offset, length); - message.MergeFrom(input); - input.CheckReadEndOfStreamTag(); - } + public static void MergeFrom(this IMessage message, byte[] data, int offset, int length) => + MergeFrom(message, data, offset, length, false); /// /// Merges data from the given byte string into an existing message. /// /// The message to merge the data into. /// The data to merge, which must be protobuf-encoded binary data. - public static void MergeFrom(this IMessage message, ByteString data) - { - ProtoPreconditions.CheckNotNull(message, "message"); - ProtoPreconditions.CheckNotNull(data, "data"); - CodedInputStream input = data.CreateCodedInput(); - message.MergeFrom(input); - input.CheckReadEndOfStreamTag(); - } + public static void MergeFrom(this IMessage message, ByteString data) => + MergeFrom(message, data, false); /// /// Merges data from the given stream into an existing message. /// /// The message to merge the data into. /// Stream containing the data to merge, which must be protobuf-encoded binary data. - public static void MergeFrom(this IMessage message, Stream input) - { - ProtoPreconditions.CheckNotNull(message, "message"); - ProtoPreconditions.CheckNotNull(input, "input"); - CodedInputStream codedInput = new CodedInputStream(input); - message.MergeFrom(codedInput); - codedInput.CheckReadEndOfStreamTag(); - } + public static void MergeFrom(this IMessage message, Stream input) => + MergeFrom(message, input, false); /// /// Merges length-delimited data from the given stream into an existing message. @@ -106,14 +82,8 @@ namespace Google.Protobuf /// /// The message to merge the data into. /// Stream containing the data to merge, which must be protobuf-encoded binary data. - public static void MergeDelimitedFrom(this IMessage message, Stream input) - { - ProtoPreconditions.CheckNotNull(message, "message"); - ProtoPreconditions.CheckNotNull(input, "input"); - int size = (int) CodedInputStream.ReadRawVarint32(input); - Stream limitedStream = new LimitedInputStream(input, size); - message.MergeFrom(limitedStream); - } + public static void MergeDelimitedFrom(this IMessage message, Stream input) => + MergeDelimitedFrom(message, input, false); /// /// Converts the given message into a byte array in protobuf encoding. @@ -168,6 +138,56 @@ namespace Google.Protobuf { ProtoPreconditions.CheckNotNull(message, "message"); return ByteString.AttachBytes(message.ToByteArray()); - } + } + + // Implementations allowing unknown fields to be discarded. + internal static void MergeFrom(this IMessage message, byte[] data, bool discardUnknownFields) + { + ProtoPreconditions.CheckNotNull(message, "message"); + ProtoPreconditions.CheckNotNull(data, "data"); + CodedInputStream input = new CodedInputStream(data); + input.DiscardUnknownFields = discardUnknownFields; + message.MergeFrom(input); + input.CheckReadEndOfStreamTag(); + } + + internal static void MergeFrom(this IMessage message, byte[] data, int offset, int length, bool discardUnknownFields) + { + ProtoPreconditions.CheckNotNull(message, "message"); + ProtoPreconditions.CheckNotNull(data, "data"); + CodedInputStream input = new CodedInputStream(data, offset, length); + input.DiscardUnknownFields = discardUnknownFields; + message.MergeFrom(input); + input.CheckReadEndOfStreamTag(); + } + + internal static void MergeFrom(this IMessage message, ByteString data, bool discardUnknownFields) + { + ProtoPreconditions.CheckNotNull(message, "message"); + ProtoPreconditions.CheckNotNull(data, "data"); + CodedInputStream input = data.CreateCodedInput(); + input.DiscardUnknownFields = discardUnknownFields; + message.MergeFrom(input); + input.CheckReadEndOfStreamTag(); + } + + internal static void MergeFrom(this IMessage message, Stream input, bool discardUnknownFields) + { + ProtoPreconditions.CheckNotNull(message, "message"); + ProtoPreconditions.CheckNotNull(input, "input"); + CodedInputStream codedInput = new CodedInputStream(input); + codedInput.DiscardUnknownFields = discardUnknownFields; + message.MergeFrom(codedInput); + codedInput.CheckReadEndOfStreamTag(); + } + + internal static void MergeDelimitedFrom(this IMessage message, Stream input, bool discardUnknownFields) + { + ProtoPreconditions.CheckNotNull(message, "message"); + ProtoPreconditions.CheckNotNull(input, "input"); + int size = (int) CodedInputStream.ReadRawVarint32(input); + Stream limitedStream = new LimitedInputStream(input, size); + MergeFrom(message, limitedStream, discardUnknownFields); + } } } diff --git a/csharp/src/Google.Protobuf/MessageParser.cs b/csharp/src/Google.Protobuf/MessageParser.cs index 66d44135..4d35554a 100644 --- a/csharp/src/Google.Protobuf/MessageParser.cs +++ b/csharp/src/Google.Protobuf/MessageParser.cs @@ -42,10 +42,13 @@ namespace Google.Protobuf public class MessageParser { private Func factory; + // TODO: When we use a C# 7.1 compiler, make this private protected. + internal bool DiscardUnknownFields { get; } - internal MessageParser(Func factory) + internal MessageParser(Func factory, bool discardUnknownFields) { this.factory = factory; + DiscardUnknownFields = discardUnknownFields; } /// @@ -65,7 +68,7 @@ namespace Google.Protobuf public IMessage ParseFrom(byte[] data) { IMessage message = factory(); - message.MergeFrom(data); + message.MergeFrom(data, DiscardUnknownFields); return message; } @@ -79,7 +82,7 @@ namespace Google.Protobuf public IMessage ParseFrom(byte[] data, int offset, int length) { IMessage message = factory(); - message.MergeFrom(data, offset, length); + message.MergeFrom(data, offset, length, DiscardUnknownFields); return message; } @@ -91,7 +94,7 @@ namespace Google.Protobuf public IMessage ParseFrom(ByteString data) { IMessage message = factory(); - message.MergeFrom(data); + message.MergeFrom(data, DiscardUnknownFields); return message; } @@ -103,7 +106,7 @@ namespace Google.Protobuf public IMessage ParseFrom(Stream input) { IMessage message = factory(); - message.MergeFrom(input); + message.MergeFrom(input, DiscardUnknownFields); return message; } @@ -119,7 +122,7 @@ namespace Google.Protobuf public IMessage ParseDelimitedFrom(Stream input) { IMessage message = factory(); - message.MergeDelimitedFrom(input); + message.MergeDelimitedFrom(input, DiscardUnknownFields); return message; } @@ -131,7 +134,7 @@ namespace Google.Protobuf public IMessage ParseFrom(CodedInputStream input) { IMessage message = factory(); - message.MergeFrom(input); + MergeFrom(message, input); return message; } @@ -148,6 +151,29 @@ namespace Google.Protobuf JsonParser.Default.Merge(message, json); return message; } + + // TODO: When we're using a C# 7.1 compiler, make this private protected. + internal void MergeFrom(IMessage message, CodedInputStream codedInput) + { + bool originalDiscard = codedInput.DiscardUnknownFields; + try + { + codedInput.DiscardUnknownFields = DiscardUnknownFields; + message.MergeFrom(codedInput); + } + finally + { + codedInput.DiscardUnknownFields = originalDiscard; + } + } + + /// + /// Creates a new message parser which optionally discards unknown fields when parsing. + /// + /// Whether or not to discard unknown fields when parsing. + /// A newly configured message parser. + public MessageParser WithDiscardUnknownFields(bool discardUnknownFields) => + new MessageParser(factory, discardUnknownFields); } /// @@ -182,7 +208,11 @@ namespace Google.Protobuf /// to require a parameterless constructor: delegates are significantly faster to execute. /// /// Function to invoke when a new, empty message is required. - public MessageParser(Func factory) : base(() => factory()) + public MessageParser(Func factory) : this(factory, false) + { + } + + internal MessageParser(Func factory, bool discardUnknownFields) : base(() => factory(), discardUnknownFields) { this.factory = factory; } @@ -204,7 +234,7 @@ namespace Google.Protobuf public new T ParseFrom(byte[] data) { T message = factory(); - message.MergeFrom(data); + message.MergeFrom(data, DiscardUnknownFields); return message; } @@ -218,7 +248,7 @@ namespace Google.Protobuf public new T ParseFrom(byte[] data, int offset, int length) { T message = factory(); - message.MergeFrom(data, offset, length); + message.MergeFrom(data, offset, length, DiscardUnknownFields); return message; } @@ -230,7 +260,7 @@ namespace Google.Protobuf public new T ParseFrom(ByteString data) { T message = factory(); - message.MergeFrom(data); + message.MergeFrom(data, DiscardUnknownFields); return message; } @@ -242,7 +272,7 @@ namespace Google.Protobuf public new T ParseFrom(Stream input) { T message = factory(); - message.MergeFrom(input); + message.MergeFrom(input, DiscardUnknownFields); return message; } @@ -258,7 +288,7 @@ namespace Google.Protobuf public new T ParseDelimitedFrom(Stream input) { T message = factory(); - message.MergeDelimitedFrom(input); + message.MergeDelimitedFrom(input, DiscardUnknownFields); return message; } @@ -270,7 +300,7 @@ namespace Google.Protobuf public new T ParseFrom(CodedInputStream input) { T message = factory(); - message.MergeFrom(input); + MergeFrom(message, input); return message; } @@ -287,5 +317,13 @@ namespace Google.Protobuf JsonParser.Default.Merge(message, json); return message; } + + /// + /// Creates a new message parser which optionally discards unknown fields when parsing. + /// + /// Whether or not to discard unknown fields when parsing. + /// A newly configured message parser. + public new MessageParser WithDiscardUnknownFields(bool discardUnknownFields) => + new MessageParser(factory, discardUnknownFields); } } diff --git a/csharp/src/Google.Protobuf/UnknownFieldSet.cs b/csharp/src/Google.Protobuf/UnknownFieldSet.cs index b43f4774..6404c3c0 100644 --- a/csharp/src/Google.Protobuf/UnknownFieldSet.cs +++ b/csharp/src/Google.Protobuf/UnknownFieldSet.cs @@ -230,7 +230,8 @@ namespace Google.Protobuf /// /// Create a new UnknownFieldSet if unknownFields is null. /// Parse a single field from and merge it - /// into unknownFields. + /// into unknownFields. If is configured to discard unknown fields, + /// will be returned as-is and the field will be skipped. /// /// The UnknownFieldSet which need to be merged /// The coded input stream containing the field @@ -238,6 +239,11 @@ namespace Google.Protobuf public static UnknownFieldSet MergeFieldFrom(UnknownFieldSet unknownFields, CodedInputStream input) { + if (input.DiscardUnknownFields) + { + input.SkipLastField(); + return unknownFields; + } if (unknownFields == null) { unknownFields = new UnknownFieldSet(); -- cgit v1.2.3