diff options
Diffstat (limited to 'csharp')
| -rw-r--r-- | csharp/Google.Protobuf.Tools.nuspec | 2 | ||||
| -rwxr-xr-x[-rw-r--r--] | csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs | 83 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf.Test/JsonParserTest.cs | 16 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/CodedInputStream.cs | 6 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs | 47 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/Google.Protobuf.csproj | 2 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/JsonParser.cs | 5 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/Reflection/OneofAccessor.cs | 2 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs | 156 | ||||
| -rw-r--r-- | csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs | 4 |
10 files changed, 274 insertions, 49 deletions
diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index 62eb3b58..4294949f 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -5,7 +5,7 @@ <title>Google Protocol Buffers tools</title> <summary>Tools for Protocol Buffers - Google's data interchange format.</summary> <description>See project site for more info.</description> - <version>3.5.1</version> + <version>3.5.2</version> <authors>Google Inc.</authors> <owners>protobuf-packages</owners> <licenseUrl>https://github.com/google/protobuf/blob/master/LICENSE</licenseUrl> diff --git a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs index e7c6b805..8795fa65 100644..100755 --- a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs +++ b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs @@ -615,5 +615,88 @@ namespace Google.Protobuf var stream = new CodedInputStream(new byte[10]);
stream.Dispose();
}
+
+ [Test]
+ public void TestParseMessagesCloseTo2G()
+ {
+ byte[] serializedMessage = GenerateBigSerializedMessage();
+ // How many of these big messages do we need to take us near our 2GB limit?
+ int count = Int32.MaxValue / serializedMessage.Length;
+ // Now make a MemoryStream that will fake a near-2GB stream of messages by returning
+ // our big serialized message 'count' times.
+ using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count))
+ {
+ Assert.DoesNotThrow(()=>TestAllTypes.Parser.ParseFrom(stream));
+ }
+ }
+
+ [Test]
+ public void TestParseMessagesOver2G()
+ {
+ byte[] serializedMessage = GenerateBigSerializedMessage();
+ // How many of these big messages do we need to take us near our 2GB limit?
+ int count = Int32.MaxValue / serializedMessage.Length;
+ // Now add one to take us over the 2GB limit
+ count++;
+ // Now make a MemoryStream that will fake a near-2GB stream of messages by returning
+ // our big serialized message 'count' times.
+ using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count))
+ {
+ Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(stream),
+ "Protocol message was too large. May be malicious. " +
+ "Use CodedInputStream.SetSizeLimit() to increase the size limit.");
+ }
+ }
+
+ /// <returns>A serialized big message</returns>
+ private static byte[] GenerateBigSerializedMessage()
+ {
+ byte[] value = new byte[16 * 1024 * 1024];
+ TestAllTypes message = SampleMessages.CreateFullTestAllTypes();
+ message.SingleBytes = ByteString.CopyFrom(value);
+ return message.ToByteArray();
+ }
+
+ /// <summary>
+ /// A MemoryStream that repeats a byte arrays' content a number of times.
+ /// Simulates really large input without consuming loads of memory. Used above
+ /// to test the parsing behavior when the input size exceeds 2GB or close to it.
+ /// </summary>
+ private class RepeatingMemoryStream: MemoryStream
+ {
+ private readonly byte[] bytes;
+ private readonly int maxIterations;
+ private int index = 0;
+
+ public RepeatingMemoryStream(byte[] bytes, int maxIterations)
+ {
+ this.bytes = bytes;
+ this.maxIterations = maxIterations;
+ }
+
+ public override int Read(byte[] buffer, int offset, int count)
+ {
+ if (bytes.Length == 0)
+ {
+ return 0;
+ }
+ int numBytesCopiedTotal = 0;
+ while (numBytesCopiedTotal < count && index < maxIterations)
+ {
+ int numBytesToCopy = Math.Min(bytes.Length - (int)Position, count);
+ Array.Copy(bytes, (int)Position, buffer, offset, numBytesToCopy);
+ numBytesCopiedTotal += numBytesToCopy;
+ offset += numBytesToCopy;
+ count -= numBytesCopiedTotal;
+ Position += numBytesToCopy;
+ if (Position >= bytes.Length)
+ {
+ Position = 0;
+ index++;
+ }
+ }
+ return numBytesCopiedTotal;
+ }
+ }
}
}
\ No newline at end of file diff --git a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs index 329ae9be..a6cf04ab 100644 --- a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs @@ -696,6 +696,22 @@ namespace Google.Protobuf } [Test] + public void Value_List_WithNullElement() + { + var expected = Value.ForList(Value.ForString("x"), Value.ForNull(), Value.ForString("y")); + var actual = Value.Parser.ParseJson("[\"x\", null, \"y\"]"); + Assert.AreEqual(expected, actual); + } + + [Test] + public void StructValue_NullElement() + { + var expected = Value.ForStruct(new Struct { Fields = { { "x", Value.ForNull() } } }); + var actual = Value.Parser.ParseJson("{ \"x\": null }"); + Assert.AreEqual(expected, actual); + } + + [Test] public void ParseListValue() { Assert.AreEqual(new ListValue { Values = { Value.ForNumber(1), Value.ForString("x") } }, ListValue.Parser.ParseJson("[1, \"x\"]")); diff --git a/csharp/src/Google.Protobuf/CodedInputStream.cs b/csharp/src/Google.Protobuf/CodedInputStream.cs index 6bee238f..0a829545 100644 --- a/csharp/src/Google.Protobuf/CodedInputStream.cs +++ b/csharp/src/Google.Protobuf/CodedInputStream.cs @@ -94,7 +94,7 @@ namespace Google.Protobuf private bool hasNextTag = false;
internal const int DefaultRecursionLimit = 64;
- internal const int DefaultSizeLimit = 64 << 20; // 64MB
+ internal const int DefaultSizeLimit = Int32.MaxValue;
internal const int BufferSize = 4096;
/// <summary>
@@ -248,7 +248,7 @@ namespace Google.Protobuf /// <remarks>
/// This limit is applied when reading from the underlying stream, as a sanity check. It is
/// not applied when reading from a byte array data source without an underlying stream.
- /// The default value is 64MB.
+ /// The default value is Int32.MaxValue.
/// </remarks>
/// <value>
/// The size limit.
@@ -1058,7 +1058,7 @@ namespace Google.Protobuf RecomputeBufferSizeAfterLimit();
int totalBytesRead =
totalBytesRetired + bufferSize + bufferSizeAfterLimit;
- if (totalBytesRead > sizeLimit || totalBytesRead < 0)
+ if (totalBytesRead < 0 || totalBytesRead > sizeLimit)
{
throw InvalidProtocolBufferException.SizeLimitExceeded();
}
diff --git a/csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs b/csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs new file mode 100644 index 00000000..7b946cb6 --- /dev/null +++ b/csharp/src/Google.Protobuf/Compatibility/MethodInfoExtensions.cs @@ -0,0 +1,47 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2017 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NET35 +using System; +using System.Reflection; + +namespace Google.Protobuf.Compatibility +{ + // .NET Core (at least netstandard1.0) doesn't have Delegate.CreateDelegate, and .NET 3.5 doesn't have + // MethodInfo.CreateDelegate. Proxy from one to the other on .NET 3.5... + internal static class MethodInfoExtensions + { + internal static Delegate CreateDelegate(this MethodInfo method, Type type) => + Delegate.CreateDelegate(type, method); + } +} +#endif diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index 4df0bda6..6e4e1e2b 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -4,7 +4,7 @@ <Description>C# runtime library for Protocol Buffers - Google's data interchange format.</Description> <Copyright>Copyright 2015, Google Inc.</Copyright> <AssemblyTitle>Google Protocol Buffers</AssemblyTitle> - <VersionPrefix>3.5.1</VersionPrefix> + <VersionPrefix>3.5.2</VersionPrefix> <Authors>Google Inc.</Authors> <TargetFrameworks>netstandard1.0;net45</TargetFrameworks> <GenerateDocumentationFile>true</GenerateDocumentationFile> diff --git a/csharp/src/Google.Protobuf/JsonParser.cs b/csharp/src/Google.Protobuf/JsonParser.cs index 3621b0c0..284bce93 100644 --- a/csharp/src/Google.Protobuf/JsonParser.cs +++ b/csharp/src/Google.Protobuf/JsonParser.cs @@ -264,11 +264,12 @@ namespace Google.Protobuf return; } tokenizer.PushBack(token); - if (token.Type == JsonToken.TokenType.Null) + object value = ParseSingleValue(field, tokenizer); + if (value == null) { throw new InvalidProtocolBufferException("Repeated field elements cannot be null"); } - list.Add(ParseSingleValue(field, tokenizer)); + list.Add(value); } } diff --git a/csharp/src/Google.Protobuf/Reflection/OneofAccessor.cs b/csharp/src/Google.Protobuf/Reflection/OneofAccessor.cs index 8714ab18..97596218 100644 --- a/csharp/src/Google.Protobuf/Reflection/OneofAccessor.cs +++ b/csharp/src/Google.Protobuf/Reflection/OneofAccessor.cs @@ -52,7 +52,7 @@ namespace Google.Protobuf.Reflection throw new ArgumentException("Cannot read from property"); } this.descriptor = descriptor; - caseDelegate = ReflectionUtil.CreateFuncIMessageT<int>(caseProperty.GetGetMethod()); + caseDelegate = ReflectionUtil.CreateFuncIMessageInt32(caseProperty.GetGetMethod()); this.descriptor = descriptor; clearDelegate = ReflectionUtil.CreateActionIMessage(clearMethod); diff --git a/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs b/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs index df820ca3..1afaa90c 100644 --- a/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs +++ b/csharp/src/Google.Protobuf/Reflection/ReflectionUtil.cs @@ -30,11 +30,14 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endregion +using Google.Protobuf.Compatibility; using System; -using System.Collections.Generic; -using System.Linq.Expressions; using System.Reflection; +#if NET35 +using Google.Protobuf.Compatibility; +#endif + namespace Google.Protobuf.Reflection { /// <summary> @@ -53,55 +56,130 @@ namespace Google.Protobuf.Reflection internal static readonly Type[] EmptyTypes = new Type[0]; /// <summary> - /// Creates a delegate which will cast the argument to the appropriate method target type, + /// Creates a delegate which will cast the argument to the type that declares the method, /// call the method on it, then convert the result to object. /// </summary> - internal static Func<IMessage, object> CreateFuncIMessageObject(MethodInfo method) - { - ParameterExpression parameter = Expression.Parameter(typeof(IMessage), "p"); - Expression downcast = Expression.Convert(parameter, method.DeclaringType); - Expression call = Expression.Call(downcast, method); - Expression upcast = Expression.Convert(call, typeof(object)); - return Expression.Lambda<Func<IMessage, object>>(upcast, parameter).Compile(); - } + /// <param name="method">The method to create a delegate for, which must be declared in an IMessage + /// implementation.</param> + internal static Func<IMessage, object> CreateFuncIMessageObject(MethodInfo method) => + GetReflectionHelper(method.DeclaringType, method.ReturnType).CreateFuncIMessageObject(method); /// <summary> - /// Creates a delegate which will cast the argument to the appropriate method target type, - /// call the method on it, then convert the result to the specified type. + /// Creates a delegate which will cast the argument to the type that declares the method, + /// call the method on it, then convert the result to the specified type. The method is expected + /// to actually return an enum (because of where we're calling it - for oneof cases). Sometimes that + /// means we need some extra work to perform conversions. /// </summary> - internal static Func<IMessage, T> CreateFuncIMessageT<T>(MethodInfo method) - { - ParameterExpression parameter = Expression.Parameter(typeof(IMessage), "p"); - Expression downcast = Expression.Convert(parameter, method.DeclaringType); - Expression call = Expression.Call(downcast, method); - Expression upcast = Expression.Convert(call, typeof(T)); - return Expression.Lambda<Func<IMessage, T>>(upcast, parameter).Compile(); - } + /// <param name="method">The method to create a delegate for, which must be declared in an IMessage + /// implementation.</param> + internal static Func<IMessage, int> CreateFuncIMessageInt32(MethodInfo method) => + GetReflectionHelper(method.DeclaringType, method.ReturnType).CreateFuncIMessageInt32(method); /// <summary> /// Creates a delegate which will execute the given method after casting the first argument to - /// the target type of the method, and the second argument to the first parameter type of the method. + /// the type that declares the method, and the second argument to the first parameter type of the method. /// </summary> - internal static Action<IMessage, object> CreateActionIMessageObject(MethodInfo method) - { - ParameterExpression targetParameter = Expression.Parameter(typeof(IMessage), "target"); - ParameterExpression argParameter = Expression.Parameter(typeof(object), "arg"); - Expression castTarget = Expression.Convert(targetParameter, method.DeclaringType); - Expression castArgument = Expression.Convert(argParameter, method.GetParameters()[0].ParameterType); - Expression call = Expression.Call(castTarget, method, castArgument); - return Expression.Lambda<Action<IMessage, object>>(call, targetParameter, argParameter).Compile(); - } + /// <param name="method">The method to create a delegate for, which must be declared in an IMessage + /// implementation.</param> + internal static Action<IMessage, object> CreateActionIMessageObject(MethodInfo method) => + GetReflectionHelper(method.DeclaringType, method.GetParameters()[0].ParameterType).CreateActionIMessageObject(method); /// <summary> /// Creates a delegate which will execute the given method after casting the first argument to - /// the target type of the method. + /// type that declares the method. /// </summary> - internal static Action<IMessage> CreateActionIMessage(MethodInfo method) + /// <param name="method">The method to create a delegate for, which must be declared in an IMessage + /// implementation.</param> + internal static Action<IMessage> CreateActionIMessage(MethodInfo method) => + GetReflectionHelper(method.DeclaringType, typeof(object)).CreateActionIMessage(method); + + /// <summary> + /// Creates a reflection helper for the given type arguments. Currently these are created on demand + /// rather than cached; this will be "busy" when initially loading a message's descriptor, but after that + /// they can be garbage collected. We could cache them by type if that proves to be important, but creating + /// an object is pretty cheap. + /// </summary> + private static IReflectionHelper GetReflectionHelper(Type t1, Type t2) => + (IReflectionHelper) Activator.CreateInstance(typeof(ReflectionHelper<,>).MakeGenericType(t1, t2)); + + // Non-generic interface allowing us to use an instance of ReflectionHelper<T1, T2> without statically + // knowing the types involved. + private interface IReflectionHelper + { + Func<IMessage, int> CreateFuncIMessageInt32(MethodInfo method); + Action<IMessage> CreateActionIMessage(MethodInfo method); + Func<IMessage, object> CreateFuncIMessageObject(MethodInfo method); + Action<IMessage, object> CreateActionIMessageObject(MethodInfo method); + } + + private class ReflectionHelper<T1, T2> : IReflectionHelper { - ParameterExpression targetParameter = Expression.Parameter(typeof(IMessage), "target"); - Expression castTarget = Expression.Convert(targetParameter, method.DeclaringType); - Expression call = Expression.Call(castTarget, method); - return Expression.Lambda<Action<IMessage>>(call, targetParameter).Compile(); - } + + public Func<IMessage, int> CreateFuncIMessageInt32(MethodInfo method) + { + // On pleasant runtimes, we can create a Func<int> from a method returning + // an enum based on an int. That's the fast path. + if (CanConvertEnumFuncToInt32Func) + { + var del = (Func<T1, int>) method.CreateDelegate(typeof(Func<T1, int>)); + return message => del((T1) message); + } + else + { + // On some runtimes (e.g. old Mono) the return type has to be exactly correct, + // so we go via boxing. Reflection is already fairly inefficient, and this is + // only used for one-of case checking, fortunately. + var del = (Func<T1, T2>) method.CreateDelegate(typeof(Func<T1, T2>)); + return message => (int) (object) del((T1) message); + } + } + + public Action<IMessage> CreateActionIMessage(MethodInfo method) + { + var del = (Action<T1>) method.CreateDelegate(typeof(Action<T1>)); + return message => del((T1) message); + } + + public Func<IMessage, object> CreateFuncIMessageObject(MethodInfo method) + { + var del = (Func<T1, T2>) method.CreateDelegate(typeof(Func<T1, T2>)); + return message => del((T1) message); + } + + public Action<IMessage, object> CreateActionIMessageObject(MethodInfo method) + { + var del = (Action<T1, T2>) method.CreateDelegate(typeof(Action<T1, T2>)); + return (message, arg) => del((T1) message, (T2) arg); + } + } + + // Runtime compatibility checking code - see ReflectionHelper<T1, T2>.CreateFuncIMessageInt32 for + // details about why we're doing this. + + // Deliberately not inside the generic type. We only want to check this once. + private static bool CanConvertEnumFuncToInt32Func { get; } = CheckCanConvertEnumFuncToInt32Func(); + + private static bool CheckCanConvertEnumFuncToInt32Func() + { + try + { + MethodInfo method = typeof(ReflectionUtil).GetMethod(nameof(SampleEnumMethod)); + // If this passes, we're in a reasonable runtime. + method.CreateDelegate(typeof(Func<int>)); + return true; + } + catch (ArgumentException) + { + return false; + } + } + + public enum SampleEnum + { + X + } + + // Public to make the reflection simpler. + public static SampleEnum SampleEnumMethod() => SampleEnum.X; } -}
\ No newline at end of file +} diff --git a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs index 3565f433..b73930b2 100644 --- a/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs +++ b/csharp/src/Google.Protobuf/WellKnownTypes/FieldMask.cs @@ -244,8 +244,8 @@ namespace Google.Protobuf.WellKnownTypes { /// /// ## Field Mask Verification /// - /// The implementation of the all the API methods, which have any FieldMask type - /// field in the request, should verify the included field paths, and return + /// The implementation of any API method which has a FieldMask type field in the + /// request should verify the included field paths, and return an /// `INVALID_ARGUMENT` error if any path is duplicated or unmappable. /// </summary> public sealed partial class FieldMask : pb::IMessage<FieldMask> { |