diff options
19 files changed, 391 insertions, 29 deletions
diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 5aa35c66..3de0e380 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -1,9 +1,11 @@ +benchmarks_protoc_inputs_benchmark_wrapper = \ + benchmarks.proto + benchmarks_protoc_inputs = \ - benchmarks.proto \ - datasets/google_message1/benchmark_message1_proto3.proto + datasets/google_message1/proto3/benchmark_message1_proto3.proto benchmarks_protoc_inputs_proto2 = \ - datasets/google_message1/benchmark_message1_proto2.proto \ + datasets/google_message1/proto2/benchmark_message1_proto2.proto \ datasets/google_message2/benchmark_message2.proto \ datasets/google_message3/benchmark_message3.proto \ datasets/google_message3/benchmark_message3_1.proto \ @@ -26,7 +28,7 @@ make_tmp_dir: if USE_EXTERNAL_PROTOC protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) touch protoc_middleman protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2) @@ -38,8 +40,8 @@ else # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. -protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) +protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) ) touch protoc_middleman protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) @@ -54,14 +56,14 @@ all_data = `find . -type f -name "dataset.*.pb"` benchmarks_protoc_outputs = \ benchmarks.pb.cc \ - datasets/google_message1/benchmark_message1_proto3.pb.cc + datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc benchmarks_protoc_outputs_header = \ benchmarks.pb.h \ - datasets/google_message1/benchmark_message1_proto3.pb.h + datasets/google_message1/proto3/benchmark_message1_proto3.pb.h benchmarks_protoc_outputs_proto2_header = \ - datasets/google_message1/benchmark_message1_proto2.pb.h \ + datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ datasets/google_message2/benchmark_message2.pb.h \ datasets/google_message3/benchmark_message3.pb.h \ datasets/google_message3/benchmark_message3_1.pb.h \ @@ -78,7 +80,7 @@ benchmarks_protoc_outputs_proto2_header = \ datasets/google_message4/benchmark_message4_3.pb.h benchmarks_protoc_outputs_proto2 = \ - datasets/google_message1/benchmark_message1_proto2.pb.cc \ + datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ datasets/google_message2/benchmark_message2.pb.cc \ datasets/google_message3/benchmark_message3.pb.cc \ datasets/google_message3/benchmark_message3_1.pb.cc \ @@ -224,6 +226,78 @@ python-cpp-generated-code: python-cpp-generated-code-benchmark ############# PYTHON RULES END ############## +############# GO RULES BEGIN ############## + +benchmarks_protoc_inputs_proto2_message1 = \ + datasets/google_message1/proto2/benchmark_message1_proto2.proto + +benchmarks_protoc_inputs_proto2_message2 = \ + datasets/google_message2/benchmark_message2.proto + +benchmarks_protoc_inputs_proto2_message3 = \ + datasets/google_message3/benchmark_message3.proto \ + datasets/google_message3/benchmark_message3_1.proto \ + datasets/google_message3/benchmark_message3_2.proto \ + datasets/google_message3/benchmark_message3_3.proto \ + datasets/google_message3/benchmark_message3_4.proto \ + datasets/google_message3/benchmark_message3_5.proto \ + datasets/google_message3/benchmark_message3_6.proto \ + datasets/google_message3/benchmark_message3_7.proto \ + datasets/google_message3/benchmark_message3_8.proto + +benchmarks_protoc_inputs_proto2_message4 = \ + datasets/google_message4/benchmark_message4.proto \ + datasets/google_message4/benchmark_message4_1.proto \ + datasets/google_message4/benchmark_message4_2.proto \ + datasets/google_message4/benchmark_message4_3.proto + +if USE_EXTERNAL_PROTOC + +go_protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) + touch protoc_middleman + +go_protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) + $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) + touch protoc_middleman2 + +else + +# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is +# relative to srcdir, which may not be the same as the current directory when +# building out-of-tree. +go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) ) + touch protoc_middleman + +go_protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) ) + touch protoc_middleman2 + +endif + +go-benchmark: go_protoc_middleman go_protoc_middleman2 + @echo "Writing shortcut script go-benchmark..." + @echo '#! /bin/sh' > go-benchmark + @echo 'mkdir tmp_cc && mv *.cc tmp_cc' >> go-benchmark + @echo 'go test -bench=. -- $$@' >> go-benchmark + @echo 'mv tmp_cc/* . && rm -rf tmp_cc' >> go-benchmark + @chmod +x go-benchmark + +go: go_protoc_middleman go_protoc_middleman2 go-benchmark + ./go-benchmark $(all_data) + +############# GO RULES END ############## + + MAINTAINERCLEANFILES = \ Makefile.in @@ -241,7 +315,10 @@ CLEANFILES = \ python_cpp_proto_library \ python-pure-python-benchmark \ python-cpp-reflection-benchmark \ - python-cpp-generated-code-benchmark + python-cpp-generated-code-benchmark \ + go-benchmark \ + go_protoc_middleman \ + go_protoc_middleman2 clean-local: -rm -rf tmp/* diff --git a/benchmarks/README.md b/benchmarks/README.md index 459c7b9d..74c042d8 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -36,6 +36,21 @@ $ sudo apt-get install python3-dev ``` And you also need to make sure `pkg-config` is installed. +### Go +Go protobufs are maintained at [github.com/golang/protobuf]( +http://github.com/golang/protobuf). If not done already, you need to install the +toolchain and the Go protoc-gen-go plugin for protoc. + +To install protoc-gen-go, run: + +``` +$ go get -u github.com/golang/protobuf/protoc-gen-go +$ export PATH=$PATH:$(go env GOPATH)/bin +``` + +The first command installs `protoc-gen-go` into the `bin` directory in your local `GOPATH`. +The second command adds the `bin` directory to your `PATH` so that `protoc` can locate the plugin later. + ### Big data There's some optional big testing data which is not included in the directory @@ -87,6 +102,11 @@ $ make python-cpp-reflection $ make python-cpp-generated-code ``` +### Go +``` +$ make go +``` + To run a specific dataset: ### Java: @@ -126,6 +146,13 @@ $ make python-cpp-generated-code-benchmark $ ./python-cpp-generated-code-benchmark $(specific generated dataset file name) ``` +### Go: +``` +$ make go-benchmark +$ ./go-benchmark $(specific generated dataset file name) +``` + + ## Benchmark datasets Each data set is in the format of benchmarks.proto: diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp_benchmark.cc index 0ba4dc52..f8b55291 100644 --- a/benchmarks/cpp_benchmark.cc +++ b/benchmarks/cpp_benchmark.cc @@ -32,8 +32,8 @@ #include <iostream> #include "benchmark/benchmark_api.h" #include "benchmarks.pb.h" -#include "datasets/google_message1/benchmark_message1_proto2.pb.h" -#include "datasets/google_message1/benchmark_message1_proto3.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" #include "datasets/google_message2/benchmark_message2.pb.h" #include "datasets/google_message3/benchmark_message3.pb.h" #include "datasets/google_message4/benchmark_message4.pb.h" diff --git a/benchmarks/datasets/google_message1/benchmark_message1_proto2.proto b/benchmarks/datasets/google_message1/proto2/benchmark_message1_proto2.proto index 21261905..21261905 100644 --- a/benchmarks/datasets/google_message1/benchmark_message1_proto2.proto +++ b/benchmarks/datasets/google_message1/proto2/benchmark_message1_proto2.proto diff --git a/benchmarks/datasets/google_message1/dataset.google_message1_proto2.pb b/benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb Binary files differindex f6fe7848..f6fe7848 100644 --- a/benchmarks/datasets/google_message1/dataset.google_message1_proto2.pb +++ b/benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb diff --git a/benchmarks/datasets/google_message1/benchmark_message1_proto3.proto b/benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto index 090b554b..090b554b 100644 --- a/benchmarks/datasets/google_message1/benchmark_message1_proto3.proto +++ b/benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto diff --git a/benchmarks/datasets/google_message1/dataset.google_message1_proto3.pb b/benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb Binary files differindex 4955bed3..4955bed3 100644 --- a/benchmarks/datasets/google_message1/dataset.google_message1_proto3.pb +++ b/benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb diff --git a/benchmarks/go_benchmark_test.go b/benchmarks/go_benchmark_test.go new file mode 100644 index 00000000..e747465e --- /dev/null +++ b/benchmarks/go_benchmark_test.go @@ -0,0 +1,124 @@ +package main + +import ( + benchmarkWrapper "./tmp" + googleMessage1Proto2 "./tmp/datasets/google_message1/proto2" + googleMessage1Proto3 "./tmp/datasets/google_message1/proto3" + googleMessage2 "./tmp/datasets/google_message2" + googleMessage3 "./tmp/datasets/google_message3" + googleMessage4 "./tmp/datasets/google_message4" + "flag" + "github.com/golang/protobuf/proto" + "io/ioutil" + "testing" +) + +// Data is returned by the Load function. +type Dataset struct { + name string + newMessage func() proto.Message + marshaled [][]byte + unmarshaled []proto.Message +} + +var datasets []Dataset + +// This is used to getDefaultInstance for a message type. +func generateNewMessageFunction(dataset benchmarkWrapper.BenchmarkDataset) func() proto.Message { + switch dataset.MessageName { + case "benchmarks.proto3.GoogleMessage1": + return func() proto.Message { return new(googleMessage1Proto3.GoogleMessage1) } + case "benchmarks.proto2.GoogleMessage1": + return func() proto.Message { return new(googleMessage1Proto2.GoogleMessage1) } + case "benchmarks.proto2.GoogleMessage2": + return func() proto.Message { return new(googleMessage2.GoogleMessage2) } + case "benchmarks.google_message3.GoogleMessage3": + return func() proto.Message { return new(googleMessage3.GoogleMessage3) } + case "benchmarks.google_message4.GoogleMessage4": + return func() proto.Message { return new(googleMessage4.GoogleMessage4) } + default: + panic("Unknown message type: " + dataset.MessageName) + } +} + +func init() { + flag.Parse() + for _, f := range flag.Args() { + // Load the benchmark. + b, err := ioutil.ReadFile(f) + if err != nil { + panic(err) + } + + // Parse the benchmark. + var dm benchmarkWrapper.BenchmarkDataset + if err := proto.Unmarshal(b, &dm); err != nil { + panic(err) + } + + // Determine the concrete protobuf message type to use. + var ds Dataset + ds.newMessage = generateNewMessageFunction(dm) + + // Unmarshal each test message. + for _, payload := range dm.Payload { + ds.marshaled = append(ds.marshaled, payload) + m := ds.newMessage() + if err := proto.Unmarshal(payload, m); err != nil { + panic(err) + } + ds.unmarshaled = append(ds.unmarshaled, m) + } + ds.name = f + + datasets = append(datasets, ds) + } +} + +func Benchmark(b *testing.B) { + for _, ds := range datasets { + b.Run(ds.name, func(b *testing.B) { + b.Run("Unmarshal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for j, payload := range ds.marshaled { + out := ds.newMessage() + if err := proto.Unmarshal(payload, out); err != nil { + b.Fatalf("can't unmarshal message %d %v", j, err) + } + } + } + }) + b.Run("Marshal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for j, m := range ds.unmarshaled { + if _, err := proto.Marshal(m); err != nil { + b.Fatalf("can't marshal message %d %+v: %v", j, m, err) + } + } + } + }) + b.Run("Size", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + proto.Size(m) + } + } + }) + b.Run("Clone", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + proto.Clone(m) + } + } + }) + b.Run("Merge", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + out := ds.newMessage() + proto.Merge(out, m) + } + } + }) + }) + } +} diff --git a/benchmarks/py_benchmark.py b/benchmarks/py_benchmark.py index c3d80492..ba7a3470 100755 --- a/benchmarks/py_benchmark.py +++ b/benchmarks/py_benchmark.py @@ -17,8 +17,8 @@ elif sys.argv[1] != "false": raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") # END CPP GENERATED MESSAGE -import datasets.google_message1.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 -import datasets.google_message1.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 +import datasets.google_message1.proto2.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 +import datasets.google_message1.proto3.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2 import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2 import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2 diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index 7f68f4b2..22bbbfb3 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -2325,6 +2325,24 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, } )"); RunValidJsonTest( + "ValueAcceptListWithNull", REQUIRED, + R"({"optionalValue": ["x", null, "y"]})", + R"( + optional_value: { + list_value: { + values: { + string_value: "x" + } + values: { + null_value: NULL_VALUE + } + values: { + string_value: "y" + } + } + } + )"); + RunValidJsonTest( "ValueAcceptObject", REQUIRED, R"({"optionalValue": {"value": 1}})", R"( diff --git a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs index e7c6b805..8795fa65 100644..100755 --- a/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs +++ b/csharp/src/Google.Protobuf.Test/CodedInputStreamTest.cs @@ -615,5 +615,88 @@ namespace Google.Protobuf var stream = new CodedInputStream(new byte[10]);
stream.Dispose();
}
+
+ [Test]
+ public void TestParseMessagesCloseTo2G()
+ {
+ byte[] serializedMessage = GenerateBigSerializedMessage();
+ // How many of these big messages do we need to take us near our 2GB limit?
+ int count = Int32.MaxValue / serializedMessage.Length;
+ // Now make a MemoryStream that will fake a near-2GB stream of messages by returning
+ // our big serialized message 'count' times.
+ using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count))
+ {
+ Assert.DoesNotThrow(()=>TestAllTypes.Parser.ParseFrom(stream));
+ }
+ }
+
+ [Test]
+ public void TestParseMessagesOver2G()
+ {
+ byte[] serializedMessage = GenerateBigSerializedMessage();
+ // How many of these big messages do we need to take us near our 2GB limit?
+ int count = Int32.MaxValue / serializedMessage.Length;
+ // Now add one to take us over the 2GB limit
+ count++;
+ // Now make a MemoryStream that will fake a near-2GB stream of messages by returning
+ // our big serialized message 'count' times.
+ using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count))
+ {
+ Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(stream),
+ "Protocol message was too large. May be malicious. " +
+ "Use CodedInputStream.SetSizeLimit() to increase the size limit.");
+ }
+ }
+
+ /// <returns>A serialized big message</returns>
+ private static byte[] GenerateBigSerializedMessage()
+ {
+ byte[] value = new byte[16 * 1024 * 1024];
+ TestAllTypes message = SampleMessages.CreateFullTestAllTypes();
+ message.SingleBytes = ByteString.CopyFrom(value);
+ return message.ToByteArray();
+ }
+
+ /// <summary>
+ /// A MemoryStream that repeats a byte arrays' content a number of times.
+ /// Simulates really large input without consuming loads of memory. Used above
+ /// to test the parsing behavior when the input size exceeds 2GB or close to it.
+ /// </summary>
+ private class RepeatingMemoryStream: MemoryStream
+ {
+ private readonly byte[] bytes;
+ private readonly int maxIterations;
+ private int index = 0;
+
+ public RepeatingMemoryStream(byte[] bytes, int maxIterations)
+ {
+ this.bytes = bytes;
+ this.maxIterations = maxIterations;
+ }
+
+ public override int Read(byte[] buffer, int offset, int count)
+ {
+ if (bytes.Length == 0)
+ {
+ return 0;
+ }
+ int numBytesCopiedTotal = 0;
+ while (numBytesCopiedTotal < count && index < maxIterations)
+ {
+ int numBytesToCopy = Math.Min(bytes.Length - (int)Position, count);
+ Array.Copy(bytes, (int)Position, buffer, offset, numBytesToCopy);
+ numBytesCopiedTotal += numBytesToCopy;
+ offset += numBytesToCopy;
+ count -= numBytesCopiedTotal;
+ Position += numBytesToCopy;
+ if (Position >= bytes.Length)
+ {
+ Position = 0;
+ index++;
+ }
+ }
+ return numBytesCopiedTotal;
+ }
+ }
}
}
\ No newline at end of file diff --git a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs index 329ae9be..a6cf04ab 100644 --- a/csharp/src/Google.Protobuf.Test/JsonParserTest.cs +++ b/csharp/src/Google.Protobuf.Test/JsonParserTest.cs @@ -696,6 +696,22 @@ namespace Google.Protobuf } [Test] + public void Value_List_WithNullElement() + { + var expected = Value.ForList(Value.ForString("x"), Value.ForNull(), Value.ForString("y")); + var actual = Value.Parser.ParseJson("[\"x\", null, \"y\"]"); + Assert.AreEqual(expected, actual); + } + + [Test] + public void StructValue_NullElement() + { + var expected = Value.ForStruct(new Struct { Fields = { { "x", Value.ForNull() } } }); + var actual = Value.Parser.ParseJson("{ \"x\": null }"); + Assert.AreEqual(expected, actual); + } + + [Test] public void ParseListValue() { Assert.AreEqual(new ListValue { Values = { Value.ForNumber(1), Value.ForString("x") } }, ListValue.Parser.ParseJson("[1, \"x\"]")); diff --git a/csharp/src/Google.Protobuf/CodedInputStream.cs b/csharp/src/Google.Protobuf/CodedInputStream.cs index 6bee238f..0a829545 100644 --- a/csharp/src/Google.Protobuf/CodedInputStream.cs +++ b/csharp/src/Google.Protobuf/CodedInputStream.cs @@ -94,7 +94,7 @@ namespace Google.Protobuf private bool hasNextTag = false;
internal const int DefaultRecursionLimit = 64;
- internal const int DefaultSizeLimit = 64 << 20; // 64MB
+ internal const int DefaultSizeLimit = Int32.MaxValue;
internal const int BufferSize = 4096;
/// <summary>
@@ -248,7 +248,7 @@ namespace Google.Protobuf /// <remarks>
/// This limit is applied when reading from the underlying stream, as a sanity check. It is
/// not applied when reading from a byte array data source without an underlying stream.
- /// The default value is 64MB.
+ /// The default value is Int32.MaxValue.
/// </remarks>
/// <value>
/// The size limit.
@@ -1058,7 +1058,7 @@ namespace Google.Protobuf RecomputeBufferSizeAfterLimit();
int totalBytesRead =
totalBytesRetired + bufferSize + bufferSizeAfterLimit;
- if (totalBytesRead > sizeLimit || totalBytesRead < 0)
+ if (totalBytesRead < 0 || totalBytesRead > sizeLimit)
{
throw InvalidProtocolBufferException.SizeLimitExceeded();
}
diff --git a/csharp/src/Google.Protobuf/JsonParser.cs b/csharp/src/Google.Protobuf/JsonParser.cs index 3621b0c0..284bce93 100644 --- a/csharp/src/Google.Protobuf/JsonParser.cs +++ b/csharp/src/Google.Protobuf/JsonParser.cs @@ -264,11 +264,12 @@ namespace Google.Protobuf return; } tokenizer.PushBack(token); - if (token.Type == JsonToken.TokenType.Null) + object value = ParseSingleValue(field, tokenizer); + if (value == null) { throw new InvalidProtocolBufferException("Repeated field elements cannot be null"); } - list.Add(ParseSingleValue(field, tokenizer)); + list.Add(value); } } diff --git a/objectivec/GPBCodedOutputStream.m b/objectivec/GPBCodedOutputStream.m index f832e8a6..b846c2fc 100644 --- a/objectivec/GPBCodedOutputStream.m +++ b/objectivec/GPBCodedOutputStream.m @@ -942,7 +942,10 @@ static void GPBWriteRawLittleEndian64(GPBOutputBufferState *state, state_.position = length; } else { // Write is very big. Let's do it all at once. - [state_.output write:((uint8_t *)value) + offset maxLength:length]; + NSInteger written = [state_.output write:((uint8_t *)value) + offset maxLength:length]; + if (written != (NSInteger)length) { + [NSException raise:GPBCodedOutputStreamException_WriteFailed format:@""]; + } } } } diff --git a/objectivec/Tests/GPBCodedOuputStreamTests.m b/objectivec/Tests/GPBCodedOuputStreamTests.m index 878e7aa9..109239d5 100644 --- a/objectivec/Tests/GPBCodedOuputStreamTests.m +++ b/objectivec/Tests/GPBCodedOuputStreamTests.m @@ -423,4 +423,14 @@ } } +- (void)testThatItThrowsWhenWriteRawPtrFails { + NSOutputStream *output = [NSOutputStream outputStreamToMemory]; + GPBCodedOutputStream *codedOutput = + [GPBCodedOutputStream streamWithOutputStream:output bufferSize:0]; // Skip buffering. + [output close]; // Close the output stream to force failure on write. + const char *cString = "raw"; + XCTAssertThrowsSpecificNamed([codedOutput writeRawPtr:cString offset:0 length:strlen(cString)], + NSException, GPBCodedOutputStreamException_WriteFailed); +} + @end diff --git a/src/google/protobuf/stubs/hash.h b/src/google/protobuf/stubs/hash.h index d04a25e1..fd8ba156 100644 --- a/src/google/protobuf/stubs/hash.h +++ b/src/google/protobuf/stubs/hash.h @@ -93,7 +93,7 @@ # endif // GCC <= 4.1 does not define std::tr1::hash for `long long int` or `long long unsigned int` -# if __GNUC__ == 4 && defined(__GNUC__MINOR__) && __GNUC__MINOR__ <= 1 +# if __GNUC__ == 4 && defined(__GNUC_MINOR__) && __GNUC_MINOR__ <= 1 # undef GOOGLE_PROTOBUF_HAS_TR1 # undef GOOGLE_PROTOBUF_HAVE_HASH_MAP # undef GOOGLE_PROTOBUF_HAVE_HASH_SET diff --git a/src/google/protobuf/stubs/port.h b/src/google/protobuf/stubs/port.h index de83def0..6b52305f 100644 --- a/src/google/protobuf/stubs/port.h +++ b/src/google/protobuf/stubs/port.h @@ -91,6 +91,7 @@ // These #includes are for the byte swap functions declared later on. #ifdef _MSC_VER #include <stdlib.h> // NOLINT(build/include) +#include <intrin.h> #elif defined(__APPLE__) #include <libkern/OSByteOrder.h> #elif defined(__GLIBC__) || defined(__BIONIC__) || defined(__CYGWIN__) @@ -414,12 +415,10 @@ class Bits { static uint32 Log2FloorNonZero(uint32 n) { #if defined(__GNUC__) return 31 ^ static_cast<uint32>(__builtin_clz(n)); -#elif defined(COMPILER_MSVC) && defined(_M_IX86) - _asm { - bsr ebx, n - mov n, ebx - } - return n; +#elif defined(_MSC_VER) + unsigned long where; + _BitScanReverse(&where, n); + return where; #else return Log2FloorNonZero_Portable(n); #endif @@ -434,6 +433,10 @@ class Bits { // implementation instead. #if defined(__GNUC__) && !defined(GOOGLE_PROTOBUF_USE_PORTABLE_LOG2) return 63 ^ static_cast<uint32>(__builtin_clzll(n)); +#elif defined(_MSC_VER) && defined(_M_X64) + unsigned long where; + _BitScanReverse64(&where, n); + return where; #else return Log2FloorNonZero64_Portable(n); #endif @@ -35,7 +35,7 @@ internal_build_cpp() { build_cpp() { internal_build_cpp - make check -j2 + make check -j2 || (cat src/test-suite.log; false) cd conformance && make test_cpp && cd .. # The benchmark code depends on cmake, so test if it is installed before |