From e841bac4fcf47f809e089a70d5f84ac37b3883df Mon Sep 17 00:00:00 2001 From: Feng Xiao Date: Fri, 11 Dec 2015 17:09:20 -0800 Subject: Down-integrate from internal code base. --- Makefile.am | 9 +- appveyor.yml | 64 +- cmake/extract_includes.bat.in | 2 + cmake/libprotobuf.cmake | 1 + cmake/libprotoc.cmake | 1 + conformance/ConformanceJava.java | 28 +- conformance/conformance.proto | 61 + conformance/conformance_cpp.cc | 17 +- conformance/conformance_test.cc | 1495 ++++++++++- conformance/conformance_test.h | 23 +- conformance/conformance_test_runner.cc | 24 +- conformance/failure_list_cpp.txt | 88 + conformance/failure_list_java.txt | 74 + java/pom.xml | 5 + .../java/com/google/protobuf/AbstractMessage.java | 13 +- .../java/com/google/protobuf/BooleanArrayList.java | 11 +- .../com/google/protobuf/BoundedByteString.java | 68 +- .../main/java/com/google/protobuf/ByteString.java | 237 +- .../java/com/google/protobuf/CodedInputStream.java | 153 +- .../com/google/protobuf/CodedOutputStream.java | 91 +- .../main/java/com/google/protobuf/Descriptors.java | 43 + .../java/com/google/protobuf/DoubleArrayList.java | 11 +- .../java/com/google/protobuf/FloatArrayList.java | 11 +- .../com/google/protobuf/GeneratedMessageLite.java | 31 +- .../java/com/google/protobuf/IntArrayList.java | 11 +- .../com/google/protobuf/LiteralByteString.java | 213 +- .../java/com/google/protobuf/LongArrayList.java | 11 +- .../java/com/google/protobuf/MapFieldLite.java | 9 +- .../com/google/protobuf/MessageLiteToString.java | 0 .../java/com/google/protobuf/NioByteString.java | 309 +++ .../com/google/protobuf/ProtobufArrayList.java | 4 + .../java/com/google/protobuf/RopeByteString.java | 252 +- .../main/java/com/google/protobuf/TextFormat.java | 54 +- .../com/google/protobuf/TextFormatEscaper.java | 0 .../com/google/protobuf/UnsafeByteStrings.java | 55 + .../com/google/protobuf/BooleanArrayListTest.java | 8 +- .../com/google/protobuf/BoundedByteStringTest.java | 2 +- .../java/com/google/protobuf/ByteStringTest.java | 16 +- .../com/google/protobuf/CodedOutputStreamTest.java | 45 +- .../java/com/google/protobuf/DescriptorsTest.java | 9 + .../com/google/protobuf/DoubleArrayListTest.java | 6 +- .../com/google/protobuf/FloatArrayListTest.java | 6 +- .../com/google/protobuf/LiteralByteStringTest.java | 2 +- .../com/google/protobuf/LongArrayListTest.java | 6 +- .../com/google/protobuf/NioByteStringTest.java | 546 ++++ .../com/google/protobuf/ProtobufArrayListTest.java | 4 +- .../protobuf/RopeByteStringSubstringTest.java | 2 +- .../test/java/com/google/protobuf/TestUtil.java | 154 ++ .../test/java/com/google/protobuf/map_test.proto | 1 - .../com/google/protobuf/test_bad_identifiers.proto | 1 - .../com/google/protobuf/util/FieldMaskUtil.java | 79 +- .../java/com/google/protobuf/util/JsonFormat.java | 226 +- .../java/com/google/protobuf/util/TimeUtil.java | 12 +- .../google/protobuf/util/FieldMaskUtilTest.java | 48 +- .../com/google/protobuf/util/JsonFormatTest.java | 390 ++- .../com/google/protobuf/util/TimeUtilTest.java | 67 + .../java/com/google/protobuf/util/json_test.proto | 20 +- js/binary/arith.js | 413 +++ js/binary/arith_test.js | 355 +++ js/binary/constants.js | 320 +++ js/binary/decoder.js | 1005 ++++++++ js/binary/decoder_test.js | 327 +++ js/binary/proto_test.js | 588 +++++ js/binary/reader.js | 1127 +++++++++ js/binary/reader_test.js | 889 +++++++ js/binary/utils.js | 979 ++++++++ js/binary/utils_test.js | 632 +++++ js/binary/writer.js | 2124 ++++++++++++++++ js/binary/writer_test.js | 123 + js/data.proto | 51 + js/debug.js | 140 ++ js/debug_test.js | 101 + js/message.js | 1125 +++++++++ js/message_test.js | 970 ++++++++ js/proto3_test.js | 279 +++ js/proto3_test.proto | 89 + js/test.proto | 212 ++ js/test2.proto | 54 + js/test3.proto | 53 + js/test4.proto | 42 + js/test5.proto | 44 + js/test_bootstrap.js | 41 + js/testbinary.proto | 185 ++ js/testempty.proto | 34 + objectivec/google/protobuf/Any.pbobjc.h | 7 +- objectivec/google/protobuf/Api.pbobjc.h | 1 - objectivec/google/protobuf/FieldMask.pbobjc.h | 2 +- objectivec/google/protobuf/Type.pbobjc.h | 57 +- python/google/protobuf/descriptor.py | 12 +- python/google/protobuf/descriptor_database.py | 4 + python/google/protobuf/descriptor_pool.py | 69 +- python/google/protobuf/internal/any_test.proto | 42 + python/google/protobuf/internal/containers.py | 23 + .../protobuf/internal/descriptor_pool_test.py | 176 +- python/google/protobuf/internal/descriptor_test.py | 20 +- .../google/protobuf/internal/json_format_test.py | 58 +- .../protobuf/internal/message_factory_test.py | 7 +- .../protobuf/internal/message_set_extensions.proto | 8 + python/google/protobuf/internal/message_test.py | 81 +- python/google/protobuf/internal/python_message.py | 45 +- python/google/protobuf/internal/reflection_test.py | 18 +- .../google/protobuf/internal/text_format_test.py | 170 +- .../google/protobuf/internal/well_known_types.py | 622 +++++ .../protobuf/internal/well_known_types_test.py | 509 ++++ python/google/protobuf/json_format.py | 269 +- python/google/protobuf/message_factory.py | 4 +- python/google/protobuf/pyext/descriptor.cc | 29 +- .../google/protobuf/pyext/descriptor_database.cc | 145 ++ python/google/protobuf/pyext/descriptor_database.h | 75 + python/google/protobuf/pyext/descriptor_pool.cc | 111 +- python/google/protobuf/pyext/descriptor_pool.h | 10 + python/google/protobuf/pyext/extension_dict.cc | 47 +- python/google/protobuf/pyext/extension_dict.h | 5 - python/google/protobuf/pyext/map_container.cc | 912 +++++++ python/google/protobuf/pyext/map_container.h | 133 + python/google/protobuf/pyext/message.cc | 299 ++- python/google/protobuf/pyext/message.h | 1 + .../google/protobuf/pyext/message_map_container.cc | 569 ----- .../google/protobuf/pyext/message_map_container.h | 126 - .../google/protobuf/pyext/scalar_map_container.cc | 542 ---- .../google/protobuf/pyext/scalar_map_container.h | 119 - python/google/protobuf/symbol_database.py | 32 +- python/google/protobuf/text_format.py | 264 +- python/setup.py | 1 + src/Makefile.am | 8 +- src/google/protobuf/any.cc | 13 +- src/google/protobuf/any.h | 2 +- src/google/protobuf/any.pb.cc | 1 + src/google/protobuf/any.proto | 17 +- src/google/protobuf/api.pb.cc | 1 + src/google/protobuf/api.proto | 5 +- src/google/protobuf/arena.h | 14 +- src/google/protobuf/arenastring.h | 1 - .../protobuf/compiler/command_line_interface.cc | 59 +- .../protobuf/compiler/command_line_interface.h | 3 + .../compiler/command_line_interface_unittest.cc | 20 + src/google/protobuf/compiler/cpp/cpp_file.cc | 1 + src/google/protobuf/compiler/importer.cc | 13 + src/google/protobuf/compiler/importer.h | 9 + src/google/protobuf/compiler/importer_unittest.cc | 8 + src/google/protobuf/compiler/java/java_enum.cc | 16 +- .../protobuf/compiler/java/java_enum_field_lite.cc | 40 +- .../protobuf/compiler/java/java_enum_lite.cc | 32 +- src/google/protobuf/compiler/java/java_file.cc | 34 +- .../compiler/java/java_primitive_field_lite.cc | 38 +- .../protobuf/compiler/java/java_string_field.cc | 80 +- .../protobuf/compiler/java/java_string_field.h | 1 - .../compiler/java/java_string_field_lite.cc | 66 +- .../compiler/java/java_string_field_lite.h | 1 - src/google/protobuf/compiler/js/js_generator.cc | 2620 ++++++++++++++++++++ src/google/protobuf/compiler/js/js_generator.h | 265 ++ src/google/protobuf/compiler/main.cc | 6 + .../protobuf/compiler/mock_code_generator.cc | 6 + src/google/protobuf/compiler/parser.cc | 58 + src/google/protobuf/compiler/parser.h | 2 + src/google/protobuf/compiler/parser_unittest.cc | 56 +- src/google/protobuf/compiler/plugin.pb.cc | 1 + src/google/protobuf/compiler/subprocess.cc | 1 + src/google/protobuf/descriptor.cc | 35 +- src/google/protobuf/descriptor.pb.cc | 1 + src/google/protobuf/descriptor_unittest.cc | 193 +- src/google/protobuf/duration.pb.cc | 1 + src/google/protobuf/duration.proto | 9 +- src/google/protobuf/dynamic_message.cc | 2 +- src/google/protobuf/empty.pb.cc | 43 +- src/google/protobuf/empty.pb.h | 13 + src/google/protobuf/empty.proto | 8 +- src/google/protobuf/field_mask.pb.cc | 1 + src/google/protobuf/field_mask.proto | 11 +- src/google/protobuf/io/coded_stream.h | 1 + src/google/protobuf/io/strtod.cc | 11 + src/google/protobuf/io/strtod.h | 5 + src/google/protobuf/io/tokenizer.cc | 2 +- src/google/protobuf/io/tokenizer_unittest.cc | 2 + src/google/protobuf/map.h | 26 +- src/google/protobuf/repeated_field.h | 24 +- src/google/protobuf/source_context.pb.cc | 1 + src/google/protobuf/source_context.proto | 7 +- src/google/protobuf/struct.pb.cc | 1 + src/google/protobuf/struct.proto | 9 +- src/google/protobuf/stubs/strutil.cc | 86 +- src/google/protobuf/stubs/strutil.h | 24 +- src/google/protobuf/text_format.cc | 39 +- src/google/protobuf/text_format.h | 14 + src/google/protobuf/timestamp.pb.cc | 43 +- src/google/protobuf/timestamp.pb.h | 13 + src/google/protobuf/timestamp.proto | 11 +- src/google/protobuf/type.pb.cc | 153 +- src/google/protobuf/type.pb.h | 55 + src/google/protobuf/type.proto | 60 +- src/google/protobuf/unittest_custom_options.proto | 27 + src/google/protobuf/util/field_comparator.h | 2 +- src/google/protobuf/util/field_comparator_test.cc | 7 +- src/google/protobuf/util/internal/datapiece.cc | 74 +- src/google/protobuf/util/internal/datapiece.h | 3 +- .../util/internal/default_value_objectwriter.cc | 90 +- .../util/internal/default_value_objectwriter.h | 23 +- .../internal/default_value_objectwriter_test.cc | 35 +- src/google/protobuf/util/internal/error_listener.h | 2 + .../util/internal/json_objectwriter_test.cc | 205 +- .../protobuf/util/internal/json_stream_parser.cc | 2 +- .../util/internal/json_stream_parser_test.cc | 1 + src/google/protobuf/util/internal/object_writer.h | 5 - src/google/protobuf/util/internal/proto_writer.cc | 744 ++++++ src/google/protobuf/util/internal/proto_writer.h | 315 +++ .../util/internal/protostream_objectsource.cc | 187 +- .../util/internal/protostream_objectsource.h | 15 +- .../util/internal/protostream_objectsource_test.cc | 10 +- .../util/internal/protostream_objectwriter.cc | 1560 ++++-------- .../util/internal/protostream_objectwriter.h | 318 +-- .../util/internal/protostream_objectwriter_test.cc | 126 +- .../protobuf/util/internal/testdata/books.proto | 2 +- .../util/internal/testdata/default_value.proto | 7 +- .../internal/testdata/default_value_test.proto | 7 + src/google/protobuf/util/internal/type_info.cc | 3 +- src/google/protobuf/util/internal/utility.cc | 29 +- src/google/protobuf/util/internal/utility.h | 5 + src/google/protobuf/util/json_format_proto3.proto | 9 + src/google/protobuf/util/message_differencer.cc | 22 +- src/google/protobuf/util/message_differencer.h | 10 +- .../protobuf/util/message_differencer_unittest.cc | 24 +- src/google/protobuf/util/type_resolver_util.cc | 45 +- .../protobuf/util/type_resolver_util_test.cc | 14 + src/google/protobuf/wrappers.pb.cc | 429 +++- src/google/protobuf/wrappers.pb.h | 205 +- src/google/protobuf/wrappers.proto | 9 +- 226 files changed, 26965 insertions(+), 5063 deletions(-) create mode 100644 conformance/failure_list_java.txt create mode 100644 java/src/main/java/com/google/protobuf/MessageLiteToString.java create mode 100644 java/src/main/java/com/google/protobuf/NioByteString.java create mode 100644 java/src/main/java/com/google/protobuf/TextFormatEscaper.java create mode 100644 java/src/main/java/com/google/protobuf/UnsafeByteStrings.java create mode 100644 java/src/test/java/com/google/protobuf/NioByteStringTest.java create mode 100644 js/binary/arith.js create mode 100644 js/binary/arith_test.js create mode 100644 js/binary/constants.js create mode 100644 js/binary/decoder.js create mode 100644 js/binary/decoder_test.js create mode 100644 js/binary/proto_test.js create mode 100644 js/binary/reader.js create mode 100644 js/binary/reader_test.js create mode 100644 js/binary/utils.js create mode 100644 js/binary/utils_test.js create mode 100644 js/binary/writer.js create mode 100644 js/binary/writer_test.js create mode 100644 js/data.proto create mode 100644 js/debug.js create mode 100644 js/debug_test.js create mode 100644 js/message.js create mode 100644 js/message_test.js create mode 100644 js/proto3_test.js create mode 100644 js/proto3_test.proto create mode 100644 js/test.proto create mode 100644 js/test2.proto create mode 100644 js/test3.proto create mode 100644 js/test4.proto create mode 100644 js/test5.proto create mode 100644 js/test_bootstrap.js create mode 100644 js/testbinary.proto create mode 100644 js/testempty.proto create mode 100644 python/google/protobuf/internal/any_test.proto create mode 100644 python/google/protobuf/internal/well_known_types.py create mode 100644 python/google/protobuf/internal/well_known_types_test.py create mode 100644 python/google/protobuf/pyext/descriptor_database.cc create mode 100644 python/google/protobuf/pyext/descriptor_database.h create mode 100644 python/google/protobuf/pyext/map_container.cc create mode 100644 python/google/protobuf/pyext/map_container.h delete mode 100644 python/google/protobuf/pyext/message_map_container.cc delete mode 100644 python/google/protobuf/pyext/message_map_container.h delete mode 100644 python/google/protobuf/pyext/scalar_map_container.cc delete mode 100644 python/google/protobuf/pyext/scalar_map_container.h create mode 100755 src/google/protobuf/compiler/js/js_generator.cc create mode 100755 src/google/protobuf/compiler/js/js_generator.h create mode 100644 src/google/protobuf/util/internal/proto_writer.cc create mode 100644 src/google/protobuf/util/internal/proto_writer.h diff --git a/Makefile.am b/Makefile.am index bb735177..6ebe3639 100644 --- a/Makefile.am +++ b/Makefile.am @@ -215,6 +215,7 @@ java_EXTRA_DIST= \ java/src/main/java/com/google/protobuf/Message.java \ java/src/main/java/com/google/protobuf/MessageLite.java \ java/src/main/java/com/google/protobuf/MessageLiteOrBuilder.java \ + java/src/main/java/com/google/protobuf/MessageLiteToString.java \ java/src/main/java/com/google/protobuf/MessageOrBuilder.java \ java/src/main/java/com/google/protobuf/MessageReflection.java \ java/src/main/java/com/google/protobuf/MutabilityOracle.java \ @@ -233,6 +234,7 @@ java_EXTRA_DIST= \ java/src/main/java/com/google/protobuf/SingleFieldBuilder.java \ java/src/main/java/com/google/protobuf/SmallSortedMap.java \ java/src/main/java/com/google/protobuf/TextFormat.java \ + java/src/main/java/com/google/protobuf/TextFormatEscaper.java \ java/src/main/java/com/google/protobuf/UninitializedMessageException.java \ java/src/main/java/com/google/protobuf/UnknownFieldSet.java \ java/src/main/java/com/google/protobuf/UnknownFieldSetLite.java \ @@ -504,6 +506,7 @@ objectivec_EXTRA_DIST= \ python_EXTRA_DIST= \ python/MANIFEST.in \ + python/google/protobuf/internal/any_test.proto \ python/google/protobuf/internal/api_implementation.cc \ python/google/protobuf/internal/api_implementation.py \ python/google/protobuf/internal/containers.py \ @@ -540,6 +543,8 @@ python_EXTRA_DIST= \ python/google/protobuf/internal/text_format_test.py \ python/google/protobuf/internal/type_checkers.py \ python/google/protobuf/internal/unknown_fields_test.py \ + python/google/protobuf/internal/well_known_types.py \ + python/google/protobuf/internal/well_known_types_test.py \ python/google/protobuf/internal/wire_format.py \ python/google/protobuf/internal/wire_format_test.py \ python/google/protobuf/internal/__init__.py \ @@ -558,8 +563,6 @@ python_EXTRA_DIST= \ python/google/protobuf/pyext/extension_dict.cc \ python/google/protobuf/pyext/message.h \ python/google/protobuf/pyext/message.cc \ - python/google/protobuf/pyext/message_map_container.cc \ - python/google/protobuf/pyext/message_map_container.h \ python/google/protobuf/pyext/proto2_api_test.proto \ python/google/protobuf/pyext/python.proto \ python/google/protobuf/pyext/python_protobuf.h \ @@ -567,8 +570,6 @@ python_EXTRA_DIST= \ python/google/protobuf/pyext/repeated_composite_container.cc \ python/google/protobuf/pyext/repeated_scalar_container.h \ python/google/protobuf/pyext/repeated_scalar_container.cc \ - python/google/protobuf/pyext/scalar_map_container.cc \ - python/google/protobuf/pyext/scalar_map_container.h \ python/google/protobuf/pyext/scoped_pyobject_ptr.h \ python/google/protobuf/pyext/__init__.py \ python/google/protobuf/descriptor.py \ diff --git a/appveyor.yml b/appveyor.yml index 3447602e..c84ecae2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,32 +1,32 @@ -# Only test one combination: "Visual Studio 12 + Win64 + Debug + DLL". We can -# test more combinations but AppVeyor just takes too long to finish (each -# combination takes ~15mins). -platform: - - Win64 - -configuration: - - Debug - -environment: - matrix: - - language: cpp - BUILD_DLL: ON - - - language: csharp - -install: - - ps: Start-FileDownload https://googlemock.googlecode.com/files/gmock-1.7.0.zip - - 7z x gmock-1.7.0.zip - - rename gmock-1.7.0 gmock - -before_build: - - if %platform%==Win32 set generator=Visual Studio 12 - - if %platform%==Win64 set generator=Visual Studio 12 Win64 - - if %platform%==Win32 set vcplatform=Win32 - - if %platform%==Win64 set vcplatform=x64 - -build_script: - - CALL appveyor.bat - -skip_commits: - message: /.*\[skip appveyor\].*/ +# Only test one combination: "Visual Studio 12 + Win64 + Debug + DLL". We can +# test more combinations but AppVeyor just takes too long to finish (each +# combination takes ~15mins). +platform: + - Win64 + +configuration: + - Debug + +environment: + matrix: + - language: cpp + BUILD_DLL: ON + + - language: csharp + +install: + - ps: Start-FileDownload https://googlemock.googlecode.com/files/gmock-1.7.0.zip + - 7z x gmock-1.7.0.zip + - rename gmock-1.7.0 gmock + +before_build: + - if %platform%==Win32 set generator=Visual Studio 12 + - if %platform%==Win64 set generator=Visual Studio 12 Win64 + - if %platform%==Win32 set vcplatform=Win32 + - if %platform%==Win64 set vcplatform=x64 + +build_script: + - CALL appveyor.bat + +skip_commits: + message: /.*\[skip appveyor\].*/ diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in index f6d8b893..49964501 100644 --- a/cmake/extract_includes.bat.in +++ b/cmake/extract_includes.bat.in @@ -6,6 +6,7 @@ mkdir include\google\protobuf\compiler\cpp mkdir include\google\protobuf\compiler\csharp mkdir include\google\protobuf\compiler\java mkdir include\google\protobuf\compiler\javanano +mkdir include\google\protobuf\compiler\js mkdir include\google\protobuf\compiler\objectivec mkdir include\google\protobuf\compiler\python mkdir include\google\protobuf\compiler\ruby @@ -26,6 +27,7 @@ copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\importer.h in copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_generator.h include\google\protobuf\compiler\java\java_generator.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\java\java_names.h include\google\protobuf\compiler\java\java_names.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\javanano\javanano_generator.h include\google\protobuf\compiler\javanano\javanano_generator.h +copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\js\js_generator.h include\google\protobuf\compiler\js\js_generator.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_generator.h include\google\protobuf\compiler\objectivec\objectivec_generator.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\objectivec\objectivec_helpers.h include\google\protobuf\compiler\objectivec\objectivec_helpers.h copy ${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\compiler\parser.h include\google\protobuf\compiler\parser.h diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index a6ee8f2e..8930c1ca 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -40,6 +40,7 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/util/internal/json_objectwriter.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/json_stream_parser.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/object_writer.cc + ${protobuf_source_dir}/src/google/protobuf/util/internal/proto_writer.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/protostream_objectsource.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/protostream_objectwriter.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/type_info.cc diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index 4e628905..e70e453e 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -70,6 +70,7 @@ set(libprotoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/javanano/javanano_message.cc ${protobuf_source_dir}/src/google/protobuf/compiler/javanano/javanano_message_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc + ${protobuf_source_dir}/src/google/protobuf/compiler/js/js_generator.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_extension.cc diff --git a/conformance/ConformanceJava.java b/conformance/ConformanceJava.java index 25646b85..a983ba3c 100644 --- a/conformance/ConformanceJava.java +++ b/conformance/ConformanceJava.java @@ -1,9 +1,12 @@ import com.google.protobuf.conformance.Conformance; +import com.google.protobuf.util.JsonFormat; +import com.google.protobuf.util.JsonFormat.TypeRegistry; import com.google.protobuf.InvalidProtocolBufferException; class ConformanceJava { private int testCount = 0; + private TypeRegistry typeRegistry; private boolean readFromStdin(byte[] buf, int len) throws Exception { int ofs = 0; @@ -29,7 +32,10 @@ class ConformanceJava { if (!readFromStdin(buf, 4)) { return -1; } - return buf[0] | (buf[1] << 1) | (buf[2] << 2) | (buf[3] << 3); + return (buf[0] & 0xff) + | ((buf[1] & 0xff) << 8) + | ((buf[2] & 0xff) << 16) + | ((buf[3] & 0xff) << 24); } private void writeLittleEndianIntToStdout(int val) throws Exception { @@ -54,7 +60,15 @@ class ConformanceJava { break; } case JSON_PAYLOAD: { - return Conformance.ConformanceResponse.newBuilder().setSkipped("JSON not yet supported.").build(); + try { + Conformance.TestAllTypes.Builder builder = Conformance.TestAllTypes.newBuilder(); + JsonFormat.parser().usingTypeRegistry(typeRegistry) + .merge(request.getJsonPayload(), builder); + testMessage = builder.build(); + } catch (InvalidProtocolBufferException e) { + return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build(); + } + break; } case PAYLOAD_NOT_SET: { throw new RuntimeException("Request didn't have payload."); @@ -73,7 +87,13 @@ class ConformanceJava { return Conformance.ConformanceResponse.newBuilder().setProtobufPayload(testMessage.toByteString()).build(); case JSON: - return Conformance.ConformanceResponse.newBuilder().setSkipped("JSON not yet supported.").build(); + try { + return Conformance.ConformanceResponse.newBuilder().setJsonPayload( + JsonFormat.printer().usingTypeRegistry(typeRegistry).print(testMessage)).build(); + } catch (InvalidProtocolBufferException | IllegalArgumentException e) { + return Conformance.ConformanceResponse.newBuilder().setSerializeError( + e.getMessage()).build(); + } default: { throw new RuntimeException("Unexpected request output."); @@ -106,6 +126,8 @@ class ConformanceJava { } public void run() throws Exception { + typeRegistry = TypeRegistry.newBuilder().add( + Conformance.TestAllTypes.getDescriptor()).build(); while (doTestIo()) { // Empty. } diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 714cbe78..fc96074a 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -32,6 +32,13 @@ syntax = "proto3"; package conformance; option java_package = "com.google.protobuf.conformance"; +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/field_mask.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, // the suite will send a ConformanceRequest message and expect a @@ -84,6 +91,11 @@ message ConformanceResponse { // test. Some of the test cases are intentionally invalid input. string parse_error = 1; + // If the input was successfully parsed but errors occurred when + // serializing it to the requested output format, set the error message in + // this field. + string serialize_error = 6; + // This should be set if some other error occurred. This will always // indicate that the test failed. The string can provide more information // about the failure. @@ -199,6 +211,55 @@ message TestAllTypes { string oneof_string = 113; bytes oneof_bytes = 114; } + + // Well-known types + google.protobuf.BoolValue optional_bool_wrapper = 201; + google.protobuf.Int32Value optional_int32_wrapper = 202; + google.protobuf.Int64Value optional_int64_wrapper = 203; + google.protobuf.UInt32Value optional_uint32_wrapper = 204; + google.protobuf.UInt64Value optional_uint64_wrapper = 205; + google.protobuf.FloatValue optional_float_wrapper = 206; + google.protobuf.DoubleValue optional_double_wrapper = 207; + google.protobuf.StringValue optional_string_wrapper = 208; + google.protobuf.BytesValue optional_bytes_wrapper = 209; + + repeated google.protobuf.BoolValue repeated_bool_wrapper = 211; + repeated google.protobuf.Int32Value repeated_int32_wrapper = 212; + repeated google.protobuf.Int64Value repeated_int64_wrapper = 213; + repeated google.protobuf.UInt32Value repeated_uint32_wrapper = 214; + repeated google.protobuf.UInt64Value repeated_uint64_wrapper = 215; + repeated google.protobuf.FloatValue repeated_float_wrapper = 216; + repeated google.protobuf.DoubleValue repeated_double_wrapper = 217; + repeated google.protobuf.StringValue repeated_string_wrapper = 218; + repeated google.protobuf.BytesValue repeated_bytes_wrapper = 219; + + google.protobuf.Duration optional_duration = 301; + google.protobuf.Timestamp optional_timestamp = 302; + google.protobuf.FieldMask optional_field_mask = 303; + google.protobuf.Struct optional_struct = 304; + google.protobuf.Any optional_any = 305; + google.protobuf.Value optional_value = 306; + + repeated google.protobuf.Duration repeated_duration = 311; + repeated google.protobuf.Timestamp repeated_timestamp = 312; + repeated google.protobuf.FieldMask repeated_fieldmask = 313; + repeated google.protobuf.Struct repeated_struct = 324; + repeated google.protobuf.Any repeated_any = 315; + repeated google.protobuf.Value repeated_value = 316; + + // Test field-name-to-JSON-name convention. + int32 fieldname1 = 401; + int32 field_name2 = 402; + int32 _field_name3 = 403; + int32 field__name4_ = 404; + int32 field0name5 = 405; + int32 field_0_name6 = 406; + int32 fieldName7 = 407; + int32 FieldName8 = 408; + int32 field_Name9 = 409; + int32 Field_Name10 = 410; + int32 FIELD_NAME11 = 411; + int32 FIELD_name12 = 412; } message ForeignMessage { diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 863b6a5b..1a265493 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -108,7 +108,11 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { return; } - GOOGLE_CHECK(test_message.ParseFromString(proto_binary)); + if (!test_message.ParseFromString(proto_binary)) { + response->set_runtime_error( + "Parsing JSON generates invalid proto output."); + return; + } break; } @@ -132,9 +136,18 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { GOOGLE_CHECK(test_message.SerializeToString(&proto_binary)); Status status = BinaryToJsonString(type_resolver, *type_url, proto_binary, response->mutable_json_payload()); - GOOGLE_CHECK(status.ok()); + if (!status.ok()) { + response->set_serialize_error( + string("Failed to serialize JSON output: ") + + status.error_message().as_string()); + return; + } break; } + + default: + GOOGLE_LOG(FATAL) << "Unknown output format: " + << request.requested_output_format(); } } diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index c250cb1e..0516bb29 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -37,10 +37,14 @@ #include #include #include +#include #include #include #include +#include "third_party/jsoncpp/value.h" +#include "third_party/jsoncpp/reader.h" + using conformance::ConformanceRequest; using conformance::ConformanceResponse; using conformance::TestAllTypes; @@ -49,6 +53,7 @@ using google::protobuf::Descriptor; using google::protobuf::FieldDescriptor; using google::protobuf::internal::WireFormatLite; using google::protobuf::TextFormat; +using google::protobuf::util::DefaultFieldComparator; using google::protobuf::util::JsonToBinaryString; using google::protobuf::util::MessageDifferencer; using google::protobuf::util::NewTypeResolverForDescriptorPool; @@ -220,7 +225,7 @@ void ConformanceTestSuite::RunTest(const string& test_name, string serialized_response; request.SerializeToString(&serialized_request); - runner_->RunTest(serialized_request, &serialized_response); + runner_->RunTest(test_name, serialized_request, &serialized_response); if (!response->ParseFromString(serialized_response)) { response->Clear(); @@ -240,7 +245,9 @@ void ConformanceTestSuite::RunValidInputTest( const string& equivalent_text_format, WireFormat requested_output) { TestAllTypes reference_message; GOOGLE_CHECK( - TextFormat::ParseFromString(equivalent_text_format, &reference_message)); + TextFormat::ParseFromString(equivalent_text_format, &reference_message)) + << "Failed to parse data for test case: " << test_name + << ", data: " << equivalent_text_format; ConformanceRequest request; ConformanceResponse response; @@ -254,9 +261,8 @@ void ConformanceTestSuite::RunValidInputTest( request.set_json_payload(input); break; - case conformance::UNSPECIFIED: + default: GOOGLE_LOG(FATAL) << "Unspecified input format"; - } request.set_requested_output_format(requested_output); @@ -268,8 +274,9 @@ void ConformanceTestSuite::RunValidInputTest( switch (response.result_case()) { case ConformanceResponse::kParseError: case ConformanceResponse::kRuntimeError: + case ConformanceResponse::kSerializeError: ReportFailure(test_name, request, response, - "Failed to parse valid JSON input."); + "Failed to parse JSON input or produce JSON output."); return; case ConformanceResponse::kSkipped: @@ -313,13 +320,20 @@ void ConformanceTestSuite::RunValidInputTest( break; } + + default: + GOOGLE_LOG(FATAL) << test_name << ": unknown payload type: " + << response.result_case(); } MessageDifferencer differencer; + DefaultFieldComparator field_comparator; + field_comparator.set_treat_nan_as_equal(true); + differencer.set_field_comparator(&field_comparator); string differences; differencer.ReportDifferencesToString(&differences); - if (differencer.Equals(reference_message, test_message)) { + if (differencer.Compare(reference_message, test_message)) { ReportSuccess(test_name); } else { ReportFailure(test_name, request, response, @@ -362,13 +376,103 @@ void ConformanceTestSuite::ExpectHardParseFailureForProto( void ConformanceTestSuite::RunValidJsonTest( const string& test_name, const string& input_json, const string& equivalent_text_format) { - RunValidInputTest("JsonInput." + test_name + ".JsonOutput", input_json, + RunValidInputTest("JsonInput." + test_name + ".ProtobufOutput", input_json, conformance::JSON, equivalent_text_format, conformance::PROTOBUF); - RunValidInputTest("JsonInput." + test_name + ".ProtobufOutput", input_json, conformance::JSON, + RunValidInputTest("JsonInput." + test_name + ".JsonOutput", input_json, + conformance::JSON, equivalent_text_format, + conformance::JSON); +} + +void ConformanceTestSuite::RunValidJsonTestWithProtobufInput( + const string& test_name, const TestAllTypes& input, + const string& equivalent_text_format) { + RunValidInputTest("ProtobufInput." + test_name + ".JsonOutput", + input.SerializeAsString(), conformance::PROTOBUF, equivalent_text_format, conformance::JSON); } +// According to proto3 JSON specification, JSON serializers follow more strict +// rules than parsers (e.g., a serializer must serialize int32 values as JSON +// numbers while the parser is allowed to accept them as JSON strings). This +// method allows strict checking on a proto3 JSON serializer by inspecting +// the JSON output directly. +void ConformanceTestSuite::RunValidJsonTestWithValidator( + const string& test_name, const string& input_json, + const Validator& validator) { + ConformanceRequest request; + ConformanceResponse response; + request.set_json_payload(input_json); + request.set_requested_output_format(conformance::JSON); + + string effective_test_name = "JsonInput." + test_name + ".Validator"; + + RunTest(effective_test_name, request, &response); + + if (response.result_case() != ConformanceResponse::kJsonPayload) { + ReportFailure(effective_test_name, request, response, + "Expected JSON payload but got type %d.", + response.result_case()); + return; + } + Json::Reader reader; + Json::Value value; + if (!reader.parse(response.json_payload(), value)) { + ReportFailure(effective_test_name, request, response, + "JSON payload cannot be parsed as valid JSON: %s", + reader.getFormattedErrorMessages().c_str()); + return; + } + if (!validator(value)) { + ReportFailure(effective_test_name, request, response, + "JSON payload validation failed."); + return; + } + ReportSuccess(effective_test_name); +} + +void ConformanceTestSuite::ExpectParseFailureForJson( + const string& test_name, const string& input_json) { + ConformanceRequest request; + ConformanceResponse response; + request.set_json_payload(input_json); + string effective_test_name = "JsonInput." + test_name; + + // We don't expect output, but if the program erroneously accepts the protobuf + // we let it send its response as this. We must not leave it unspecified. + request.set_requested_output_format(conformance::JSON); + + RunTest(effective_test_name, request, &response); + if (response.result_case() == ConformanceResponse::kParseError) { + ReportSuccess(effective_test_name); + } else { + ReportFailure(effective_test_name, request, response, + "Should have failed to parse, but didn't."); + } +} + +void ConformanceTestSuite::ExpectSerializeFailureForJson( + const string& test_name, const string& text_format) { + TestAllTypes payload_message; + GOOGLE_CHECK( + TextFormat::ParseFromString(text_format, &payload_message)) + << "Failed to parse: " << text_format; + + ConformanceRequest request; + ConformanceResponse response; + request.set_protobuf_payload(payload_message.SerializeAsString()); + string effective_test_name = test_name + ".JsonOutput"; + request.set_requested_output_format(conformance::JSON); + + RunTest(effective_test_name, request, &response); + if (response.result_case() == ConformanceResponse::kSerializeError) { + ReportSuccess(effective_test_name); + } else { + ReportFailure(effective_test_name, request, response, + "Should have failed to serialize, but didn't."); + } +} + void ConformanceTestSuite::TestPrematureEOFForType(FieldDescriptor::Type type) { // Incomplete values for each wire type. static const string incompletes[6] = { @@ -500,20 +604,1373 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner, RunValidJsonTest("HelloWorld", "{\"optionalString\":\"Hello, World!\"}", "optional_string: 'Hello, World!'"); - bool ok = - CheckSetEmpty(expected_to_fail_, - "These tests were listed in the failure list, but they " - "don't exist. Remove them from the failure list") && + // Test field name conventions. + RunValidJsonTest( + "FieldNameInSnakeCase", + R"({ + "fieldname1": 1, + "fieldName2": 2, + "FieldName3": 3 + })", + R"( + fieldname1: 1 + field_name2: 2 + _field_name3: 3 + )"); + RunValidJsonTest( + "FieldNameWithNumbers", + R"({ + "field0name5": 5, + "field0Name6": 6 + })", + R"( + field0name5: 5 + field_0_name6: 6 + )"); + RunValidJsonTest( + "FieldNameWithMixedCases", + R"({ + "fieldName7": 7, + "fieldName8": 8, + "fieldName9": 9, + "fieldName10": 10, + "fIELDNAME11": 11, + "fIELDName12": 12 + })", + R"( + fieldName7: 7 + FieldName8: 8 + field_Name9: 9 + Field_Name10: 10 + FIELD_NAME11: 11 + FIELD_name12: 12 + )"); + // Using the original proto field name in JSON is also allowed. + RunValidJsonTest( + "OriginalProtoFieldName", + R"({ + "fieldname1": 1, + "field_name2": 2, + "_field_name3": 3, + "field0name5": 5, + "field_0_name6": 6, + "fieldName7": 7, + "FieldName8": 8, + "field_Name9": 9, + "Field_Name10": 10, + "FIELD_NAME11": 11, + "FIELD_name12": 12 + })", + R"( + fieldname1: 1 + field_name2: 2 + _field_name3: 3 + field0name5: 5 + field_0_name6: 6 + fieldName7: 7 + FieldName8: 8 + field_Name9: 9 + Field_Name10: 10 + FIELD_NAME11: 11 + FIELD_name12: 12 + )"); + // Field names can be escaped. + RunValidJsonTest( + "FieldNameEscaped", + R"({"fieldn\u0061me1": 1})", + "fieldname1: 1"); + // Field names must be quoted (or it's not valid JSON). + ExpectParseFailureForJson( + "FieldNameNotQuoted", + "{fieldname1: 1}"); + // Trailing comma is not allowed (not valid JSON). + ExpectParseFailureForJson( + "TrailingCommaInAnObject", + R"({"fieldname1":1,})"); + // JSON doesn't support comments. + ExpectParseFailureForJson( + "JsonWithComments", + R"({ + // This is a comment. + "fieldname1": 1 + })"); + // Duplicated field names are not allowed. + ExpectParseFailureForJson( + "FieldNameDuplicate", + R"({ + "optionalNestedMessage": {a: 1}, + "optionalNestedMessage": {} + })"); + ExpectParseFailureForJson( + "FieldNameDuplicateDifferentCasing1", + R"({ + "optional_nested_message": {a: 1}, + "optionalNestedMessage": {} + })"); + ExpectParseFailureForJson( + "FieldNameDuplicateDifferentCasing2", + R"({ + "optionalNestedMessage": {a: 1}, + "optional_nested_message": {} + })"); + // Serializers should use lowerCamelCase by default. + RunValidJsonTestWithValidator( + "FieldNameInLowerCamelCase", + R"({ + "fieldname1": 1, + "fieldName2": 2, + "FieldName3": 3 + })", + [](const Json::Value& value) { + return value.isMember("fieldname1") && + value.isMember("fieldName2") && + value.isMember("FieldName3"); + }); + RunValidJsonTestWithValidator( + "FieldNameWithNumbers", + R"({ + "field0name5": 5, + "field0Name6": 6 + })", + [](const Json::Value& value) { + return value.isMember("field0name5") && + value.isMember("field0Name6"); + }); + RunValidJsonTestWithValidator( + "FieldNameWithMixedCases", + R"({ + "fieldName7": 7, + "fieldName8": 8, + "fieldName9": 9, + "fieldName10": 10, + "fIELDNAME11": 11, + "fIELDName12": 12 + })", + [](const Json::Value& value) { + return value.isMember("fieldName7") && + value.isMember("fieldName8") && + value.isMember("fieldName9") && + value.isMember("fieldName10") && + value.isMember("fIELDNAME11") && + value.isMember("fIELDName12"); + }); + + // Integer fields. + RunValidJsonTest( + "Int32FieldMaxValue", + R"({"optionalInt32": 2147483647})", + "optional_int32: 2147483647"); + RunValidJsonTest( + "Int32FieldMinValue", + R"({"optionalInt32": -2147483648})", + "optional_int32: -2147483648"); + RunValidJsonTest( + "Uint32FieldMaxValue", + R"({"optionalUint32": 4294967295})", + "optional_uint32: 4294967295"); + RunValidJsonTest( + "Int64FieldMaxValue", + R"({"optionalInt64": "9223372036854775807"})", + "optional_int64: 9223372036854775807"); + RunValidJsonTest( + "Int64FieldMinValue", + R"({"optionalInt64": "-9223372036854775808"})", + "optional_int64: -9223372036854775808"); + RunValidJsonTest( + "Uint64FieldMaxValue", + R"({"optionalUint64": "18446744073709551615"})", + "optional_uint64: 18446744073709551615"); + RunValidJsonTest( + "Int64FieldMaxValueNotQuoted", + R"({"optionalInt64": 9223372036854775807})", + "optional_int64: 9223372036854775807"); + RunValidJsonTest( + "Int64FieldMinValueNotQuoted", + R"({"optionalInt64": -9223372036854775808})", + "optional_int64: -9223372036854775808"); + RunValidJsonTest( + "Uint64FieldMaxValueNotQuoted", + R"({"optionalUint64": 18446744073709551615})", + "optional_uint64: 18446744073709551615"); + // Values can be represented as JSON strings. + RunValidJsonTest( + "Int32FieldStringValue", + R"({"optionalInt32": "2147483647"})", + "optional_int32: 2147483647"); + RunValidJsonTest( + "Int32FieldStringValueEscaped", + R"({"optionalInt32": "2\u003147483647"})", + "optional_int32: 2147483647"); + + // Parsers reject out-of-bound integer values. + ExpectParseFailureForJson( + "Int32FieldTooLarge", + R"({"optionalInt32": 2147483648})"); + ExpectParseFailureForJson( + "Int32FieldTooSmall", + R"({"optionalInt32": -2147483649})"); + ExpectParseFailureForJson( + "Uint32FieldTooLarge", + R"({"optionalUint32": 4294967296})"); + ExpectParseFailureForJson( + "Int64FieldTooLarge", + R"({"optionalInt64": "9223372036854775808"})"); + ExpectParseFailureForJson( + "Int64FieldTooSmall", + R"({"optionalInt64": "-9223372036854775809"})"); + ExpectParseFailureForJson( + "Uint64FieldTooLarge", + R"({"optionalUint64": "18446744073709551616"})"); + // Parser reject non-integer numeric values as well. + ExpectParseFailureForJson( + "Int32FieldNotInteger", + R"({"optionalInt32": 0.5})"); + ExpectParseFailureForJson( + "Uint32FieldNotInteger", + R"({"optionalUint32": 0.5})"); + ExpectParseFailureForJson( + "Int64FieldNotInteger", + R"({"optionalInt64": "0.5"})"); + ExpectParseFailureForJson( + "Uint64FieldNotInteger", + R"({"optionalUint64": "0.5"})"); + + // Integers but represented as float values are accepted. + RunValidJsonTest( + "Int32FieldFloatTrailingZero", + R"({"optionalInt32": 100000.000})", + "optional_int32: 100000"); + RunValidJsonTest( + "Int32FieldExponentialFormat", + R"({"optionalInt32": 1e5})", + "optional_int32: 100000"); + RunValidJsonTest( + "Int32FieldMaxFloatValue", + R"({"optionalInt32": 2.147483647e9})", + "optional_int32: 2147483647"); + RunValidJsonTest( + "Int32FieldMinFloatValue", + R"({"optionalInt32": -2.147483648e9})", + "optional_int32: -2147483648"); + RunValidJsonTest( + "Uint32FieldMaxFloatValue", + R"({"optionalUint32": 4.294967295e9})", + "optional_uint32: 4294967295"); + + // Parser reject non-numeric values. + ExpectParseFailureForJson( + "Int32FieldNotNumber", + R"({"optionalInt32": "3x3"})"); + ExpectParseFailureForJson( + "Uint32FieldNotNumber", + R"({"optionalUint32": "3x3"})"); + ExpectParseFailureForJson( + "Int64FieldNotNumber", + R"({"optionalInt64": "3x3"})"); + ExpectParseFailureForJson( + "Uint64FieldNotNumber", + R"({"optionalUint64": "3x3"})"); + // JSON does not allow "+" on numric values. + ExpectParseFailureForJson( + "Int32FieldPlusSign", + R"({"optionalInt32": +1})"); + // JSON doesn't allow leading 0s. + ExpectParseFailureForJson( + "Int32FieldLeadingZero", + R"({"optionalInt32": 01})"); + ExpectParseFailureForJson( + "Int32FieldNegativeWithLeadingZero", + R"({"optionalInt32": -01})"); + // String values must follow the same syntax rule. Specifically leading + // or traling spaces are not allowed. + ExpectParseFailureForJson( + "Int32FieldLeadingSpace", + R"({"optionalInt32": " 1"})"); + ExpectParseFailureForJson( + "Int32FieldTrailingSpace", + R"({"optionalInt32": "1 "})"); + + // 64-bit values are serialized as strings. + RunValidJsonTestWithValidator( + "Int64FieldBeString", + R"({"optionalInt64": 1})", + [](const Json::Value& value) { + return value["optionalInt64"].type() == Json::stringValue && + value["optionalInt64"].asString() == "1"; + }); + RunValidJsonTestWithValidator( + "Uint64FieldBeString", + R"({"optionalUint64": 1})", + [](const Json::Value& value) { + return value["optionalUint64"].type() == Json::stringValue && + value["optionalUint64"].asString() == "1"; + }); + + // Bool fields. + RunValidJsonTest( + "BoolFieldTrue", + R"({"optionalBool":true})", + "optional_bool: true"); + RunValidJsonTest( + "BoolFieldFalse", + R"({"optionalBool":false})", + "optional_bool: false"); + + // Other forms are not allowed. + ExpectParseFailureForJson( + "BoolFieldIntegerZero", + R"({"optionalBool":0})"); + ExpectParseFailureForJson( + "BoolFieldIntegerOne", + R"({"optionalBool":1})"); + ExpectParseFailureForJson( + "BoolFieldCamelCaseTrue", + R"({"optionalBool":True})"); + ExpectParseFailureForJson( + "BoolFieldCamelCaseFalse", + R"({"optionalBool":False})"); + ExpectParseFailureForJson( + "BoolFieldAllCapitalTrue", + R"({"optionalBool":TRUE})"); + ExpectParseFailureForJson( + "BoolFieldAllCapitalFalse", + R"({"optionalBool":FALSE})"); + ExpectParseFailureForJson( + "BoolFieldDoubleQuotedTrue", + R"({"optionalBool":"true"})"); + ExpectParseFailureForJson( + "BoolFieldDoubleQuotedFalse", + R"({"optionalBool":"false"})"); + + // Float fields. + RunValidJsonTest( + "FloatFieldMinPositiveValue", + R"({"optionalFloat": 1.175494e-38})", + "optional_float: 1.175494e-38"); + RunValidJsonTest( + "FloatFieldMaxNegativeValue", + R"({"optionalFloat": -1.175494e-38})", + "optional_float: -1.175494e-38"); + RunValidJsonTest( + "FloatFieldMaxPositiveValue", + R"({"optionalFloat": 3.402823e+38})", + "optional_float: 3.402823e+38"); + RunValidJsonTest( + "FloatFieldMinNegativeValue", + R"({"optionalFloat": 3.402823e+38})", + "optional_float: 3.402823e+38"); + // Values can be quoted. + RunValidJsonTest( + "FloatFieldQuotedValue", + R"({"optionalFloat": "1"})", + "optional_float: 1"); + // Special values. + RunValidJsonTest( + "FloatFieldNan", + R"({"optionalFloat": "NaN"})", + "optional_float: nan"); + RunValidJsonTest( + "FloatFieldInfinity", + R"({"optionalFloat": "Infinity"})", + "optional_float: inf"); + RunValidJsonTest( + "FloatFieldNegativeInfinity", + R"({"optionalFloat": "-Infinity"})", + "optional_float: -inf"); + // Non-cannonical Nan will be correctly normalized. + { + TestAllTypes message; + // IEEE floating-point standard 32-bit quiet NaN: + // 0111 1111 1xxx xxxx xxxx xxxx xxxx xxxx + message.set_optional_float( + WireFormatLite::DecodeFloat(0x7FA12345)); + RunValidJsonTestWithProtobufInput( + "FloatFieldNormalizeQuietNan", message, + "optional_float: nan"); + // IEEE floating-point standard 64-bit signaling NaN: + // 1111 1111 1xxx xxxx xxxx xxxx xxxx xxxx + message.set_optional_float( + WireFormatLite::DecodeFloat(0xFFB54321)); + RunValidJsonTestWithProtobufInput( + "FloatFieldNormalizeSignalingNan", message, + "optional_float: nan"); + } - CheckSetEmpty(unexpected_failing_tests_, - "These tests failed. If they can't be fixed right now, " - "you can add them to the failure list so the overall " - "suite can succeed") && + // Special values must be quoted. + ExpectParseFailureForJson( + "FloatFieldNanNotQuoted", + R"({"optionalFloat": NaN})"); + ExpectParseFailureForJson( + "FloatFieldInfinityNotQuoted", + R"({"optionalFloat": Infinity})"); + ExpectParseFailureForJson( + "FloatFieldNegativeInfinityNotQuoted", + R"({"optionalFloat": -Infinity})"); + // Parsers should reject out-of-bound values. + ExpectParseFailureForJson( + "FloatFieldTooSmall", + R"({"optionalFloat": -3.502823e+38})"); + ExpectParseFailureForJson( + "FloatFieldTooLarge", + R"({"optionalFloat": 3.502823e+38})"); + + // Double fields. + RunValidJsonTest( + "DoubleFieldMinPositiveValue", + R"({"optionalDouble": 2.22507e-308})", + "optional_double: 2.22507e-308"); + RunValidJsonTest( + "DoubleFieldMaxNegativeValue", + R"({"optionalDouble": -2.22507e-308})", + "optional_double: -2.22507e-308"); + RunValidJsonTest( + "DoubleFieldMaxPositiveValue", + R"({"optionalDouble": 1.79769e+308})", + "optional_double: 1.79769e+308"); + RunValidJsonTest( + "DoubleFieldMinNegativeValue", + R"({"optionalDouble": -1.79769e+308})", + "optional_double: -1.79769e+308"); + // Values can be quoted. + RunValidJsonTest( + "DoubleFieldQuotedValue", + R"({"optionalDouble": "1"})", + "optional_double: 1"); + // Speical values. + RunValidJsonTest( + "DoubleFieldNan", + R"({"optionalDouble": "NaN"})", + "optional_double: nan"); + RunValidJsonTest( + "DoubleFieldInfinity", + R"({"optionalDouble": "Infinity"})", + "optional_double: inf"); + RunValidJsonTest( + "DoubleFieldNegativeInfinity", + R"({"optionalDouble": "-Infinity"})", + "optional_double: -inf"); + // Non-cannonical Nan will be correctly normalized. + { + TestAllTypes message; + message.set_optional_double( + WireFormatLite::DecodeDouble(0x7FFA123456789ABCLL)); + RunValidJsonTestWithProtobufInput( + "DoubleFieldNormalizeQuietNan", message, + "optional_double: nan"); + message.set_optional_double( + WireFormatLite::DecodeDouble(0xFFFBCBA987654321LL)); + RunValidJsonTestWithProtobufInput( + "DoubleFieldNormalizeSignalingNan", message, + "optional_double: nan"); + } + + // Special values must be quoted. + ExpectParseFailureForJson( + "DoubleFieldNanNotQuoted", + R"({"optionalDouble": NaN})"); + ExpectParseFailureForJson( + "DoubleFieldInfinityNotQuoted", + R"({"optionalDouble": Infinity})"); + ExpectParseFailureForJson( + "DoubleFieldNegativeInfinityNotQuoted", + R"({"optionalDouble": -Infinity})"); + + // Parsers should reject out-of-bound values. + ExpectParseFailureForJson( + "DoubleFieldTooSmall", + R"({"optionalDouble": -1.89769e+308})"); + ExpectParseFailureForJson( + "DoubleFieldTooLarge", + R"({"optionalDouble": +1.89769e+308})"); + + // Enum fields. + RunValidJsonTest( + "EnumField", + R"({"optionalNestedEnum": "FOO"})", + "optional_nested_enum: FOO"); + // Enum values must be represented as strings. + ExpectParseFailureForJson( + "EnumFieldNotQuoted", + R"({"optionalNestedEnum": FOO})"); + // Numeric values are allowed. + RunValidJsonTest( + "EnumFieldNumericValueZero", + R"({"optionalNestedEnum": 0})", + "optional_nested_enum: FOO"); + RunValidJsonTest( + "EnumFieldNumericValueNonZero", + R"({"optionalNestedEnum": 1})", + "optional_nested_enum: BAR"); + // Unknown enum values are represented as numeric values. + RunValidJsonTestWithValidator( + "EnumFieldUnknownValue", + R"({"optionalNestedEnum": 123})", + [](const Json::Value& value) { + return value["optionalNestedEnum"].type() == Json::intValue && + value["optionalNestedEnum"].asInt() == 123; + }); + + // String fields. + RunValidJsonTest( + "StringField", + R"({"optionalString": "Hello world!"})", + "optional_string: \"Hello world!\""); + RunValidJsonTest( + "StringFieldUnicode", + // Google in Chinese. + R"({"optionalString": "谷歌"})", + R"(optional_string: "谷歌")"); + RunValidJsonTest( + "StringFieldEscape", + R"({"optionalString": "\"\\\/\b\f\n\r\t"})", + R"(optional_string: "\"\\/\b\f\n\r\t")"); + RunValidJsonTest( + "StringFieldUnicodeEscape", + R"({"optionalString": "\u8C37\u6B4C"})", + R"(optional_string: "谷歌")"); + RunValidJsonTest( + "StringFieldUnicodeEscapeWithLowercaseHexLetters", + R"({"optionalString": "\u8c37\u6b4c"})", + R"(optional_string: "谷歌")"); + RunValidJsonTest( + "StringFieldSurrogatePair", + // The character is an emoji: grinning face with smiling eyes. 😁 + R"({"optionalString": "\uD83D\uDE01"})", + R"(optional_string: "\xF0\x9F\x98\x81")"); + + // Unicode escapes must start with "\u" (lowercase u). + ExpectParseFailureForJson( + "StringFieldUppercaseEscapeLetter", + R"({"optionalString": "\U8C37\U6b4C"})"); + ExpectParseFailureForJson( + "StringFieldInvalidEscape", + R"({"optionalString": "\uXXXX\u6B4C"})"); + ExpectParseFailureForJson( + "StringFieldUnterminatedEscape", + R"({"optionalString": "\u8C3"})"); + ExpectParseFailureForJson( + "StringFieldUnpairedHighSurrogate", + R"({"optionalString": "\uD800"})"); + ExpectParseFailureForJson( + "StringFieldUnpairedLowSurrogate", + R"({"optionalString": "\uDC00"})"); + ExpectParseFailureForJson( + "StringFieldSurrogateInWrongOrder", + R"({"optionalString": "\uDE01\uD83D"})"); + ExpectParseFailureForJson( + "StringFieldNotAString", + R"({"optionalString": 12345})"); + + // Bytes fields. + RunValidJsonTest( + "BytesField", + R"({"optionalBytes": "AQI="})", + R"(optional_bytes: "\x01\x02")"); + ExpectParseFailureForJson( + "BytesFieldNoPadding", + R"({"optionalBytes": "AQI"})"); + ExpectParseFailureForJson( + "BytesFieldInvalidBase64Characters", + R"({"optionalBytes": "-_=="})"); + + // Message fields. + RunValidJsonTest( + "MessageField", + R"({"optionalNestedMessage": {"a": 1234}})", + "optional_nested_message: {a: 1234}"); + + // Oneof fields. + ExpectParseFailureForJson( + "OneofFieldDuplicate", + R"({"oneofUint32": 1, "oneofString": "test"})"); + + // Repeated fields. + RunValidJsonTest( + "PrimitiveRepeatedField", + R"({"repeatedInt32": [1, 2, 3, 4]})", + "repeated_int32: [1, 2, 3, 4]"); + RunValidJsonTest( + "EnumRepeatedField", + R"({"repeatedNestedEnum": ["FOO", "BAR", "BAZ"]})", + "repeated_nested_enum: [FOO, BAR, BAZ]"); + RunValidJsonTest( + "StringRepeatedField", + R"({"repeatedString": ["Hello", "world"]})", + R"(repeated_string: ["Hello", "world"])"); + RunValidJsonTest( + "BytesRepeatedField", + R"({"repeatedBytes": ["AAEC", "AQI="]})", + R"(repeated_bytes: ["\x00\x01\x02", "\x01\x02"])"); + RunValidJsonTest( + "MessageRepeatedField", + R"({"repeatedNestedMessage": [{"a": 1234}, {"a": 5678}]})", + "repeated_nested_message: {a: 1234}" + "repeated_nested_message: {a: 5678}"); + + // Repeated field elements are of incorrect type. + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingIntegersGotBool", + R"({"repeatedInt32": [1, false, 3, 4]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingIntegersGotString", + R"({"repeatedInt32": [1, 2, "name", 4]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingIntegersGotMessage", + R"({"repeatedInt32": [1, 2, 3, {"a": 4}]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingStringsGotInt", + R"({"repeatedString": ["1", 2, "3", "4"]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingStringsGotBool", + R"({"repeatedString": ["1", "2", false, "4"]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingStringsGotMessage", + R"({"repeatedString": ["1", 2, "3", {"a": 4}]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingMessagesGotInt", + R"({"repeatedNestedMessage": [{"a": 1}, 2]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingMessagesGotBool", + R"({"repeatedNestedMessage": [{"a": 1}, false]})"); + ExpectParseFailureForJson( + "RepeatedFieldWrongElementTypeExpectingMessagesGotString", + R"({"repeatedNestedMessage": [{"a": 1}, "2"]})"); + // Trailing comma in the repeated field is not allowed. + ExpectParseFailureForJson( + "RepeatedFieldTrailingComma", + R"({"repeatedInt32": [1, 2, 3, 4,]})"); + + // Map fields. + RunValidJsonTest( + "Int32MapField", + R"({"mapInt32Int32": {"1": 2, "3": 4}})", + "map_int32_int32: {key: 1 value: 2}" + "map_int32_int32: {key: 3 value: 4}"); + ExpectParseFailureForJson( + "Int32MapFieldKeyNotQuoted", + R"({"mapInt32Int32": {1: 2, 3: 4}})"); + RunValidJsonTest( + "Uint32MapField", + R"({"mapUint32Uint32": {"1": 2, "3": 4}})", + "map_uint32_uint32: {key: 1 value: 2}" + "map_uint32_uint32: {key: 3 value: 4}"); + ExpectParseFailureForJson( + "Uint32MapFieldKeyNotQuoted", + R"({"mapUint32Uint32": {1: 2, 3: 4}})"); + RunValidJsonTest( + "Int64MapField", + R"({"mapInt64Int64": {"1": 2, "3": 4}})", + "map_int64_int64: {key: 1 value: 2}" + "map_int64_int64: {key: 3 value: 4}"); + ExpectParseFailureForJson( + "Int64MapFieldKeyNotQuoted", + R"({"mapInt64Int64": {1: 2, 3: 4}})"); + RunValidJsonTest( + "Uint64MapField", + R"({"mapUint64Uint64": {"1": 2, "3": 4}})", + "map_uint64_uint64: {key: 1 value: 2}" + "map_uint64_uint64: {key: 3 value: 4}"); + ExpectParseFailureForJson( + "Uint64MapFieldKeyNotQuoted", + R"({"mapUint64Uint64": {1: 2, 3: 4}})"); + RunValidJsonTest( + "BoolMapField", + R"({"mapBoolBool": {"true": true, "false": false}})", + "map_bool_bool: {key: true value: true}" + "map_bool_bool: {key: false value: false}"); + ExpectParseFailureForJson( + "BoolMapFieldKeyNotQuoted", + R"({"mapBoolBool": {true: true, false: false}})"); + RunValidJsonTest( + "MessageMapField", + R"({ + "mapStringNestedMessage": { + "hello": {"a": 1234}, + "world": {"a": 5678} + } + })", + R"( + map_string_nested_message: { + key: "hello" + value: {a: 1234} + } + map_string_nested_message: { + key: "world" + value: {a: 5678} + } + )"); + // Since Map keys are represented as JSON strings, escaping should be allowed. + RunValidJsonTest( + "Int32MapEscapedKey", + R"({"mapInt32Int32": {"\u0031": 2}})", + "map_int32_int32: {key: 1 value: 2}"); + RunValidJsonTest( + "Int64MapEscapedKey", + R"({"mapInt64Int64": {"\u0031": 2}})", + "map_int64_int64: {key: 1 value: 2}"); + RunValidJsonTest( + "BoolMapEscapedKey", + R"({"mapBoolBool": {"tr\u0075e": true}})", + "map_bool_bool: {key: true value: true}"); + + // "null" is accepted for all fields types. + RunValidJsonTest( + "AllFieldAcceptNull", + R"({ + "optionalInt32": null, + "optionalInt64": null, + "optionalUint32": null, + "optionalUint64": null, + "optionalBool": null, + "optionalString": null, + "optionalBytes": null, + "optionalNestedEnum": null, + "optionalNestedMessage": null, + "repeatedInt32": null, + "repeatedInt64": null, + "repeatedUint32": null, + "repeatedUint64": null, + "repeatedBool": null, + "repeatedString": null, + "repeatedBytes": null, + "repeatedNestedEnum": null, + "repeatedNestedMessage": null, + "mapInt32Int32": null, + "mapBoolBool": null, + "mapStringNestedMessage": null + })", + ""); + + // Repeated field elements cannot be null. + ExpectParseFailureForJson( + "RepeatedFieldPrimitiveElementIsNull", + R"({"repeatedInt32": [1, null, 2]})"); + ExpectParseFailureForJson( + "RepeatedFieldMessageElementIsNull", + R"({"repeatedNestedMessage": [{"a":1}, null, {"a":2}]})"); + // Map field keys cannot be null. + ExpectParseFailureForJson( + "MapFieldKeyIsNull", + R"({"mapInt32Int32": {null: 1}})"); + // Map field values cannot be null. + ExpectParseFailureForJson( + "MapFieldValueIsNull", + R"({"mapInt32Int32": {"0": null}})"); + + // Wrapper types. + RunValidJsonTest( + "OptionalBoolWrapper", + R"({"optionalBoolWrapper": false})", + "optional_bool_wrapper: {value: false}"); + RunValidJsonTest( + "OptionalInt32Wrapper", + R"({"optionalInt32Wrapper": 0})", + "optional_int32_wrapper: {value: 0}"); + RunValidJsonTest( + "OptionalUint32Wrapper", + R"({"optionalUint32Wrapper": 0})", + "optional_uint32_wrapper: {value: 0}"); + RunValidJsonTest( + "OptionalInt64Wrapper", + R"({"optionalInt64Wrapper": 0})", + "optional_int64_wrapper: {value: 0}"); + RunValidJsonTest( + "OptionalUint64Wrapper", + R"({"optionalUint64Wrapper": 0})", + "optional_uint64_wrapper: {value: 0}"); + RunValidJsonTest( + "OptionalFloatWrapper", + R"({"optionalFloatWrapper": 0})", + "optional_float_wrapper: {value: 0}"); + RunValidJsonTest( + "OptionalDoubleWrapper", + R"({"optionalDoubleWrapper": 0})", + "optional_double_wrapper: {value: 0}"); + RunValidJsonTest( + "OptionalStringWrapper", + R"({"optionalStringWrapper": ""})", + R"(optional_string_wrapper: {value: ""})"); + RunValidJsonTest( + "OptionalBytesWrapper", + R"({"optionalBytesWrapper": ""})", + R"(optional_bytes_wrapper: {value: ""})"); + RunValidJsonTest( + "OptionalWrapperTypesWithNonDefaultValue", + R"({ + "optionalBoolWrapper": true, + "optionalInt32Wrapper": 1, + "optionalUint32Wrapper": 1, + "optionalInt64Wrapper": "1", + "optionalUint64Wrapper": "1", + "optionalFloatWrapper": 1, + "optionalDoubleWrapper": 1, + "optionalStringWrapper": "1", + "optionalBytesWrapper": "AQI=" + })", + R"( + optional_bool_wrapper: {value: true} + optional_int32_wrapper: {value: 1} + optional_uint32_wrapper: {value: 1} + optional_int64_wrapper: {value: 1} + optional_uint64_wrapper: {value: 1} + optional_float_wrapper: {value: 1} + optional_double_wrapper: {value: 1} + optional_string_wrapper: {value: "1"} + optional_bytes_wrapper: {value: "\x01\x02"} + )"); + RunValidJsonTest( + "RepeatedBoolWrapper", + R"({"repeatedBoolWrapper": [true, false]})", + "repeated_bool_wrapper: {value: true}" + "repeated_bool_wrapper: {value: false}"); + RunValidJsonTest( + "RepeatedInt32Wrapper", + R"({"repeatedInt32Wrapper": [0, 1]})", + "repeated_int32_wrapper: {value: 0}" + "repeated_int32_wrapper: {value: 1}"); + RunValidJsonTest( + "RepeatedUint32Wrapper", + R"({"repeatedUint32Wrapper": [0, 1]})", + "repeated_uint32_wrapper: {value: 0}" + "repeated_uint32_wrapper: {value: 1}"); + RunValidJsonTest( + "RepeatedInt64Wrapper", + R"({"repeatedInt64Wrapper": [0, 1]})", + "repeated_int64_wrapper: {value: 0}" + "repeated_int64_wrapper: {value: 1}"); + RunValidJsonTest( + "RepeatedUint64Wrapper", + R"({"repeatedUint64Wrapper": [0, 1]})", + "repeated_uint64_wrapper: {value: 0}" + "repeated_uint64_wrapper: {value: 1}"); + RunValidJsonTest( + "RepeatedFloatWrapper", + R"({"repeatedFloatWrapper": [0, 1]})", + "repeated_float_wrapper: {value: 0}" + "repeated_float_wrapper: {value: 1}"); + RunValidJsonTest( + "RepeatedDoubleWrapper", + R"({"repeatedDoubleWrapper": [0, 1]})", + "repeated_double_wrapper: {value: 0}" + "repeated_double_wrapper: {value: 1}"); + RunValidJsonTest( + "RepeatedStringWrapper", + R"({"repeatedStringWrapper": ["", "AQI="]})", + R"( + repeated_string_wrapper: {value: ""} + repeated_string_wrapper: {value: "AQI="} + )"); + RunValidJsonTest( + "RepeatedBytesWrapper", + R"({"repeatedBytesWrapper": ["", "AQI="]})", + R"( + repeated_bytes_wrapper: {value: ""} + repeated_bytes_wrapper: {value: "\x01\x02"} + )"); + RunValidJsonTest( + "WrapperTypesWithNullValue", + R"({ + "optionalBoolWrapper": null, + "optionalInt32Wrapper": null, + "optionalUint32Wrapper": null, + "optionalInt64Wrapper": null, + "optionalUint64Wrapper": null, + "optionalFloatWrapper": null, + "optionalDoubleWrapper": null, + "optionalStringWrapper": null, + "optionalBytesWrapper": null, + "repeatedBoolWrapper": null, + "repeatedInt32Wrapper": null, + "repeatedUint32Wrapper": null, + "repeatedInt64Wrapper": null, + "repeatedUint64Wrapper": null, + "repeatedFloatWrapper": null, + "repeatedDoubleWrapper": null, + "repeatedStringWrapper": null, + "repeatedBytesWrapper": null + })", + ""); + + // Duration + RunValidJsonTest( + "DurationMinValue", + R"({"optionalDuration": "-315576000000.999999999s"})", + "optional_duration: {seconds: -315576000000 nanos: -999999999}"); + RunValidJsonTest( + "DurationMaxValue", + R"({"optionalDuration": "315576000000.999999999s"})", + "optional_duration: {seconds: 315576000000 nanos: 999999999}"); + RunValidJsonTest( + "DurationRepeatedValue", + R"({"repeatedDuration": ["1.5s", "-1.5s"]})", + "repeated_duration: {seconds: 1 nanos: 500000000}" + "repeated_duration: {seconds: -1 nanos: -500000000}"); + + ExpectParseFailureForJson( + "DurationMissingS", + R"({"optionalDuration": "1"})"); + ExpectParseFailureForJson( + "DurationJsonInputTooSmall", + R"({"optionalDuration": "-315576000001.000000000s"})"); + ExpectParseFailureForJson( + "DurationJsonInputTooLarge", + R"({"optionalDuration": "315576000001.000000000s"})"); + ExpectSerializeFailureForJson( + "DurationProtoInputTooSmall", + "optional_duration: {seconds: -315576000001 nanos: 0}"); + ExpectSerializeFailureForJson( + "DurationProtoInputTooLarge", + "optional_duration: {seconds: 315576000001 nanos: 0}"); + + RunValidJsonTestWithValidator( + "DurationHasZeroFractionalDigit", + R"({"optionalDuration": "1.000000000s"})", + [](const Json::Value& value) { + return value["optionalDuration"].asString() == "1s"; + }); + RunValidJsonTestWithValidator( + "DurationHas3FractionalDigits", + R"({"optionalDuration": "1.010000000s"})", + [](const Json::Value& value) { + return value["optionalDuration"].asString() == "1.010s"; + }); + RunValidJsonTestWithValidator( + "DurationHas6FractionalDigits", + R"({"optionalDuration": "1.000010000s"})", + [](const Json::Value& value) { + return value["optionalDuration"].asString() == "1.000010s"; + }); + RunValidJsonTestWithValidator( + "DurationHas9FractionalDigits", + R"({"optionalDuration": "1.000000010s"})", + [](const Json::Value& value) { + return value["optionalDuration"].asString() == "1.000000010s"; + }); + + // Timestamp + RunValidJsonTest( + "TimestampMinValue", + R"({"optionalTimestamp": "0001-01-01T00:00:00Z"})", + "optional_timestamp: {seconds: -62135596800}"); + RunValidJsonTest( + "TimestampMaxValue", + R"({"optionalTimestamp": "9999-12-31T23:59:59.999999999Z"})", + "optional_timestamp: {seconds: 253402300799 nanos: 999999999}"); + RunValidJsonTest( + "TimestampRepeatedValue", + R"({ + "repeatedTimestamp": [ + "0001-01-01T00:00:00Z", + "9999-12-31T23:59:59.999999999Z" + ] + })", + "repeated_timestamp: {seconds: -62135596800}" + "repeated_timestamp: {seconds: 253402300799 nanos: 999999999}"); + RunValidJsonTest( + "TimestampWithPositiveOffset", + R"({"optionalTimestamp": "1970-01-01T08:00:00+08:00"})", + "optional_timestamp: {seconds: 0}"); + RunValidJsonTest( + "TimestampWithNegativeOffset", + R"({"optionalTimestamp": "1969-12-31T16:00:00-08:00"})", + "optional_timestamp: {seconds: 0}"); + + ExpectParseFailureForJson( + "TimestampJsonInputTooSmall", + R"({"optionalTimestamp": "0000-01-01T00:00:00Z"})"); + ExpectParseFailureForJson( + "TimestampJsonInputTooLarge", + R"({"optionalTimestamp": "10000-01-01T00:00:00Z"})"); + ExpectParseFailureForJson( + "TimestampJsonInputMissingZ", + R"({"optionalTimestamp": "0001-01-01T00:00:00"})"); + ExpectParseFailureForJson( + "TimestampJsonInputMissingT", + R"({"optionalTimestamp": "0001-01-01 00:00:00Z"})"); + ExpectParseFailureForJson( + "TimestampJsonInputLowercaseZ", + R"({"optionalTimestamp": "0001-01-01T00:00:00z"})"); + ExpectParseFailureForJson( + "TimestampJsonInputLowercaseT", + R"({"optionalTimestamp": "0001-01-01t00:00:00Z"})"); + ExpectSerializeFailureForJson( + "TimestampProtoInputTooSmall", + "optional_timestamp: {seconds: -62135596801}"); + ExpectSerializeFailureForJson( + "TimestampProtoInputTooLarge", + "optional_timestamp: {seconds: 253402300800}"); + RunValidJsonTestWithValidator( + "TimestampZeroNormalized", + R"({"optionalTimestamp": "1969-12-31T16:00:00-08:00"})", + [](const Json::Value& value) { + return value["optionalTimestamp"].asString() == + "1970-01-01T00:00:00Z"; + }); + RunValidJsonTestWithValidator( + "TimestampHasZeroFractionalDigit", + R"({"optionalTimestamp": "1970-01-01T00:00:00.000000000Z"})", + [](const Json::Value& value) { + return value["optionalTimestamp"].asString() == + "1970-01-01T00:00:00Z"; + }); + RunValidJsonTestWithValidator( + "TimestampHas3FractionalDigits", + R"({"optionalTimestamp": "1970-01-01T00:00:00.010000000Z"})", + [](const Json::Value& value) { + return value["optionalTimestamp"].asString() == + "1970-01-01T00:00:00.010Z"; + }); + RunValidJsonTestWithValidator( + "TimestampHas6FractionalDigits", + R"({"optionalTimestamp": "1970-01-01T00:00:00.000010000Z"})", + [](const Json::Value& value) { + return value["optionalTimestamp"].asString() == + "1970-01-01T00:00:00.000010Z"; + }); + RunValidJsonTestWithValidator( + "TimestampHas9FractionalDigits", + R"({"optionalTimestamp": "1970-01-01T00:00:00.000000010Z"})", + [](const Json::Value& value) { + return value["optionalTimestamp"].asString() == + "1970-01-01T00:00:00.000000010Z"; + }); + + // FieldMask + RunValidJsonTest( + "FieldMask", + R"({"optionalFieldMask": "foo,barBaz"})", + R"(optional_field_mask: {paths: "foo" paths: "bar_baz"})"); + ExpectParseFailureForJson( + "FieldMaskInvalidCharacter", + R"({"optionalFieldMask": "foo,bar_bar"})"); + ExpectSerializeFailureForJson( + "FieldMaskPathsDontRoundTrip", + R"(optional_field_mask: {paths: "fooBar"})"); + ExpectSerializeFailureForJson( + "FieldMaskNumbersDontRoundTrip", + R"(optional_field_mask: {paths: "foo_3_bar"})"); + ExpectSerializeFailureForJson( + "FieldMaskTooManyUnderscore", + R"(optional_field_mask: {paths: "foo__bar"})"); + + // Struct + RunValidJsonTest( + "Struct", + R"({ + "optionalStruct": { + "nullValue": null, + "intValue": 1234, + "boolValue": true, + "doubleValue": 1234.5678, + "stringValue": "Hello world!", + "listValue": [1234, "5678"], + "objectValue": { + "value": 0 + } + } + })", + R"( + optional_struct: { + fields: { + key: "nullValue" + value: {null_value: NULL_VALUE} + } + fields: { + key: "intValue" + value: {number_value: 1234} + } + fields: { + key: "boolValue" + value: {bool_value: true} + } + fields: { + key: "doubleValue" + value: {number_value: 1234.5678} + } + fields: { + key: "stringValue" + value: {string_value: "Hello world!"} + } + fields: { + key: "listValue" + value: { + list_value: { + values: { + number_value: 1234 + } + values: { + string_value: "5678" + } + } + } + } + fields: { + key: "objectValue" + value: { + struct_value: { + fields: { + key: "value" + value: { + number_value: 0 + } + } + } + } + } + } + )"); + // Value + RunValidJsonTest( + "ValueAcceptInteger", + R"({"optionalValue": 1})", + "optional_value: { number_value: 1}"); + RunValidJsonTest( + "ValueAcceptFloat", + R"({"optionalValue": 1.5})", + "optional_value: { number_value: 1.5}"); + RunValidJsonTest( + "ValueAcceptBool", + R"({"optionalValue": false})", + "optional_value: { bool_value: false}"); + RunValidJsonTest( + "ValueAcceptNull", + R"({"optionalValue": null})", + "optional_value: { null_value: NULL_VALUE}"); + RunValidJsonTest( + "ValueAcceptString", + R"({"optionalValue": "hello"})", + R"(optional_value: { string_value: "hello"})"); + RunValidJsonTest( + "ValueAcceptList", + R"({"optionalValue": [0, "hello"]})", + R"( + optional_value: { + list_value: { + values: { + number_value: 0 + } + values: { + string_value: "hello" + } + } + } + )"); + RunValidJsonTest( + "ValueAcceptObject", + R"({"optionalValue": {"value": 1}})", + R"( + optional_value: { + struct_value: { + fields: { + key: "value" + value: { + number_value: 1 + } + } + } + } + )"); + + // Any + RunValidJsonTest( + "Any", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/conformance.TestAllTypes", + "optionalInt32": 12345 + } + })", + R"( + optional_any: { + [type.googleapis.com/conformance.TestAllTypes] { + optional_int32: 12345 + } + } + )"); + RunValidJsonTest( + "AnyNested", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Any", + "value": { + "@type": "type.googleapis.com/conformance.TestAllTypes", + "optionalInt32": 12345 + } + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Any] { + [type.googleapis.com/conformance.TestAllTypes] { + optional_int32: 12345 + } + } + } + )"); + // The special "@type" tag is not required to appear first. + RunValidJsonTest( + "AnyUnorderedTypeTag", + R"({ + "optionalAny": { + "optionalInt32": 12345, + "@type": "type.googleapis.com/conformance.TestAllTypes" + } + })", + R"( + optional_any: { + [type.googleapis.com/conformance.TestAllTypes] { + optional_int32: 12345 + } + } + )"); + // Well-known types in Any. + RunValidJsonTest( + "AnyWithInt32ValueWrapper", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Int32Value", + "value": 12345 + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Int32Value] { + value: 12345 + } + } + )"); + RunValidJsonTest( + "AnyWithDuration", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Duration", + "value": "1.5s" + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Duration] { + seconds: 1 + nanos: 500000000 + } + } + )"); + RunValidJsonTest( + "AnyWithTimestamp", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Timestamp", + "value": "1970-01-01T00:00:00Z" + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Timestamp] { + seconds: 0 + nanos: 0 + } + } + )"); + RunValidJsonTest( + "AnyWithFieldMask", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.FieldMask", + "value": "foo,barBaz" + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.FieldMask] { + paths: ["foo", "bar_baz"] + } + } + )"); + RunValidJsonTest( + "AnyWithStruct", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Struct", + "value": { + "foo": 1 + } + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Struct] { + fields: { + key: "foo" + value: { + number_value: 1 + } + } + } + } + )"); + RunValidJsonTest( + "AnyWithValueForJsonObject", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Value", + "value": { + "foo": 1 + } + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Value] { + struct_value: { + fields: { + key: "foo" + value: { + number_value: 1 + } + } + } + } + } + )"); + RunValidJsonTest( + "AnyWithValueForInteger", + R"({ + "optionalAny": { + "@type": "type.googleapis.com/google.protobuf.Value", + "value": 1 + } + })", + R"( + optional_any: { + [type.googleapis.com/google.protobuf.Value] { + number_value: 1 + } + } + )"); + + bool ok = true; + if (!CheckSetEmpty(expected_to_fail_, + "These tests were listed in the failure list, but they " + "don't exist. Remove them from the failure list")) { + ok = false; + } + if (!CheckSetEmpty(unexpected_failing_tests_, + "These tests failed. If they can't be fixed right now, " + "you can add them to the failure list so the overall " + "suite can succeed")) { + ok = false; + } - CheckSetEmpty(unexpected_succeeding_tests_, - "These tests succeeded, even though they were listed in " - "the failure list. Remove them from the failure list"); + // Sometimes the testee may be fixed before we update the failure list (e.g., + // the testee is from a different component). We warn about this case but + // don't consider it an overall test failure. + CheckSetEmpty(unexpected_succeeding_tests_, + "These tests succeeded, even though they were listed in " + "the failure list. Remove them from the failure list"); + CheckSetEmpty(skipped_, + "These tests were skipped (probably because support for some " + "features is not implemented)"); if (verbose_) { CheckSetEmpty(skipped_, "These tests were skipped (probably because support for some " diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index 9e6cdaee..b89d3700 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -38,14 +38,19 @@ #ifndef CONFORMANCE_CONFORMANCE_TEST_H #define CONFORMANCE_CONFORMANCE_TEST_H +#include #include #include #include #include +#include "third_party/jsoncpp/value.h" +#include "third_party/jsoncpp/reader.h" + namespace conformance { class ConformanceRequest; class ConformanceResponse; +class TestAllTypes; } // namespace conformance namespace google { @@ -53,6 +58,8 @@ namespace protobuf { class ConformanceTestRunner { public: + virtual ~ConformanceTestRunner() {} + // Call to run a single conformance test. // // "input" is a serialized conformance.ConformanceRequest. @@ -60,7 +67,9 @@ class ConformanceTestRunner { // // If there is any error in running the test itself, set "runtime_error" in // the response. - virtual void RunTest(const std::string& input, std::string* output) = 0; + virtual void RunTest(const std::string& test_name, + const std::string& input, + std::string* output) = 0; }; // Class representing the test suite itself. To run it, implement your own @@ -118,6 +127,18 @@ class ConformanceTestSuite { conformance::WireFormat requested_output); void RunValidJsonTest(const string& test_name, const string& input_json, const string& equivalent_text_format); + void RunValidJsonTestWithProtobufInput(const string& test_name, + const conformance::TestAllTypes& input, + const string& equivalent_text_format); + + typedef std::function Validator; + void RunValidJsonTestWithValidator(const string& test_name, + const string& input_json, + const Validator& validator); + void ExpectParseFailureForJson(const string& test_name, + const string& input_json); + void ExpectSerializeFailureForJson(const string& test_name, + const string& text_format); void ExpectParseFailureForProto(const std::string& proto, const std::string& test_name); void ExpectHardParseFailureForProto(const std::string& proto, diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index c3b3db22..f2b0dabf 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -53,6 +53,7 @@ // 3. testee sends 4-byte length M (little endian) // 4. testee sends M bytes representing a ConformanceResponse proto +#include #include #include #include @@ -80,13 +81,19 @@ using std::vector; class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { public: ForkPipeRunner(const std::string &executable) - : executable_(executable), running_(false) {} + : running_(false), executable_(executable) {} - void RunTest(const std::string& request, std::string* response) { + virtual ~ForkPipeRunner() {} + + void RunTest(const std::string& test_name, + const std::string& request, + std::string* response) { if (!running_) { SpawnTestProgram(); } + current_test_name_ = test_name; + uint32_t len = request.size(); CheckedWrite(write_fd_, &len, sizeof(uint32_t)); CheckedWrite(write_fd_, request.c_str(), request.size()); @@ -158,7 +165,9 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { void CheckedWrite(int fd, const void *buf, size_t len) { if (write(fd, buf, len) != len) { - GOOGLE_LOG(FATAL) << "Error writing to test program: " << strerror(errno); + GOOGLE_LOG(FATAL) << current_test_name_ + << ": error writing to test program: " + << strerror(errno); } } @@ -168,9 +177,12 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { ssize_t bytes_read = read(fd, (char*)buf + ofs, len); if (bytes_read == 0) { - GOOGLE_LOG(FATAL) << "Unexpected EOF from test program"; + GOOGLE_LOG(FATAL) << current_test_name_ + << ": unexpected EOF from test program"; } else if (bytes_read < 0) { - GOOGLE_LOG(FATAL) << "Error reading from test program: " << strerror(errno); + GOOGLE_LOG(FATAL) << current_test_name_ + << ": error reading from test program: " + << strerror(errno); } len -= bytes_read; @@ -182,6 +194,7 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner { int read_fd_; bool running_; std::string executable_; + std::string current_test_name_; }; void UsageError() { @@ -223,7 +236,6 @@ void ParseFailureList(const char *filename, vector* failure_list) { } int main(int argc, char *argv[]) { - int arg = 1; char *program; google::protobuf::ConformanceTestSuite suite; diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index 53d90c99..240d4118 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt @@ -7,6 +7,92 @@ # TODO(haberman): insert links to corresponding bugs tracking the issue. # Should we use GitHub issues or the Google-internal bug tracker? +FieldMaskNumbersDontRoundTrip.JsonOutput +FieldMaskPathsDontRoundTrip.JsonOutput +FieldMaskTooManyUnderscore.JsonOutput +JsonInput.AnyUnorderedTypeTag.JsonOutput +JsonInput.AnyUnorderedTypeTag.ProtobufOutput +JsonInput.AnyWithValueForInteger.JsonOutput +JsonInput.AnyWithValueForInteger.ProtobufOutput +JsonInput.AnyWithValueForJsonObject.JsonOutput +JsonInput.AnyWithValueForJsonObject.ProtobufOutput +JsonInput.BoolFieldDoubleQuotedFalse +JsonInput.BoolFieldDoubleQuotedTrue +JsonInput.BoolFieldIntegerOne +JsonInput.BoolFieldIntegerZero +JsonInput.BytesFieldInvalidBase64Characters +JsonInput.BytesFieldNoPadding +JsonInput.DoubleFieldTooSmall +JsonInput.DurationHasZeroFractionalDigit.Validator +JsonInput.DurationJsonInputTooLarge +JsonInput.DurationJsonInputTooSmall +JsonInput.DurationMissingS +JsonInput.EnumFieldUnknownValue.Validator +JsonInput.FieldMaskInvalidCharacter +JsonInput.FieldNameDuplicate +JsonInput.FieldNameDuplicateDifferentCasing1 +JsonInput.FieldNameDuplicateDifferentCasing2 +JsonInput.FieldNameInLowerCamelCase.Validator +JsonInput.FieldNameInSnakeCase.JsonOutput +JsonInput.FieldNameInSnakeCase.ProtobufOutput +JsonInput.FieldNameNotQuoted +JsonInput.FieldNameWithMixedCases.JsonOutput +JsonInput.FieldNameWithMixedCases.ProtobufOutput +JsonInput.FieldNameWithMixedCases.Validator +JsonInput.FloatFieldTooLarge +JsonInput.FloatFieldTooSmall +JsonInput.Int32FieldLeadingSpace +JsonInput.Int32FieldLeadingZero +JsonInput.Int32FieldMinFloatValue.JsonOutput +JsonInput.Int32FieldMinFloatValue.ProtobufOutput +JsonInput.Int32FieldMinValue.JsonOutput +JsonInput.Int32FieldMinValue.ProtobufOutput +JsonInput.Int32FieldNegativeWithLeadingZero +JsonInput.Int32FieldNotInteger +JsonInput.Int32FieldNotNumber +JsonInput.Int32FieldTooLarge +JsonInput.Int32FieldTooSmall +JsonInput.Int32FieldTrailingSpace +JsonInput.Int64FieldNotInteger +JsonInput.Int64FieldNotNumber +JsonInput.Int64FieldTooLarge +JsonInput.Int64FieldTooSmall +JsonInput.MapFieldValueIsNull +JsonInput.OneofFieldDuplicate +JsonInput.RepeatedFieldMessageElementIsNull +JsonInput.RepeatedFieldPrimitiveElementIsNull +JsonInput.RepeatedFieldTrailingComma +JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool +JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotMessage +JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotString +JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotBool +JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotInt +JsonInput.RepeatedFieldWrongElementTypeExpectingMessagesGotString +JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool +JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt +JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotMessage +JsonInput.StringFieldNotAString +JsonInput.StringFieldSurrogateInWrongOrder +JsonInput.StringFieldSurrogatePair.JsonOutput +JsonInput.StringFieldSurrogatePair.ProtobufOutput +JsonInput.StringFieldUnpairedHighSurrogate +JsonInput.StringFieldUnpairedLowSurrogate +JsonInput.StringFieldUppercaseEscapeLetter +JsonInput.TimestampJsonInputLowercaseT +JsonInput.TimestampJsonInputLowercaseZ +JsonInput.TimestampJsonInputMissingT +JsonInput.TimestampJsonInputMissingZ +JsonInput.TimestampJsonInputTooLarge +JsonInput.TimestampJsonInputTooSmall +JsonInput.TrailingCommaInAnObject +JsonInput.Uint32FieldNotInteger +JsonInput.Uint32FieldNotNumber +JsonInput.Uint32FieldTooLarge +JsonInput.Uint64FieldNotInteger +JsonInput.Uint64FieldNotNumber +JsonInput.Uint64FieldTooLarge +JsonInput.WrapperTypesWithNullValue.JsonOutput +JsonInput.WrapperTypesWithNullValue.ProtobufOutput ProtobufInput.PrematureEofBeforeKnownRepeatedValue.MESSAGE ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE @@ -19,3 +105,5 @@ ProtobufInput.PrematureEofInPackedField.SINT64 ProtobufInput.PrematureEofInPackedField.UINT32 ProtobufInput.PrematureEofInPackedField.UINT64 ProtobufInput.PrematureEofInsideKnownRepeatedValue.MESSAGE +TimestampProtoInputTooLarge.JsonOutput +TimestampProtoInputTooSmall.JsonOutput diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt new file mode 100644 index 00000000..86af18b9 --- /dev/null +++ b/conformance/failure_list_java.txt @@ -0,0 +1,74 @@ +# This is the list of conformance tests that are known to fail for the Java +# implementation right now. These should be fixed. +# +# By listing them here we can keep tabs on which ones are failing and be sure +# that we don't introduce regressions in other tests. + +FieldMaskNumbersDontRoundTrip.JsonOutput +FieldMaskPathsDontRoundTrip.JsonOutput +FieldMaskTooManyUnderscore.JsonOutput +JsonInput.AnyWithFieldMask.ProtobufOutput +JsonInput.AnyWithValueForInteger.JsonOutput +JsonInput.AnyWithValueForJsonObject.JsonOutput +JsonInput.BoolFieldAllCapitalFalse +JsonInput.BoolFieldAllCapitalTrue +JsonInput.BoolFieldCamelCaseFalse +JsonInput.BoolFieldCamelCaseTrue +JsonInput.BoolFieldDoubleQuotedFalse +JsonInput.BoolFieldDoubleQuotedTrue +JsonInput.BoolMapFieldKeyNotQuoted +JsonInput.BytesFieldNoPadding +JsonInput.DoubleFieldInfinityNotQuoted +JsonInput.DoubleFieldNanNotQuoted +JsonInput.DoubleFieldNegativeInfinityNotQuoted +JsonInput.EnumFieldNotQuoted +JsonInput.EnumFieldNumericValueNonZero.JsonOutput +JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput +JsonInput.EnumFieldNumericValueZero.JsonOutput +JsonInput.EnumFieldNumericValueZero.ProtobufOutput +JsonInput.EnumFieldUnknownValue.Validator +JsonInput.FieldMask.ProtobufOutput +JsonInput.FieldMaskInvalidCharacter +JsonInput.FieldNameDuplicate +JsonInput.FieldNameDuplicateDifferentCasing1 +JsonInput.FieldNameDuplicateDifferentCasing2 +JsonInput.FieldNameInSnakeCase.JsonOutput +JsonInput.FieldNameNotQuoted +JsonInput.FieldNameWithMixedCases.JsonOutput +JsonInput.FloatFieldInfinityNotQuoted +JsonInput.FloatFieldNanNotQuoted +JsonInput.FloatFieldNegativeInfinityNotQuoted +JsonInput.Int32FieldExponentialFormat.JsonOutput +JsonInput.Int32FieldExponentialFormat.ProtobufOutput +JsonInput.Int32FieldFloatTrailingZero.JsonOutput +JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput +JsonInput.Int32FieldLeadingZero +JsonInput.Int32FieldMaxFloatValue.JsonOutput +JsonInput.Int32FieldMaxFloatValue.ProtobufOutput +JsonInput.Int32FieldMinFloatValue.JsonOutput +JsonInput.Int32FieldMinFloatValue.ProtobufOutput +JsonInput.Int32FieldMinValue.JsonOutput +JsonInput.Int32FieldNegativeWithLeadingZero +JsonInput.Int32FieldPlusSign +JsonInput.Int32MapFieldKeyNotQuoted +JsonInput.Int64MapFieldKeyNotQuoted +JsonInput.JsonWithComments +JsonInput.MapFieldValueIsNull +JsonInput.OneofFieldDuplicate +JsonInput.OriginalProtoFieldName.JsonOutput +JsonInput.RepeatedFieldMessageElementIsNull +JsonInput.RepeatedFieldPrimitiveElementIsNull +JsonInput.RepeatedFieldTrailingComma +JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool +JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt +JsonInput.StringFieldNotAString +JsonInput.StringFieldSurrogateInWrongOrder +JsonInput.StringFieldUnpairedHighSurrogate +JsonInput.StringFieldUnpairedLowSurrogate +JsonInput.StringFieldUppercaseEscapeLetter +JsonInput.Uint32FieldMaxFloatValue.JsonOutput +JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput +JsonInput.Uint32MapFieldKeyNotQuoted +JsonInput.Uint64MapFieldKeyNotQuoted +JsonInput.ValueAcceptNull.JsonOutput +JsonInput.ValueAcceptNull.ProtobufOutput diff --git a/java/pom.xml b/java/pom.xml index 7b99c035..aa5d30ea 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -295,15 +295,19 @@ **/MapFieldLite.java **/MessageLite.java **/MessageLiteOrBuilder.java + **/MessageLiteToString.java **/MutabilityOracle.java + **/NioByteString.java **/Parser.java **/ProtobufArrayList.java **/ProtocolStringList.java **/RopeByteString.java **/SmallSortedMap.java + **/TextFormatEscaper.java **/UninitializedMessageException.java **/UnknownFieldSetLite.java **/UnmodifiableLazyStringList.java + **/UnsafeByteStrings.java **/Utf8.java **/WireFormat.java @@ -316,6 +320,7 @@ **/LazyMessageLiteTest.java **/LiteTest.java **/LongArrayListTest.java + **/NioByteStringTest.java **/ProtobufArrayListTest.java **/UnknownFieldSetLiteTest.java diff --git a/java/src/main/java/com/google/protobuf/AbstractMessage.java b/java/src/main/java/com/google/protobuf/AbstractMessage.java index cc89173a..9f418f2b 100644 --- a/java/src/main/java/com/google/protobuf/AbstractMessage.java +++ b/java/src/main/java/com/google/protobuf/AbstractMessage.java @@ -30,6 +30,7 @@ package com.google.protobuf; +import com.google.protobuf.Descriptors.EnumValueDescriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.Descriptors.OneofDescriptor; import com.google.protobuf.Internal.EnumLite; @@ -161,10 +162,18 @@ public abstract class AbstractMessage extends AbstractMessageLite Descriptors.Descriptor descriptor = entry.getDescriptorForType(); Descriptors.FieldDescriptor key = descriptor.findFieldByName("key"); Descriptors.FieldDescriptor value = descriptor.findFieldByName("value"); - result.put(entry.getField(key), entry.getField(value)); + Object fieldValue = entry.getField(value); + if (fieldValue instanceof EnumValueDescriptor) { + fieldValue = ((EnumValueDescriptor) fieldValue).getNumber(); + } + result.put(entry.getField(key), fieldValue); while (iterator.hasNext()) { entry = (Message) iterator.next(); - result.put(entry.getField(key), entry.getField(value)); + fieldValue = entry.getField(value); + if (fieldValue instanceof EnumValueDescriptor) { + fieldValue = ((EnumValueDescriptor) fieldValue).getNumber(); + } + result.put(entry.getField(key), fieldValue); } return result; } diff --git a/java/src/main/java/com/google/protobuf/BooleanArrayList.java b/java/src/main/java/com/google/protobuf/BooleanArrayList.java index 45492d2f..70e042f5 100644 --- a/java/src/main/java/com/google/protobuf/BooleanArrayList.java +++ b/java/src/main/java/com/google/protobuf/BooleanArrayList.java @@ -68,10 +68,17 @@ final class BooleanArrayList private int size; /** - * Constructs a new mutable {@code BooleanArrayList}. + * Constructs a new mutable {@code BooleanArrayList} with default capacity. */ BooleanArrayList() { - array = new boolean[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code BooleanArrayList} with the provided capacity. + */ + BooleanArrayList(int capacity) { + array = new boolean[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/BoundedByteString.java b/java/src/main/java/com/google/protobuf/BoundedByteString.java index b4c3fb1b..934c9030 100644 --- a/java/src/main/java/com/google/protobuf/BoundedByteString.java +++ b/java/src/main/java/com/google/protobuf/BoundedByteString.java @@ -33,7 +33,6 @@ package com.google.protobuf; import java.io.IOException; import java.io.InvalidObjectException; import java.io.ObjectInputStream; -import java.util.NoSuchElementException; /** * This class is used to represent the substring of a {@link ByteString} over a @@ -47,7 +46,7 @@ import java.util.NoSuchElementException; * * @author carlanton@google.com (Carl Haverl) */ -class BoundedByteString extends LiteralByteString { +final class BoundedByteString extends LiteralByteString { private final int bytesOffset; private final int bytesLength; @@ -65,16 +64,7 @@ class BoundedByteString extends LiteralByteString { */ BoundedByteString(byte[] bytes, int offset, int length) { super(bytes); - if (offset < 0) { - throw new IllegalArgumentException("Offset too small: " + offset); - } - if (length < 0) { - throw new IllegalArgumentException("Length too small: " + offset); - } - if ((long) offset + length > bytes.length) { - throw new IllegalArgumentException( - "Offset+Length too large: " + offset + "+" + length); - } + checkRange(offset, offset + length, bytes.length); this.bytesOffset = offset; this.bytesLength = length; @@ -94,14 +84,7 @@ class BoundedByteString extends LiteralByteString { public byte byteAt(int index) { // We must check the index ourselves as we cannot rely on Java array index // checking for substrings. - if (index < 0) { - throw new ArrayIndexOutOfBoundsException("Index too small: " + index); - } - if (index >= size()) { - throw new ArrayIndexOutOfBoundsException( - "Index too large: " + index + ", " + size()); - } - + checkIndex(index, size()); return bytes[bytesOffset + index]; } @@ -119,8 +102,8 @@ class BoundedByteString extends LiteralByteString { // ByteString -> byte[] @Override - protected void copyToInternal(byte[] target, int sourceOffset, - int targetOffset, int numberToCopy) { + protected void copyToInternal(byte[] target, int sourceOffset, int targetOffset, + int numberToCopy) { System.arraycopy(bytes, getOffsetIntoBytes() + sourceOffset, target, targetOffset, numberToCopy); } @@ -134,47 +117,8 @@ class BoundedByteString extends LiteralByteString { return new LiteralByteString(toByteArray()); } - private void readObject(ObjectInputStream in) throws IOException { + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { throw new InvalidObjectException( "BoundedByteStream instances are not to be serialized directly"); } - - // ================================================================= - // ByteIterator - - @Override - public ByteIterator iterator() { - return new BoundedByteIterator(); - } - - private class BoundedByteIterator implements ByteIterator { - - private int position; - private final int limit; - - private BoundedByteIterator() { - position = getOffsetIntoBytes(); - limit = position + size(); - } - - public boolean hasNext() { - return (position < limit); - } - - public Byte next() { - // Boxing calls Byte.valueOf(byte), which does not instantiate. - return nextByte(); - } - - public byte nextByte() { - if (position >= limit) { - throw new NoSuchElementException(); - } - return bytes[position++]; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } } diff --git a/java/src/main/java/com/google/protobuf/ByteString.java b/java/src/main/java/com/google/protobuf/ByteString.java index b092bc36..68f20d51 100644 --- a/java/src/main/java/com/google/protobuf/ByteString.java +++ b/java/src/main/java/com/google/protobuf/ByteString.java @@ -83,6 +83,13 @@ public abstract class ByteString implements Iterable, Serializable { */ public static final ByteString EMPTY = new LiteralByteString(new byte[0]); + /** + * Cached hash value. Intentionally accessed via a data race, which + * is safe because of the Java Memory Model's "no out-of-thin-air values" + * guarantees for ints. A value of 0 implies that the hash has not been set. + */ + private int hash = 0; + // This constructor is here to prevent subclassing outside of this package, ByteString() {} @@ -105,7 +112,38 @@ public abstract class ByteString implements Iterable, Serializable { * * @return the iterator */ - public abstract ByteIterator iterator(); + @Override + public final ByteIterator iterator() { + return new ByteIterator() { + private int position = 0; + private final int limit = size(); + + @Override + public boolean hasNext() { + return position < limit; + } + + @Override + public Byte next() { + // Boxing calls Byte.valueOf(byte), which does not instantiate. + return nextByte(); + } + + @Override + public byte nextByte() { + try { + return byteAt(position++); + } catch (ArrayIndexOutOfBoundsException e) { + throw new NoSuchElementException(e.getMessage()); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } /** * This interface extends {@code Iterator}, so that we can return an @@ -134,7 +172,7 @@ public abstract class ByteString implements Iterable, Serializable { * * @return true if this is zero bytes long */ - public boolean isEmpty() { + public final boolean isEmpty() { return size() == 0; } @@ -150,7 +188,7 @@ public abstract class ByteString implements Iterable, Serializable { * @throws IndexOutOfBoundsException if {@code beginIndex < 0} or * {@code beginIndex > size()}. */ - public ByteString substring(int beginIndex) { + public final ByteString substring(int beginIndex) { return substring(beginIndex, size()); } @@ -175,7 +213,7 @@ public abstract class ByteString implements Iterable, Serializable { * argument is a prefix of the byte sequence represented by * this string; false otherwise. */ - public boolean startsWith(ByteString prefix) { + public final boolean startsWith(ByteString prefix) { return size() >= prefix.size() && substring(0, prefix.size()).equals(prefix); } @@ -189,7 +227,7 @@ public abstract class ByteString implements Iterable, Serializable { * argument is a suffix of the byte sequence represented by * this string; false otherwise. */ - public boolean endsWith(ByteString suffix) { + public final boolean endsWith(ByteString suffix) { return size() >= suffix.size() && substring(size() - suffix.size()).equals(suffix); } @@ -309,8 +347,7 @@ public abstract class ByteString implements Iterable, Serializable { */ public static ByteString readFrom(InputStream streamToDrain) throws IOException { - return readFrom( - streamToDrain, MIN_READ_FROM_CHUNK_SIZE, MAX_READ_FROM_CHUNK_SIZE); + return readFrom(streamToDrain, MIN_READ_FROM_CHUNK_SIZE, MAX_READ_FROM_CHUNK_SIZE); } /** @@ -383,10 +420,10 @@ public abstract class ByteString implements Iterable, Serializable { if (bytesRead == 0) { return null; - } else { - // Always make a copy since InputStream could steal a reference to buf. - return ByteString.copyFrom(buf, 0, bytesRead); } + + // Always make a copy since InputStream could steal a reference to buf. + return ByteString.copyFrom(buf, 0, bytesRead); } // ================================================================= @@ -402,12 +439,10 @@ public abstract class ByteString implements Iterable, Serializable { * @param other string to concatenate * @return a new {@code ByteString} instance */ - public ByteString concat(ByteString other) { - int thisSize = size(); - int otherSize = other.size(); - if ((long) thisSize + otherSize >= Integer.MAX_VALUE) { + public final ByteString concat(ByteString other) { + if (Integer.MAX_VALUE - size() < other.size()) { throw new IllegalArgumentException("ByteString would be too long: " + - thisSize + "+" + otherSize); + size() + "+" + other.size()); } return RopeByteString.concatenate(this, other); @@ -426,29 +461,29 @@ public abstract class ByteString implements Iterable, Serializable { * @return new {@code ByteString} */ public static ByteString copyFrom(Iterable byteStrings) { - Collection collection; + // Determine the size; + final int size; if (!(byteStrings instanceof Collection)) { - collection = new ArrayList(); - for (ByteString byteString : byteStrings) { - collection.add(byteString); + int tempSize = 0; + for (Iterator iter = byteStrings.iterator(); iter.hasNext(); + iter.next(), ++tempSize) { } + size = tempSize; } else { - collection = (Collection) byteStrings; + size = ((Collection) byteStrings).size(); } - ByteString result; - if (collection.isEmpty()) { - result = EMPTY; - } else { - result = balancedConcat(collection.iterator(), collection.size()); + + if (size == 0) { + return EMPTY; } - return result; + + return balancedConcat(byteStrings.iterator(), size); } // Internal function used by copyFrom(Iterable). // Create a balanced concatenation of the next "length" elements from the // iterable. - private static ByteString balancedConcat(Iterator iterator, - int length) { + private static ByteString balancedConcat(Iterator iterator, int length) { assert length >= 1; ByteString result; if (length == 1) { @@ -486,25 +521,10 @@ public abstract class ByteString implements Iterable, Serializable { * @throws IndexOutOfBoundsException if an offset or size is negative or too * large */ - public void copyTo(byte[] target, int sourceOffset, int targetOffset, + public final void copyTo(byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { - if (sourceOffset < 0) { - throw new IndexOutOfBoundsException("Source offset < 0: " + sourceOffset); - } - if (targetOffset < 0) { - throw new IndexOutOfBoundsException("Target offset < 0: " + targetOffset); - } - if (numberToCopy < 0) { - throw new IndexOutOfBoundsException("Length < 0: " + numberToCopy); - } - if (sourceOffset + numberToCopy > size()) { - throw new IndexOutOfBoundsException( - "Source end offset < 0: " + (sourceOffset + numberToCopy)); - } - if (targetOffset + numberToCopy > target.length) { - throw new IndexOutOfBoundsException( - "Target end offset < 0: " + (targetOffset + numberToCopy)); - } + checkRange(sourceOffset, sourceOffset + numberToCopy, size()); + checkRange(targetOffset, targetOffset + numberToCopy, target.length); if (numberToCopy > 0) { copyToInternal(target, sourceOffset, targetOffset, numberToCopy); } @@ -534,8 +554,8 @@ public abstract class ByteString implements Iterable, Serializable { * * @return copied bytes */ - public byte[] toByteArray() { - int size = size(); + public final byte[] toByteArray() { + final int size = size(); if (size == 0) { return Internal.EMPTY_BYTE_ARRAY; } @@ -548,6 +568,10 @@ public abstract class ByteString implements Iterable, Serializable { * Writes the complete contents of this byte string to * the specified output stream argument. * + *

It is assumed that the {@link OutputStream} will not modify the contents passed it + * it. It may be possible for a malicious {@link OutputStream} to corrupt + * the data underlying the {@link ByteString}. + * * @param out the output stream to which to write the data. * @throws IOException if an I/O error occurs. */ @@ -563,30 +587,20 @@ public abstract class ByteString implements Iterable, Serializable { * @throws IndexOutOfBoundsException if an offset or size is negative or too * large */ - void writeTo(OutputStream out, int sourceOffset, int numberToWrite) + final void writeTo(OutputStream out, int sourceOffset, int numberToWrite) throws IOException { - if (sourceOffset < 0) { - throw new IndexOutOfBoundsException("Source offset < 0: " + sourceOffset); - } - if (numberToWrite < 0) { - throw new IndexOutOfBoundsException("Length < 0: " + numberToWrite); - } - if (sourceOffset + numberToWrite > size()) { - throw new IndexOutOfBoundsException( - "Source end offset exceeded: " + (sourceOffset + numberToWrite)); - } + checkRange(sourceOffset, sourceOffset + numberToWrite, size()); if (numberToWrite > 0) { writeToInternal(out, sourceOffset, numberToWrite); } - } /** * Internal version of {@link #writeTo(OutputStream,int,int)} that assumes * all error checking has already been done. */ - abstract void writeToInternal(OutputStream out, int sourceOffset, - int numberToWrite) throws IOException; + abstract void writeToInternal(OutputStream out, int sourceOffset, int numberToWrite) + throws IOException; /** * Constructs a read-only {@code java.nio.ByteBuffer} whose content @@ -618,7 +632,7 @@ public abstract class ByteString implements Iterable, Serializable { * @return new string * @throws UnsupportedEncodingException if charset isn't recognized */ - public String toString(String charsetName) + public final String toString(String charsetName) throws UnsupportedEncodingException { try { return toString(Charset.forName(charsetName)); @@ -636,7 +650,7 @@ public abstract class ByteString implements Iterable, Serializable { * @param charset encode using this charset * @return new string */ - public String toString(Charset charset) { + public final String toString(Charset charset) { return size() == 0 ? "" : toStringInternal(charset); } @@ -657,7 +671,7 @@ public abstract class ByteString implements Iterable, Serializable { * * @return new string using UTF-8 encoding */ - public String toStringUtf8() { + public final String toStringUtf8() { return toString(Internal.UTF_8); } @@ -716,13 +730,51 @@ public abstract class ByteString implements Iterable, Serializable { public abstract boolean equals(Object o); /** - * Return a non-zero hashCode depending only on the sequence of bytes - * in this ByteString. + * Base class for leaf {@link ByteString}s (i.e. non-ropes). + */ + abstract static class LeafByteString extends ByteString { + @Override + protected final int getTreeDepth() { + return 0; + } + + @Override + protected final boolean isBalanced() { + return true; + } + + /** + * Check equality of the substring of given length of this object starting at + * zero with another {@code ByteString} substring starting at offset. + * + * @param other what to compare a substring in + * @param offset offset into other + * @param length number of bytes to compare + * @return true for equality of substrings, else false. + */ + abstract boolean equalsRange(ByteString other, int offset, int length); + } + + /** + * Compute the hashCode using the traditional algorithm from {@link + * ByteString}. * - * @return hashCode value for this object + * @return hashCode value */ @Override - public abstract int hashCode(); + public final int hashCode() { + int h = hash; + + if (h == 0) { + int size = size(); + h = partialHash(size, 0, size); + if (h == 0) { + h = 1; + } + hash = h; + } + return h; + } // ================================================================= // Input stream @@ -1034,7 +1086,9 @@ public abstract class ByteString implements Iterable, Serializable { * * @return value of cached hash code or 0 if not computed yet */ - protected abstract int peekCachedHashCode(); + protected final int peekCachedHashCode() { + return hash; + } /** * Compute the hash across the value bytes starting with the given hash, and @@ -1049,8 +1103,49 @@ public abstract class ByteString implements Iterable, Serializable { */ protected abstract int partialHash(int h, int offset, int length); + /** + * Checks that the given index falls within the specified array size. + * + * @param index the index position to be tested + * @param size the length of the array + * @throws ArrayIndexOutOfBoundsException if the index does not fall within the array. + */ + static void checkIndex(int index, int size) { + if ((index | (size - (index + 1))) < 0) { + if (index < 0) { + throw new ArrayIndexOutOfBoundsException("Index < 0: " + index); + } + throw new ArrayIndexOutOfBoundsException("Index > length: " + index + ", " + size); + } + } + + /** + * Checks that the given range falls within the bounds of an array + * + * @param startIndex the start index of the range (inclusive) + * @param endIndex the end index of the range (exclusive) + * @param size the size of the array. + * @return the length of the range. + * @throws ArrayIndexOutOfBoundsException some or all of the range falls outside of the array. + */ + static int checkRange(int startIndex, int endIndex, int size) { + final int length = endIndex - startIndex; + if ((startIndex | endIndex | length | (size - endIndex)) < 0) { + if (startIndex < 0) { + throw new IndexOutOfBoundsException("Beginning index: " + startIndex + " < 0"); + } + if (endIndex < startIndex) { + throw new IndexOutOfBoundsException( + "Beginning index larger than ending index: " + startIndex + ", " + endIndex); + } + // endIndex >= size + throw new IndexOutOfBoundsException("End index: " + endIndex + " >= " + size); + } + return length; + } + @Override - public String toString() { + public final String toString() { return String.format("", Integer.toHexString(System.identityHashCode(this)), size()); } diff --git a/java/src/main/java/com/google/protobuf/CodedInputStream.java b/java/src/main/java/com/google/protobuf/CodedInputStream.java index d201f7c5..adc91536 100644 --- a/java/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedInputStream.java @@ -1055,20 +1055,6 @@ public final class CodedInputStream { private RefillCallback refillCallback = null; - /** - * Ensures that at least {@code n} bytes are available in the buffer, reading - * more bytes from the input if necessary to make it so. Caller must ensure - * that the requested space is less than BUFFER_SIZE. - * - * @throws InvalidProtocolBufferException The end of the stream or the current - * limit was reached. - */ - private void ensureAvailable(int n) throws IOException { - if (bufferSize - bufferPos < n) { - refillBuffer(n); - } - } - /** * Reads more bytes from the input, making at least {@code n} bytes available * in the buffer. Caller must ensure that the requested space is not yet @@ -1180,86 +1166,97 @@ public final class CodedInputStream { } } - if (totalBytesRetired + bufferPos + size > currentLimit) { + // Verify that the message size so far has not exceeded sizeLimit. + int currentMessageSize = totalBytesRetired + bufferPos + size; + if (currentMessageSize > sizeLimit) { + throw InvalidProtocolBufferException.sizeLimitExceeded(); + } + + // Verify that the message size so far has not exceeded currentLimit. + if (currentMessageSize > currentLimit) { // Read to the end of the stream anyway. skipRawBytes(currentLimit - totalBytesRetired - bufferPos); - // Then fail. throw InvalidProtocolBufferException.truncatedMessage(); } - if (size < BUFFER_SIZE) { - // Reading more bytes than are in the buffer, but not an excessive number - // of bytes. We can safely allocate the resulting array ahead of time. + // We need the input stream to proceed. + if (input == null) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + + final int originalBufferPos = bufferPos; + final int bufferedBytes = bufferSize - bufferPos; + + // Mark the current buffer consumed. + totalBytesRetired += bufferSize; + bufferPos = 0; + bufferSize = 0; - // First copy what we have. + // Determine the number of bytes we need to read from the input stream. + int sizeLeft = size - bufferedBytes; + // TODO(nathanmittler): Consider using a value larger than BUFFER_SIZE. + if (sizeLeft < BUFFER_SIZE || sizeLeft <= input.available()) { + // Either the bytes we need are known to be available, or the required buffer is + // within an allowed threshold - go ahead and allocate the buffer now. final byte[] bytes = new byte[size]; - int pos = bufferSize - bufferPos; - System.arraycopy(buffer, bufferPos, bytes, 0, pos); - bufferPos = bufferSize; - // We want to refill the buffer and then copy from the buffer into our - // byte array rather than reading directly into our byte array because - // the input may be unbuffered. - ensureAvailable(size - pos); - System.arraycopy(buffer, 0, bytes, pos, size - pos); - bufferPos = size - pos; + // Copy all of the buffered bytes to the result buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); - return bytes; - } else { - // The size is very large. For security reasons, we can't allocate the - // entire byte array yet. The size comes directly from the input, so a - // maliciously-crafted message could provide a bogus very large size in - // order to trick the app into allocating a lot of memory. We avoid this - // by allocating and reading only a small chunk at a time, so that the - // malicious message must actually *be* extremely large to cause - // problems. Meanwhile, we limit the allowed size of a message elsewhere. - - // Remember the buffer markers since we'll have to copy the bytes out of - // it later. - final int originalBufferPos = bufferPos; - final int originalBufferSize = bufferSize; - - // Mark the current buffer consumed. - totalBytesRetired += bufferSize; - bufferPos = 0; - bufferSize = 0; - - // Read all the rest of the bytes we need. - int sizeLeft = size - (originalBufferSize - originalBufferPos); - final List chunks = new ArrayList(); - - while (sizeLeft > 0) { - final byte[] chunk = new byte[Math.min(sizeLeft, BUFFER_SIZE)]; - int pos = 0; - while (pos < chunk.length) { - final int n = (input == null) ? -1 : - input.read(chunk, pos, chunk.length - pos); - if (n == -1) { - throw InvalidProtocolBufferException.truncatedMessage(); - } - totalBytesRetired += n; - pos += n; + // Fill the remaining bytes from the input stream. + int pos = bufferedBytes; + while (pos < bytes.length) { + int n = input.read(bytes, pos, size - pos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); } - sizeLeft -= chunk.length; - chunks.add(chunk); + totalBytesRetired += n; + pos += n; } - // OK, got everything. Now concatenate it all into one buffer. - final byte[] bytes = new byte[size]; - - // Start by copying the leftover bytes from this.buffer. - int pos = originalBufferSize - originalBufferPos; - System.arraycopy(buffer, originalBufferPos, bytes, 0, pos); + return bytes; + } - // And now all the chunks. - for (final byte[] chunk : chunks) { - System.arraycopy(chunk, 0, bytes, pos, chunk.length); - pos += chunk.length; + // The size is very large. For security reasons, we can't allocate the + // entire byte array yet. The size comes directly from the input, so a + // maliciously-crafted message could provide a bogus very large size in + // order to trick the app into allocating a lot of memory. We avoid this + // by allocating and reading only a small chunk at a time, so that the + // malicious message must actually *be* extremely large to cause + // problems. Meanwhile, we limit the allowed size of a message elsewhere. + final List chunks = new ArrayList(); + + while (sizeLeft > 0) { + // TODO(nathanmittler): Consider using a value larger than BUFFER_SIZE. + final byte[] chunk = new byte[Math.min(sizeLeft, BUFFER_SIZE)]; + int pos = 0; + while (pos < chunk.length) { + final int n = input.read(chunk, pos, chunk.length - pos); + if (n == -1) { + throw InvalidProtocolBufferException.truncatedMessage(); + } + totalBytesRetired += n; + pos += n; } + sizeLeft -= chunk.length; + chunks.add(chunk); + } - // Done. - return bytes; + // OK, got everything. Now concatenate it all into one buffer. + final byte[] bytes = new byte[size]; + + // Start by copying the leftover bytes from this.buffer. + System.arraycopy(buffer, originalBufferPos, bytes, 0, bufferedBytes); + + // And now all the chunks. + int pos = bufferedBytes; + for (final byte[] chunk : chunks) { + System.arraycopy(chunk, 0, bytes, pos, chunk.length); + pos += chunk.length; } + + // Done. + return bytes; } /** diff --git a/java/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/src/main/java/com/google/protobuf/CodedOutputStream.java index 291bd20a..d8ebad21 100644 --- a/java/src/main/java/com/google/protobuf/CodedOutputStream.java +++ b/java/src/main/java/com/google/protobuf/CodedOutputStream.java @@ -53,7 +53,7 @@ import java.util.logging.Logger; * @author kneton@google.com Kenton Varda */ public final class CodedOutputStream { - + private static final Logger logger = Logger.getLogger(CodedOutputStream.class.getName()); // TODO(dweis): Consider migrating to a ByteBuffer. @@ -243,19 +243,6 @@ public final class CodedOutputStream { } - /** - * Write a group represented by an {@link UnknownFieldSet}. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #writeGroup}. - */ - @Deprecated - public void writeUnknownGroup(final int fieldNumber, - final MessageLite value) - throws IOException { - writeGroup(fieldNumber, value); - } - /** Write an embedded message field, including tag, to the stream. */ public void writeMessage(final int fieldNumber, final MessageLite value) throws IOException { @@ -428,7 +415,7 @@ public final class CodedOutputStream { try { efficientWriteStringNoTag(value); } catch (UnpairedSurrogateException e) { - logger.log(Level.WARNING, + logger.log(Level.WARNING, "Converting ill-formed UTF-16. Your Protocol Buffer will not round trip correctly!", e); inefficientWriteStringNoTag(value); } @@ -449,10 +436,10 @@ public final class CodedOutputStream { * Write a {@code string} field to the stream efficiently. If the {@code string} is malformed, * this method rolls back its changes and throws an {@link UnpairedSurrogateException} with the * intent that the caller will catch and retry with {@link #inefficientWriteStringNoTag(String)}. - * + * * @param value the string to write to the stream - * - * @throws UnpairedSurrogateException when {@code value} is ill-formed UTF-16. + * + * @throws UnpairedSurrogateException when {@code value} is ill-formed UTF-16. */ private void efficientWriteStringNoTag(final String value) throws IOException { // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), @@ -510,18 +497,6 @@ public final class CodedOutputStream { } - /** - * Write a group represented by an {@link UnknownFieldSet}. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #writeGroupNoTag}. - */ - @Deprecated - public void writeUnknownGroupNoTag(final MessageLite value) - throws IOException { - writeGroupNoTag(value); - } - /** Write an embedded message field to the stream. */ public void writeMessageNoTag(final MessageLite value) throws IOException { writeRawVarint32(value.getSerializedSize()); @@ -684,20 +659,6 @@ public final class CodedOutputStream { return computeTagSize(fieldNumber) * 2 + computeGroupSizeNoTag(value); } - /** - * Compute the number of bytes that would be needed to encode a - * {@code group} field represented by an {@code UnknownFieldSet}, including - * tag. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #computeGroupSize}. - */ - @Deprecated - public static int computeUnknownGroupSize(final int fieldNumber, - final MessageLite value) { - return computeGroupSize(fieldNumber, value); - } - /** * Compute the number of bytes that would be needed to encode an * embedded message field, including tag. @@ -926,19 +887,6 @@ public final class CodedOutputStream { return value.getSerializedSize(); } - /** - * Compute the number of bytes that would be needed to encode a - * {@code group} field represented by an {@code UnknownFieldSet}, including - * tag. - * - * @deprecated UnknownFieldSet now implements MessageLite, so you can just - * call {@link #computeUnknownGroupSizeNoTag}. - */ - @Deprecated - public static int computeUnknownGroupSizeNoTag(final MessageLite value) { - return computeGroupSizeNoTag(value); - } - /** * Compute the number of bytes that would be needed to encode an embedded * message field. @@ -1295,10 +1243,10 @@ public final class CodedOutputStream { * negative. */ public static int computeRawVarint32Size(final int value) { - if ((value & (0xffffffff << 7)) == 0) return 1; - if ((value & (0xffffffff << 14)) == 0) return 2; - if ((value & (0xffffffff << 21)) == 0) return 3; - if ((value & (0xffffffff << 28)) == 0) return 4; + if ((value & (~0 << 7)) == 0) return 1; + if ((value & (~0 << 14)) == 0) return 2; + if ((value & (~0 << 21)) == 0) return 3; + if ((value & (~0 << 28)) == 0) return 4; return 5; } @@ -1316,17 +1264,16 @@ public final class CodedOutputStream { } /** Compute the number of bytes that would be needed to encode a varint. */ - public static int computeRawVarint64Size(final long value) { - if ((value & (0xffffffffffffffffL << 7)) == 0) return 1; - if ((value & (0xffffffffffffffffL << 14)) == 0) return 2; - if ((value & (0xffffffffffffffffL << 21)) == 0) return 3; - if ((value & (0xffffffffffffffffL << 28)) == 0) return 4; - if ((value & (0xffffffffffffffffL << 35)) == 0) return 5; - if ((value & (0xffffffffffffffffL << 42)) == 0) return 6; - if ((value & (0xffffffffffffffffL << 49)) == 0) return 7; - if ((value & (0xffffffffffffffffL << 56)) == 0) return 8; - if ((value & (0xffffffffffffffffL << 63)) == 0) return 9; - return 10; + public static int computeRawVarint64Size(long value) { + // handle two popular special cases up front ... + if ((value & (~0L << 7)) == 0L) return 1; + if (value < 0L) return 10; + // ... leaving us with 8 remaining, which we can divide and conquer + int n = 2; + if ((value & (~0L << 35)) != 0L) { n += 4; value >>>= 28; } + if ((value & (~0L << 21)) != 0L) { n += 2; value >>>= 14; } + if ((value & (~0L << 14)) != 0L) { n += 1; } + return n; } /** Write a little-endian 32-bit integer. */ diff --git a/java/src/main/java/com/google/protobuf/Descriptors.java b/java/src/main/java/com/google/protobuf/Descriptors.java index 7cfc47f7..5e15cfbe 100644 --- a/java/src/main/java/com/google/protobuf/Descriptors.java +++ b/java/src/main/java/com/google/protobuf/Descriptors.java @@ -889,6 +889,11 @@ public final class Descriptors { */ public String getFullName() { return fullName; } + /** Get the JSON name of this field. */ + public String getJsonName() { + return jsonName; + } + /** * Get the field's java type. This is just for convenience. Every * {@code FieldDescriptorProto.Type} maps to exactly one Java type. @@ -1079,6 +1084,7 @@ public final class Descriptors { private FieldDescriptorProto proto; private final String fullName; + private final String jsonName; private final FileDescriptor file; private final Descriptor extensionScope; @@ -1157,6 +1163,38 @@ public final class Descriptors { private final Object defaultDefault; } + // TODO(xiaofeng): Implement it consistently across different languages. See b/24751348. + private static String fieldNameToLowerCamelCase(String name) { + StringBuilder result = new StringBuilder(name.length()); + boolean isNextUpperCase = false; + for (int i = 0; i < name.length(); i++) { + Character ch = name.charAt(i); + if (Character.isLowerCase(ch)) { + if (isNextUpperCase) { + result.append(Character.toUpperCase(ch)); + } else { + result.append(ch); + } + isNextUpperCase = false; + } else if (Character.isUpperCase(ch)) { + if (i == 0) { + // Force first letter to lower-case. + result.append(Character.toLowerCase(ch)); + } else { + // Capital letters after the first are left as-is. + result.append(ch); + } + isNextUpperCase = false; + } else if (Character.isDigit(ch)) { + result.append(ch); + isNextUpperCase = false; + } else { + isNextUpperCase = true; + } + } + return result.toString(); + } + private FieldDescriptor(final FieldDescriptorProto proto, final FileDescriptor file, final Descriptor parent, @@ -1167,6 +1205,11 @@ public final class Descriptors { this.proto = proto; fullName = computeFullName(file, parent, proto.getName()); this.file = file; + if (proto.hasJsonName()) { + jsonName = proto.getJsonName(); + } else { + jsonName = fieldNameToLowerCamelCase(proto.getName()); + } if (proto.hasType()) { type = Type.valueOf(proto.getType()); diff --git a/java/src/main/java/com/google/protobuf/DoubleArrayList.java b/java/src/main/java/com/google/protobuf/DoubleArrayList.java index 90ebe109..bcc9d6ee 100644 --- a/java/src/main/java/com/google/protobuf/DoubleArrayList.java +++ b/java/src/main/java/com/google/protobuf/DoubleArrayList.java @@ -68,10 +68,17 @@ final class DoubleArrayList private int size; /** - * Constructs a new mutable {@code DoubleArrayList}. + * Constructs a new mutable {@code DoubleArrayList} with default capacity. */ DoubleArrayList() { - array = new double[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code DoubleArrayList} with the provided capacity. + */ + DoubleArrayList(int capacity) { + array = new double[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/FloatArrayList.java b/java/src/main/java/com/google/protobuf/FloatArrayList.java index 293eaff6..033b5eed 100644 --- a/java/src/main/java/com/google/protobuf/FloatArrayList.java +++ b/java/src/main/java/com/google/protobuf/FloatArrayList.java @@ -67,10 +67,17 @@ final class FloatArrayList extends AbstractProtobufList implements FloatL private int size; /** - * Constructs a new mutable {@code FloatArrayList}. + * Constructs a new mutable {@code FloatArrayList} with default capacity. */ FloatArrayList() { - array = new float[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code FloatArrayList} with the provided capacity. + */ + FloatArrayList(int capacity) { + array = new float[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java index 4316efee..81e1862c 100644 --- a/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java +++ b/java/src/main/java/com/google/protobuf/GeneratedMessageLite.java @@ -102,6 +102,11 @@ public abstract class GeneratedMessageLite< * @return {@code true} unless the tag is an end-group tag. */ protected boolean parseUnknownField(int tag, CodedInputStream input) throws IOException { + // This will avoid the allocation of unknown fields when a group tag is encountered. + if (WireFormat.getTagWireType(tag) == WireFormat.WIRETYPE_END_GROUP) { + return false; + } + ensureUnknownFieldsInitialized(); return unknownFields.mergeFieldFrom(tag, input); } @@ -1173,6 +1178,10 @@ public abstract class GeneratedMessageLite< return new IntArrayList(); } + protected static IntList newIntListWithCapacity(int capacity) { + return new IntArrayList(capacity); + } + protected static IntList newIntList(List toCopy) { return new IntArrayList(toCopy); } @@ -1180,10 +1189,14 @@ public abstract class GeneratedMessageLite< protected static IntList emptyIntList() { return IntArrayList.emptyList(); } - + protected static LongList newLongList() { return new LongArrayList(); } + + protected static LongList newLongListWithCapacity(int capacity) { + return new LongArrayList(capacity); + } protected static LongList newLongList(List toCopy) { return new LongArrayList(toCopy); @@ -1197,6 +1210,10 @@ public abstract class GeneratedMessageLite< return new FloatArrayList(); } + protected static FloatList newFloatListWithCapacity(int capacity) { + return new FloatArrayList(capacity); + } + protected static FloatList newFloatList(List toCopy) { return new FloatArrayList(toCopy); } @@ -1209,6 +1226,10 @@ public abstract class GeneratedMessageLite< return new DoubleArrayList(); } + protected static DoubleList newDoubleListWithCapacity(int capacity) { + return new DoubleArrayList(capacity); + } + protected static DoubleList newDoubleList(List toCopy) { return new DoubleArrayList(toCopy); } @@ -1221,6 +1242,10 @@ public abstract class GeneratedMessageLite< return new BooleanArrayList(); } + protected static BooleanList newBooleanListWithCapacity(int capacity) { + return new BooleanArrayList(capacity); + } + protected static BooleanList newBooleanList(List toCopy) { return new BooleanArrayList(toCopy); } @@ -1237,6 +1262,10 @@ public abstract class GeneratedMessageLite< return new ProtobufArrayList(toCopy); } + protected static ProtobufList newProtobufListWithCapacity(int capacity) { + return new ProtobufArrayList(capacity); + } + protected static ProtobufList emptyProtobufList() { return ProtobufArrayList.emptyList(); } diff --git a/java/src/main/java/com/google/protobuf/IntArrayList.java b/java/src/main/java/com/google/protobuf/IntArrayList.java index f7609cc9..f4e68ed8 100644 --- a/java/src/main/java/com/google/protobuf/IntArrayList.java +++ b/java/src/main/java/com/google/protobuf/IntArrayList.java @@ -67,10 +67,17 @@ final class IntArrayList extends AbstractProtobufList implements IntLis private int size; /** - * Constructs a new mutable {@code IntArrayList}. + * Constructs a new mutable {@code IntArrayList} with default capacity. */ IntArrayList() { - array = new int[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code IntArrayList} with the provided capacity. + */ + IntArrayList(int capacity) { + array = new int[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/LiteralByteString.java b/java/src/main/java/com/google/protobuf/LiteralByteString.java index c5a8512a..a18c2792 100644 --- a/java/src/main/java/com/google/protobuf/LiteralByteString.java +++ b/java/src/main/java/com/google/protobuf/LiteralByteString.java @@ -36,9 +36,8 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.Charset; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.NoSuchElementException; /** * This class implements a {@link com.google.protobuf.ByteString} backed by a @@ -49,8 +48,7 @@ import java.util.NoSuchElementException; * * @author carlanton@google.com (Carl Haverl) */ -class LiteralByteString extends ByteString { - +class LiteralByteString extends ByteString.LeafByteString { private static final long serialVersionUID = 1L; protected final byte[] bytes; @@ -82,77 +80,56 @@ class LiteralByteString extends ByteString { // ByteString -> substring @Override - public ByteString substring(int beginIndex, int endIndex) { - if (beginIndex < 0) { - throw new IndexOutOfBoundsException( - "Beginning index: " + beginIndex + " < 0"); - } - if (endIndex > size()) { - throw new IndexOutOfBoundsException("End index: " + endIndex + " > " + - size()); - } - int substringLength = endIndex - beginIndex; - if (substringLength < 0) { - throw new IndexOutOfBoundsException( - "Beginning index larger than ending index: " + beginIndex + ", " - + endIndex); - } + public final ByteString substring(int beginIndex, int endIndex) { + final int length = checkRange(beginIndex, endIndex, size()); - ByteString result; - if (substringLength == 0) { - result = ByteString.EMPTY; - } else { - result = new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, - substringLength); + if (length == 0) { + return ByteString.EMPTY; } - return result; + + return new BoundedByteString(bytes, getOffsetIntoBytes() + beginIndex, length); } // ================================================================= // ByteString -> byte[] @Override - protected void copyToInternal(byte[] target, int sourceOffset, - int targetOffset, int numberToCopy) { + protected void copyToInternal( + byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { // Optimized form, not for subclasses, since we don't call // getOffsetIntoBytes() or check the 'numberToCopy' parameter. + // TODO(nathanmittler): Is not calling getOffsetIntoBytes really saving that much? System.arraycopy(bytes, sourceOffset, target, targetOffset, numberToCopy); } @Override - public void copyTo(ByteBuffer target) { - target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes + public final void copyTo(ByteBuffer target) { + target.put(bytes, getOffsetIntoBytes(), size()); // Copies bytes + } + + @Override + public final ByteBuffer asReadOnlyByteBuffer() { + return ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()).asReadOnlyBuffer(); } @Override - public ByteBuffer asReadOnlyByteBuffer() { - ByteBuffer byteBuffer = - ByteBuffer.wrap(bytes, getOffsetIntoBytes(), size()); - return byteBuffer.asReadOnlyBuffer(); + public final List asReadOnlyByteBufferList() { + return Collections.singletonList(asReadOnlyByteBuffer()); } @Override - public List asReadOnlyByteBufferList() { - // Return the ByteBuffer generated by asReadOnlyByteBuffer() as a singleton - List result = new ArrayList(1); - result.add(asReadOnlyByteBuffer()); - return result; - } - - @Override - public void writeTo(OutputStream outputStream) throws IOException { + public final void writeTo(OutputStream outputStream) throws IOException { outputStream.write(toByteArray()); } @Override - void writeToInternal(OutputStream outputStream, int sourceOffset, - int numberToWrite) throws IOException { - outputStream.write(bytes, getOffsetIntoBytes() + sourceOffset, - numberToWrite); + final void writeToInternal(OutputStream outputStream, int sourceOffset, int numberToWrite) + throws IOException { + outputStream.write(bytes, getOffsetIntoBytes() + sourceOffset, numberToWrite); } @Override - protected String toStringInternal(Charset charset) { + protected final String toStringInternal(Charset charset) { return new String(bytes, getOffsetIntoBytes(), size(), charset); } @@ -160,13 +137,13 @@ class LiteralByteString extends ByteString { // UTF-8 decoding @Override - public boolean isValidUtf8() { + public final boolean isValidUtf8() { int offset = getOffsetIntoBytes(); return Utf8.isValidUtf8(bytes, offset, offset + size()); } @Override - protected int partialIsValidUtf8(int state, int offset, int length) { + protected final int partialIsValidUtf8(int state, int offset, int length) { int index = getOffsetIntoBytes() + offset; return Utf8.partialIsValidUtf8(state, bytes, index, index + length); } @@ -175,7 +152,7 @@ class LiteralByteString extends ByteString { // equals() and hashCode() @Override - public boolean equals(Object other) { + public final boolean equals(Object other) { if (other == this) { return true; } @@ -194,19 +171,16 @@ class LiteralByteString extends ByteString { LiteralByteString otherAsLiteral = (LiteralByteString) other; // If we know the hash codes and they are not equal, we know the byte // strings are not equal. - if (hash != 0 - && otherAsLiteral.hash != 0 - && hash != otherAsLiteral.hash) { + int thisHash = peekCachedHashCode(); + int thatHash = otherAsLiteral.peekCachedHashCode(); + if (thisHash != 0 && thatHash != 0 && thisHash != thatHash) { return false; } return equalsRange((LiteralByteString) other, 0, size()); - } else if (other instanceof RopeByteString) { - return other.equals(this); } else { - throw new IllegalArgumentException( - "Has a new type of ByteString been created? Found " - + other.getClass()); + // RopeByteString and NioByteString. + return other.equals(this); } } @@ -219,65 +193,36 @@ class LiteralByteString extends ByteString { * @param length number of bytes to compare * @return true for equality of substrings, else false. */ - boolean equalsRange(LiteralByteString other, int offset, int length) { + @Override + final boolean equalsRange(ByteString other, int offset, int length) { if (length > other.size()) { - throw new IllegalArgumentException( - "Length too large: " + length + size()); + throw new IllegalArgumentException("Length too large: " + length + size()); } if (offset + length > other.size()) { throw new IllegalArgumentException( - "Ran off end of other: " + offset + ", " + length + ", " + - other.size()); + "Ran off end of other: " + offset + ", " + length + ", " + other.size()); } - byte[] thisBytes = bytes; - byte[] otherBytes = other.bytes; - int thisLimit = getOffsetIntoBytes() + length; - for (int thisIndex = getOffsetIntoBytes(), otherIndex = - other.getOffsetIntoBytes() + offset; - (thisIndex < thisLimit); ++thisIndex, ++otherIndex) { - if (thisBytes[thisIndex] != otherBytes[otherIndex]) { - return false; - } - } - return true; - } - - /** - * Cached hash value. Intentionally accessed via a data race, which - * is safe because of the Java Memory Model's "no out-of-thin-air values" - * guarantees for ints. - */ - private int hash = 0; - - /** - * Compute the hashCode using the traditional algorithm from {@link - * ByteString}. - * - * @return hashCode value - */ - @Override - public int hashCode() { - int h = hash; - - if (h == 0) { - int size = size(); - h = partialHash(size, 0, size); - if (h == 0) { - h = 1; + if (other instanceof LiteralByteString) { + LiteralByteString lbsOther = (LiteralByteString) other; + byte[] thisBytes = bytes; + byte[] otherBytes = lbsOther.bytes; + int thisLimit = getOffsetIntoBytes() + length; + for ( + int thisIndex = getOffsetIntoBytes(), otherIndex = lbsOther.getOffsetIntoBytes() + offset; + (thisIndex < thisLimit); ++thisIndex, ++otherIndex) { + if (thisBytes[thisIndex] != otherBytes[otherIndex]) { + return false; + } } - hash = h; + return true; } - return h; - } - @Override - protected int peekCachedHashCode() { - return hash; + return other.substring(offset, offset + length).equals(substring(0, length)); } @Override - protected int partialHash(int h, int offset, int length) { + protected final int partialHash(int h, int offset, int length) { return hashCode(h, bytes, getOffsetIntoBytes() + offset, length); } @@ -297,70 +242,20 @@ class LiteralByteString extends ByteString { // Input stream @Override - public InputStream newInput() { - return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), - size()); // No copy + public final InputStream newInput() { + return new ByteArrayInputStream(bytes, getOffsetIntoBytes(), size()); // No copy } @Override - public CodedInputStream newCodedInput() { + public final CodedInputStream newCodedInput() { // We trust CodedInputStream not to modify the bytes, or to give anyone // else access to them. return CodedInputStream.newInstance(this); } - // ================================================================= - // ByteIterator - - @Override - public ByteIterator iterator() { - return new LiteralByteIterator(); - } - - private class LiteralByteIterator implements ByteIterator { - private int position; - private final int limit; - - private LiteralByteIterator() { - position = 0; - limit = size(); - } - - public boolean hasNext() { - return (position < limit); - } - - public Byte next() { - // Boxing calls Byte.valueOf(byte), which does not instantiate. - return nextByte(); - } - - public byte nextByte() { - try { - return bytes[position++]; - } catch (ArrayIndexOutOfBoundsException e) { - throw new NoSuchElementException(e.getMessage()); - } - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } - // ================================================================= // Internal methods - @Override - protected int getTreeDepth() { - return 0; - } - - @Override - protected boolean isBalanced() { - return true; - } - /** * Offset into {@code bytes[]} to use, non-zero for substrings. * diff --git a/java/src/main/java/com/google/protobuf/LongArrayList.java b/java/src/main/java/com/google/protobuf/LongArrayList.java index 298617ff..ebe62029 100644 --- a/java/src/main/java/com/google/protobuf/LongArrayList.java +++ b/java/src/main/java/com/google/protobuf/LongArrayList.java @@ -67,10 +67,17 @@ final class LongArrayList extends AbstractProtobufList implements LongList private int size; /** - * Constructs a new mutable {@code LongArrayList}. + * Constructs a new mutable {@code LongArrayList} with default capacity. */ LongArrayList() { - array = new long[DEFAULT_CAPACITY]; + this(DEFAULT_CAPACITY); + } + + /** + * Constructs a new mutable {@code LongArrayList} with the provided capacity. + */ + LongArrayList(int capacity) { + array = new long[capacity]; size = 0; } diff --git a/java/src/main/java/com/google/protobuf/MapFieldLite.java b/java/src/main/java/com/google/protobuf/MapFieldLite.java index c17fa7b1..16d3e6d2 100644 --- a/java/src/main/java/com/google/protobuf/MapFieldLite.java +++ b/java/src/main/java/com/google/protobuf/MapFieldLite.java @@ -30,6 +30,8 @@ package com.google.protobuf; +import com.google.protobuf.Internal.EnumLite; + import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -44,7 +46,7 @@ import java.util.Set; * This class is a protobuf implementation detail. Users shouldn't use this * class directly. */ -public class MapFieldLite implements MutabilityOracle { +public final class MapFieldLite implements MutabilityOracle { private MutatabilityAwareMap mapData; private boolean isMutable; @@ -136,8 +138,9 @@ public class MapFieldLite implements MutabilityOracle { if (a instanceof byte[]) { return LiteralByteString.hashCode((byte[]) a); } - if (a instanceof Internal.EnumLite) { - return Internal.hashEnum((Internal.EnumLite) a); + // Enums should be stored as integers internally. + if (a instanceof EnumLite) { + throw new UnsupportedOperationException(); } return a.hashCode(); } diff --git a/java/src/main/java/com/google/protobuf/MessageLiteToString.java b/java/src/main/java/com/google/protobuf/MessageLiteToString.java new file mode 100644 index 00000000..e69de29b diff --git a/java/src/main/java/com/google/protobuf/NioByteString.java b/java/src/main/java/com/google/protobuf/NioByteString.java new file mode 100644 index 00000000..f71e41b2 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/NioByteString.java @@ -0,0 +1,309 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InvalidObjectException; +import java.io.ObjectInputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.InvalidMarkException; +import java.nio.channels.Channels; +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; + +/** + * A {@link ByteString} that wraps around a {@link ByteBuffer}. + */ +final class NioByteString extends ByteString.LeafByteString { + private final ByteBuffer buffer; + + NioByteString(ByteBuffer buffer) { + if (buffer == null) { + throw new NullPointerException("buffer"); + } + + this.buffer = buffer.slice(); + } + + // ================================================================= + // Serializable + + /** + * Magic method that lets us override serialization behavior. + */ + private Object writeReplace() { + return ByteString.copyFrom(buffer.slice()); + } + + /** + * Magic method that lets us override deserialization behavior. + */ + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { + throw new InvalidObjectException("NioByteString instances are not to be serialized directly"); + } + + // ================================================================= + + @Override + public byte byteAt(int index) { + try { + return buffer.get(index); + } catch (ArrayIndexOutOfBoundsException e) { + throw e; + } catch (IndexOutOfBoundsException e) { + throw new ArrayIndexOutOfBoundsException(e.getMessage()); + } + } + + @Override + public int size() { + return buffer.remaining(); + } + + @Override + public ByteString substring(int beginIndex, int endIndex) { + try { + ByteBuffer slice = slice(beginIndex, endIndex); + return new NioByteString(slice); + } catch (ArrayIndexOutOfBoundsException e) { + throw e; + } catch (IndexOutOfBoundsException e) { + throw new ArrayIndexOutOfBoundsException(e.getMessage()); + } + } + + @Override + protected void copyToInternal( + byte[] target, int sourceOffset, int targetOffset, int numberToCopy) { + ByteBuffer slice = buffer.slice(); + slice.position(sourceOffset); + slice.get(target, targetOffset, numberToCopy); + } + + @Override + public void copyTo(ByteBuffer target) { + target.put(buffer.slice()); + } + + @Override + public void writeTo(OutputStream out) throws IOException { + writeToInternal(out, buffer.position(), buffer.remaining()); + } + + @Override + boolean equalsRange(ByteString other, int offset, int length) { + return substring(0, length).equals(other.substring(offset, offset + length)); + } + + @Override + void writeToInternal(OutputStream out, int sourceOffset, int numberToWrite) throws IOException { + if (buffer.hasArray()) { + // Optimized write for array-backed buffers. + // Note that we're taking the risk that a malicious OutputStream could modify the array. + int bufferOffset = buffer.arrayOffset() + buffer.position() + sourceOffset; + out.write(buffer.array(), bufferOffset, numberToWrite); + return; + } + + // Slow path + if (out instanceof FileOutputStream || numberToWrite >= 8192) { + // Use a channel to write out the ByteBuffer. + Channels.newChannel(out).write(slice(sourceOffset, sourceOffset + numberToWrite)); + } else { + // Just copy the data to an array and write it. + out.write(toByteArray()); + } + } + + @Override + public ByteBuffer asReadOnlyByteBuffer() { + return buffer.asReadOnlyBuffer(); + } + + @Override + public List asReadOnlyByteBufferList() { + return Collections.singletonList(asReadOnlyByteBuffer()); + } + + @Override + protected String toStringInternal(Charset charset) { + byte[] bytes; + int offset; + if (buffer.hasArray()) { + bytes = buffer.array(); + offset = buffer.arrayOffset() + buffer.position(); + } else { + bytes = toByteArray(); + offset = 0; + } + return new String(bytes, offset, size(), charset); + } + + @Override + public boolean isValidUtf8() { + // TODO(nathanmittler): add a ByteBuffer fork for Utf8.isValidUtf8 to avoid the copy + byte[] bytes; + int startIndex; + if (buffer.hasArray()) { + bytes = buffer.array(); + startIndex = buffer.arrayOffset() + buffer.position(); + } else { + bytes = toByteArray(); + startIndex = 0; + } + return Utf8.isValidUtf8(bytes, startIndex, startIndex + size()); + } + + @Override + protected int partialIsValidUtf8(int state, int offset, int length) { + // TODO(nathanmittler): TODO add a ByteBuffer fork for Utf8.partialIsValidUtf8 to avoid the copy + byte[] bytes; + int startIndex; + if (buffer.hasArray()) { + bytes = buffer.array(); + startIndex = buffer.arrayOffset() + buffer.position(); + } else { + bytes = toByteArray(); + startIndex = 0; + } + return Utf8.partialIsValidUtf8(state, bytes, startIndex, startIndex + size()); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + if (!(other instanceof ByteString)) { + return false; + } + ByteString otherString = ((ByteString) other); + if (size() != otherString.size()) { + return false; + } + if (size() == 0) { + return true; + } + if (other instanceof NioByteString) { + return buffer.equals(((NioByteString) other).buffer); + } + if (other instanceof RopeByteString) { + return other.equals(this); + } + return buffer.equals(otherString.asReadOnlyByteBuffer()); + } + + @Override + protected int partialHash(int h, int offset, int length) { + for (int i = offset; i < offset + length; i++) { + h = h * 31 + buffer.get(i); + } + return h; + } + + @Override + public InputStream newInput() { + return new InputStream() { + private final ByteBuffer buf = buffer.slice(); + + @Override + public void mark(int readlimit) { + buf.mark(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public void reset() throws IOException { + try { + buf.reset(); + } catch (InvalidMarkException e) { + throw new IOException(e); + } + } + + @Override + public int available() throws IOException { + return buf.remaining(); + } + + @Override + public int read() throws IOException { + if (!buf.hasRemaining()) { + return -1; + } + return buf.get() & 0xFF; + } + + @Override + public int read(byte[] bytes, int off, int len) throws IOException { + if (!buf.hasRemaining()) { + return -1; + } + + len = Math.min(len, buf.remaining()); + buf.get(bytes, off, len); + return len; + } + }; + } + + @Override + public CodedInputStream newCodedInput() { + return CodedInputStream.newInstance(buffer); + } + + /** + * Creates a slice of a range of this buffer. + * + * @param beginIndex the beginning index of the slice (inclusive). + * @param endIndex the end index of the slice (exclusive). + * @return the requested slice. + */ + private ByteBuffer slice(int beginIndex, int endIndex) { + if (beginIndex < buffer.position() || endIndex > buffer.limit() || beginIndex > endIndex) { + throw new IllegalArgumentException( + String.format("Invalid indices [%d, %d]", beginIndex, endIndex)); + } + + ByteBuffer slice = buffer.slice(); + slice.position(beginIndex - buffer.position()); + slice.limit(endIndex - buffer.position()); + return slice; + } +} diff --git a/java/src/main/java/com/google/protobuf/ProtobufArrayList.java b/java/src/main/java/com/google/protobuf/ProtobufArrayList.java index 759368c9..d2f82ac5 100644 --- a/java/src/main/java/com/google/protobuf/ProtobufArrayList.java +++ b/java/src/main/java/com/google/protobuf/ProtobufArrayList.java @@ -60,6 +60,10 @@ class ProtobufArrayList extends AbstractProtobufList { list = new ArrayList(toCopy); } + ProtobufArrayList(int capacity) { + list = new ArrayList(capacity); + } + @Override public void add(int index, E element) { ensureIsMutable(); diff --git a/java/src/main/java/com/google/protobuf/RopeByteString.java b/java/src/main/java/com/google/protobuf/RopeByteString.java index 2c332624..6e8eb724 100644 --- a/java/src/main/java/com/google/protobuf/RopeByteString.java +++ b/java/src/main/java/com/google/protobuf/RopeByteString.java @@ -69,7 +69,7 @@ import java.util.Stack; * * @author carlanton@google.com (Carl Haverl) */ -class RopeByteString extends ByteString { +final class RopeByteString extends ByteString { /** * BAP95. Let Fn be the nth Fibonacci number. A {@link RopeByteString} of @@ -151,21 +151,24 @@ class RopeByteString extends ByteString { * @return concatenation representing the same sequence as the given strings */ static ByteString concatenate(ByteString left, ByteString right) { - ByteString result; - RopeByteString leftRope = - (left instanceof RopeByteString) ? (RopeByteString) left : null; if (right.size() == 0) { - result = left; - } else if (left.size() == 0) { - result = right; - } else { - int newLength = left.size() + right.size(); - if (newLength < ByteString.CONCATENATE_BY_COPY_SIZE) { - // Optimization from BAP95: For short (leaves in paper, but just short - // here) total length, do a copy of data to a new leaf. - result = concatenateBytes(left, right); - } else if (leftRope != null - && leftRope.right.size() + right.size() < CONCATENATE_BY_COPY_SIZE) { + return left; + } + + if (left.size() == 0) { + return right; + } + + final int newLength = left.size() + right.size(); + if (newLength < ByteString.CONCATENATE_BY_COPY_SIZE) { + // Optimization from BAP95: For short (leaves in paper, but just short + // here) total length, do a copy of data to a new leaf. + return concatenateBytes(left, right); + } + + if (left instanceof RopeByteString) { + final RopeByteString leftRope = (RopeByteString) left; + if (leftRope.right.size() + right.size() < CONCATENATE_BY_COPY_SIZE) { // Optimization from BAP95: As an optimization of the case where the // ByteString is constructed by repeated concatenate, recognize the case // where a short string is concatenated to a left-hand node whose @@ -177,9 +180,10 @@ class RopeByteString extends ByteString { // new parent node so that the depth of the result is the same as the // given left tree. ByteString newRight = concatenateBytes(leftRope.right, right); - result = new RopeByteString(leftRope.left, newRight); - } else if (leftRope != null - && leftRope.left.getTreeDepth() > leftRope.right.getTreeDepth() + return new RopeByteString(leftRope.left, newRight); + } + + if (leftRope.left.getTreeDepth() > leftRope.right.getTreeDepth() && leftRope.getTreeDepth() > right.getTreeDepth()) { // Typically for concatenate-built strings the left-side is deeper than // the right. This is our final attempt to concatenate without @@ -187,20 +191,18 @@ class RopeByteString extends ByteString { // is yet another optimization for building the string by repeatedly // concatenating on the right. ByteString newRight = new RopeByteString(leftRope.right, right); - result = new RopeByteString(leftRope.left, newRight); - } else { - // Fine, we'll add a node and increase the tree depth--unless we - // rebalance ;^) - int newDepth = Math.max(left.getTreeDepth(), right.getTreeDepth()) + 1; - if (newLength >= minLengthByDepth[newDepth]) { - // The tree is shallow enough, so don't rebalance - result = new RopeByteString(left, right); - } else { - result = new Balancer().balance(left, right); - } + return new RopeByteString(leftRope.left, newRight); } } - return result; + + // Fine, we'll add a node and increase the tree depth--unless we rebalance ;^) + int newDepth = Math.max(left.getTreeDepth(), right.getTreeDepth()) + 1; + if (newLength >= minLengthByDepth[newDepth]) { + // The tree is shallow enough, so don't rebalance + return new RopeByteString(left, right); + } + + return new Balancer().balance(left, right); } /** @@ -248,22 +250,14 @@ class RopeByteString extends ByteString { */ @Override public byte byteAt(int index) { - if (index < 0) { - throw new ArrayIndexOutOfBoundsException("Index < 0: " + index); - } - if (index > totalLength) { - throw new ArrayIndexOutOfBoundsException( - "Index > length: " + index + ", " + totalLength); - } + checkIndex(index, totalLength); - byte result; // Find the relevant piece by recursive descent if (index < leftLength) { - result = left.byteAt(index); - } else { - result = right.byteAt(index - leftLength); + return left.byteAt(index); } - return result; + + return right.byteAt(index - leftLength); } @Override @@ -309,48 +303,36 @@ class RopeByteString extends ByteString { */ @Override public ByteString substring(int beginIndex, int endIndex) { - if (beginIndex < 0) { - throw new IndexOutOfBoundsException( - "Beginning index: " + beginIndex + " < 0"); + final int length = checkRange(beginIndex, endIndex, totalLength); + + if (length == 0) { + // Empty substring + return ByteString.EMPTY; } - if (endIndex > totalLength) { - throw new IndexOutOfBoundsException( - "End index: " + endIndex + " > " + totalLength); + + if (length == totalLength) { + // The whole string + return this; } - int substringLength = endIndex - beginIndex; - if (substringLength < 0) { - throw new IndexOutOfBoundsException( - "Beginning index larger than ending index: " + beginIndex + ", " - + endIndex); + + // Proper substring + if (endIndex <= leftLength) { + // Substring on the left + return left.substring(beginIndex, endIndex); } - ByteString result; - if (substringLength == 0) { - // Empty substring - result = ByteString.EMPTY; - } else if (substringLength == totalLength) { - // The whole string - result = this; - } else { - // Proper substring - if (endIndex <= leftLength) { - // Substring on the left - result = left.substring(beginIndex, endIndex); - } else if (beginIndex >= leftLength) { - // Substring on the right - result = right - .substring(beginIndex - leftLength, endIndex - leftLength); - } else { - // Split substring - ByteString leftSub = left.substring(beginIndex); - ByteString rightSub = right.substring(0, endIndex - leftLength); - // Intentionally not rebalancing, since in many cases these two - // substrings will already be less deep than the top-level - // RopeByteString we're taking a substring of. - result = new RopeByteString(leftSub, rightSub); - } + if (beginIndex >= leftLength) { + // Substring on the right + return right.substring(beginIndex - leftLength, endIndex - leftLength); } - return result; + + // Split substring + ByteString leftSub = left.substring(beginIndex); + ByteString rightSub = right.substring(0, endIndex - leftLength); + // Intentionally not rebalancing, since in many cases these two + // substrings will already be less deep than the top-level + // RopeByteString we're taking a substring of. + return new RopeByteString(leftSub, rightSub); } // ================================================================= @@ -391,7 +373,7 @@ class RopeByteString extends ByteString { List result = new ArrayList(); PieceIterator pieces = new PieceIterator(this); while (pieces.hasNext()) { - LiteralByteString byteString = pieces.next(); + LeafByteString byteString = pieces.next(); result.add(byteString.asReadOnlyByteBuffer()); } return result; @@ -471,11 +453,10 @@ class RopeByteString extends ByteString { // hashCode if it's already computed. It's arguable we should compute the // hashCode here, and if we're going to be testing a bunch of byteStrings, // it might even make sense. - if (hash != 0) { - int cachedOtherHash = otherByteString.peekCachedHashCode(); - if (cachedOtherHash != 0 && hash != cachedOtherHash) { - return false; - } + int thisHash = peekCachedHashCode(); + int thatHash = otherByteString.peekCachedHashCode(); + if (thisHash != 0 && thatHash != 0 && thisHash != thatHash) { + return false; } return equalsFragments(otherByteString); @@ -492,12 +473,12 @@ class RopeByteString extends ByteString { */ private boolean equalsFragments(ByteString other) { int thisOffset = 0; - Iterator thisIter = new PieceIterator(this); - LiteralByteString thisString = thisIter.next(); + Iterator thisIter = new PieceIterator(this); + LeafByteString thisString = thisIter.next(); int thatOffset = 0; - Iterator thatIter = new PieceIterator(other); - LiteralByteString thatString = thatIter.next(); + Iterator thatIter = new PieceIterator(other); + LeafByteString thatString = thatIter.next(); int pos = 0; while (true) { @@ -536,33 +517,6 @@ class RopeByteString extends ByteString { } } - /** - * Cached hash value. Intentionally accessed via a data race, which is safe - * because of the Java Memory Model's "no out-of-thin-air values" guarantees - * for ints. - */ - private int hash = 0; - - @Override - public int hashCode() { - int h = hash; - - if (h == 0) { - h = totalLength; - h = partialHash(h, 0, totalLength); - if (h == 0) { - h = 1; - } - hash = h; - } - return h; - } - - @Override - protected int peekCachedHashCode() { - return hash; - } - @Override protected int partialHash(int h, int offset, int length) { int toIndex = offset + length; @@ -714,34 +668,34 @@ class RopeByteString extends ByteString { *

This iterator is used to implement * {@link RopeByteString#equalsFragments(ByteString)}. */ - private static class PieceIterator implements Iterator { + private static class PieceIterator implements Iterator { private final Stack breadCrumbs = new Stack(); - private LiteralByteString next; + private LeafByteString next; private PieceIterator(ByteString root) { next = getLeafByLeft(root); } - private LiteralByteString getLeafByLeft(ByteString root) { + private LeafByteString getLeafByLeft(ByteString root) { ByteString pos = root; while (pos instanceof RopeByteString) { RopeByteString rbs = (RopeByteString) pos; breadCrumbs.push(rbs); pos = rbs.left; } - return (LiteralByteString) pos; + return (LeafByteString) pos; } - private LiteralByteString getNextNonEmptyLeaf() { + private LeafByteString getNextNonEmptyLeaf() { while (true) { // Almost always, we go through this loop exactly once. However, if // we discover an empty string in the rope, we toss it and try again. if (breadCrumbs.isEmpty()) { return null; } else { - LiteralByteString result = getLeafByLeft(breadCrumbs.pop().right); + LeafByteString result = getLeafByLeft(breadCrumbs.pop().right); if (!result.isEmpty()) { return result; } @@ -749,6 +703,7 @@ class RopeByteString extends ByteString { } } + @Override public boolean hasNext() { return next != null; } @@ -758,15 +713,17 @@ class RopeByteString extends ByteString { * * @return next non-empty LiteralByteString or {@code null} */ - public LiteralByteString next() { + @Override + public LeafByteString next() { if (next == null) { throw new NoSuchElementException(); } - LiteralByteString result = next; + LeafByteString result = next; next = getNextNonEmptyLeaf(); return result; } + @Override public void remove() { throw new UnsupportedOperationException(); } @@ -781,52 +738,11 @@ class RopeByteString extends ByteString { return new LiteralByteString(toByteArray()); } - private void readObject(ObjectInputStream in) throws IOException { + private void readObject(@SuppressWarnings("unused") ObjectInputStream in) throws IOException { throw new InvalidObjectException( "RopeByteStream instances are not to be serialized directly"); } - // ================================================================= - // ByteIterator - - @Override - public ByteIterator iterator() { - return new RopeByteIterator(); - } - - private class RopeByteIterator implements ByteString.ByteIterator { - - private final PieceIterator pieces; - private ByteIterator bytes; - int bytesRemaining; - - private RopeByteIterator() { - pieces = new PieceIterator(RopeByteString.this); - bytes = pieces.next().iterator(); - bytesRemaining = size(); - } - - public boolean hasNext() { - return (bytesRemaining > 0); - } - - public Byte next() { - return nextByte(); // Does not instantiate a Byte - } - - public byte nextByte() { - if (!bytes.hasNext()) { - bytes = pieces.next().iterator(); - } - --bytesRemaining; - return bytes.nextByte(); - } - - public void remove() { - throw new UnsupportedOperationException(); - } - } - /** * This class is the {@link RopeByteString} equivalent for * {@link ByteArrayInputStream}. @@ -835,7 +751,7 @@ class RopeByteString extends ByteString { // Iterates through the pieces of the rope private PieceIterator pieceIterator; // The current piece - private LiteralByteString currentPiece; + private LeafByteString currentPiece; // The size of the current piece private int currentPieceSize; // The index of the next byte to read in the current piece @@ -872,7 +788,7 @@ class RopeByteString extends ByteString { /** * Internal implementation of read and skip. If b != null, then read the * next {@code length} bytes into the buffer {@code b} at - * offset {@code offset}. If b == null, then skip the next {@code length) + * offset {@code offset}. If b == null, then skip the next {@code length} * bytes. *

* This method assumes that all error checking has already happened. diff --git a/java/src/main/java/com/google/protobuf/TextFormat.java b/java/src/main/java/com/google/protobuf/TextFormat.java index 44d036c1..c99b5285 100644 --- a/java/src/main/java/com/google/protobuf/TextFormat.java +++ b/java/src/main/java/com/google/protobuf/TextFormat.java @@ -1074,6 +1074,18 @@ public final class TextFormat { private ParseException floatParseException(final NumberFormatException e) { return parseException("Couldn't parse number: " + e.getMessage()); } + + /** + * Returns a {@link UnknownFieldParseException} with the line and column + * numbers of the previous token in the description, and the unknown field + * name, suitable for throwing. + */ + public UnknownFieldParseException unknownFieldParseExceptionPreviousToken( + final String unknownField, final String description) { + // Note: People generally prefer one-based line and column numbers. + return new UnknownFieldParseException( + previousLine + 1, previousColumn + 1, unknownField, description); + } } /** Thrown when parsing an invalid text format message. */ @@ -1121,6 +1133,45 @@ public final class TextFormat { return column; } } + + /** + * Thrown when encountering an unknown field while parsing + * a text format message. + */ + public static class UnknownFieldParseException extends ParseException { + private final String unknownField; + + /** + * Create a new instance, with -1 as the line and column numbers, and an + * empty unknown field name. + */ + public UnknownFieldParseException(final String message) { + this(-1, -1, "", message); + } + + /** + * Create a new instance + * + * @param line the line number where the parse error occurred, + * using 1-offset. + * @param column the column number where the parser error occurred, + * using 1-offset. + * @param unknownField the name of the unknown field found while parsing. + */ + public UnknownFieldParseException(final int line, final int column, + final String unknownField, final String message) { + super(line, column, message); + this.unknownField = unknownField; + } + + /** + * Return the name of the unknown field encountered while parsing the + * protocol buffer string. + */ + public String getUnknownField() { + return unknownField; + } + } private static final Parser PARSER = Parser.newBuilder().build(); @@ -1388,7 +1439,8 @@ public final class TextFormat { if (field == null) { if (!allowUnknownFields) { - throw tokenizer.parseExceptionPreviousToken( + throw tokenizer.unknownFieldParseExceptionPreviousToken( + name, "Message type \"" + type.getFullName() + "\" has no field named \"" + name + "\"."); } else { diff --git a/java/src/main/java/com/google/protobuf/TextFormatEscaper.java b/java/src/main/java/com/google/protobuf/TextFormatEscaper.java new file mode 100644 index 00000000..e69de29b diff --git a/java/src/main/java/com/google/protobuf/UnsafeByteStrings.java b/java/src/main/java/com/google/protobuf/UnsafeByteStrings.java new file mode 100644 index 00000000..c1997515 --- /dev/null +++ b/java/src/main/java/com/google/protobuf/UnsafeByteStrings.java @@ -0,0 +1,55 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import java.nio.ByteBuffer; + +/** + * Provides unsafe factory methods for {@link ByteString} instances. + * + *

DISCLAIMER: The methods in this class should only be called if it is + * guaranteed that the the buffer backing the {@link ByteString} will never change! Mutation of a + * {@link ByteString} can lead to unexpected and undesirable consequences in your application, + * and will likely be difficult to debug. Proceed with caution! + */ +public final class UnsafeByteStrings { + private UnsafeByteStrings() {} + + /** + * An unsafe operation that returns a {@link ByteString} that is backed by the provided buffer. + * + * @param buffer the Java NIO buffer to be wrapped. + * @return a {@link ByteString} backed by the provided buffer. + */ + public static ByteString unsafeWrap(ByteBuffer buffer) { + return new NioByteString(buffer); + } +} diff --git a/java/src/test/java/com/google/protobuf/BooleanArrayListTest.java b/java/src/test/java/com/google/protobuf/BooleanArrayListTest.java index df89c263..b8ad1fe4 100644 --- a/java/src/test/java/com/google/protobuf/BooleanArrayListTest.java +++ b/java/src/test/java/com/google/protobuf/BooleanArrayListTest.java @@ -310,10 +310,6 @@ public class BooleanArrayListTest extends TestCase { } private void assertImmutable(BooleanArrayList list) { - if (list.contains(1)) { - throw new RuntimeException("Cannot test the immutability of lists that contain 1."); - } - try { list.add(false); fail(); @@ -413,7 +409,7 @@ public class BooleanArrayListTest extends TestCase { } try { - list.removeAll(Collections.singleton(1)); + list.removeAll(Collections.singleton(Boolean.TRUE)); fail(); } catch (UnsupportedOperationException e) { // expected @@ -434,7 +430,7 @@ public class BooleanArrayListTest extends TestCase { } try { - list.retainAll(Collections.singleton(1)); + list.retainAll(Collections.singleton(Boolean.TRUE)); fail(); } catch (UnsupportedOperationException e) { // expected diff --git a/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java b/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java index 447e6ef3..2dfae2e6 100644 --- a/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/BoundedByteStringTest.java @@ -73,7 +73,7 @@ public class BoundedByteStringTest extends LiteralByteStringTest { } @Override - public void testCharsetToString() throws UnsupportedEncodingException { + public void testCharsetToString() { String testString = "I love unicode \u1234\u5678 characters"; LiteralByteString unicode = new LiteralByteString(testString.getBytes(Internal.UTF_8)); ByteString chopped = unicode.substring(2, unicode.size() - 6); diff --git a/java/src/test/java/com/google/protobuf/ByteStringTest.java b/java/src/test/java/com/google/protobuf/ByteStringTest.java index 8af5dcde..36f64251 100644 --- a/java/src/test/java/com/google/protobuf/ByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/ByteStringTest.java @@ -39,7 +39,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.ArrayList; @@ -129,7 +128,7 @@ public class ByteStringTest extends TestCase { isArrayRange(byteString.toByteArray(), bytes, 500, bytes.length - 500)); } - public void testCopyFrom_StringEncoding() throws UnsupportedEncodingException { + public void testCopyFrom_StringEncoding() { String testString = "I love unicode \u1234\u5678 characters"; ByteString byteString = ByteString.copyFrom(testString, UTF_16); byte[] testBytes = testString.getBytes(UTF_16); @@ -137,7 +136,7 @@ public class ByteStringTest extends TestCase { isArrayRange(byteString.toByteArray(), testBytes, 0, testBytes.length)); } - public void testCopyFrom_Utf8() throws UnsupportedEncodingException { + public void testCopyFrom_Utf8() { String testString = "I love unicode \u1234\u5678 characters"; ByteString byteString = ByteString.copyFromUtf8(testString); byte[] testBytes = testString.getBytes(Internal.UTF_8); @@ -154,6 +153,7 @@ public class ByteStringTest extends TestCase { isArrayRange(byteString.toByteArray(), testBytes, 0, testBytes.length)); // Call copyFrom on an iteration that's not a collection ByteString byteStringAlt = ByteString.copyFrom(new Iterable() { + @Override public Iterator iterator() { return pieces.iterator(); } @@ -399,7 +399,7 @@ public class ByteStringTest extends TestCase { } } - public void testToStringUtf8() throws UnsupportedEncodingException { + public void testToStringUtf8() { String testString = "I love unicode \u1234\u5678 characters"; byte[] testBytes = testString.getBytes(Internal.UTF_8); ByteString byteString = ByteString.copyFrom(testBytes); @@ -419,7 +419,7 @@ public class ByteStringTest extends TestCase { // Test newOutput() using a variety of buffer sizes and a variety of (fixed) // write sizes - public void testNewOutput_ArrayWrite() throws IOException { + public void testNewOutput_ArrayWrite() { byte[] bytes = getTestBytes(); int length = bytes.length; int[] bufferSizes = {128, 256, length / 2, length - 1, length, length + 1, @@ -442,7 +442,7 @@ public class ByteStringTest extends TestCase { // Test newOutput() using a variety of buffer sizes, but writing all the // characters using write(byte); - public void testNewOutput_WriteChar() throws IOException { + public void testNewOutput_WriteChar() { byte[] bytes = getTestBytes(); int length = bytes.length; int[] bufferSizes = {0, 1, 128, 256, length / 2, @@ -461,7 +461,7 @@ public class ByteStringTest extends TestCase { // Test newOutput() in which we write the bytes using a variety of methods // and sizes, and in which we repeatedly call toByteString() in the middle. - public void testNewOutput_Mixed() throws IOException { + public void testNewOutput_Mixed() { Random rng = new Random(1); byte[] bytes = getTestBytes(); int length = bytes.length; @@ -494,7 +494,7 @@ public class ByteStringTest extends TestCase { } } - public void testNewOutputEmpty() throws IOException { + public void testNewOutputEmpty() { // Make sure newOutput() correctly builds empty byte strings ByteString byteString = ByteString.newOutput().toByteString(); assertEquals(ByteString.EMPTY, byteString); diff --git a/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java b/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java index 360e759e..6018ea55 100644 --- a/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java +++ b/java/src/test/java/com/google/protobuf/CodedOutputStreamTest.java @@ -37,6 +37,7 @@ import protobuf_unittest.UnittestProto.TestSparseEnum; import junit.framework.TestCase; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -80,8 +81,8 @@ public class CodedOutputStreamTest extends TestCase { * checks that the result matches the given bytes. */ private void assertWriteVarint(byte[] data, long value) throws Exception { - // Only do 32-bit write if the value fits in 32 bits. - if ((value >>> 32) == 0) { + // Only test 32-bit write if the value fits into an int. + if (value == (int) value) { ByteArrayOutputStream rawOutput = new ByteArrayOutputStream(); CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); output.writeRawVarint32((int) value); @@ -107,8 +108,8 @@ public class CodedOutputStreamTest extends TestCase { // Try different block sizes. for (int blockSize = 1; blockSize <= 16; blockSize *= 2) { - // Only do 32-bit write if the value fits in 32 bits. - if ((value >>> 32) == 0) { + // Only test 32-bit write if the value fits into an int. + if (value == (int) value) { ByteArrayOutputStream rawOutput = new ByteArrayOutputStream(); CodedOutputStream output = CodedOutputStream.newInstance(rawOutput, blockSize); @@ -128,6 +129,42 @@ public class CodedOutputStreamTest extends TestCase { } } + private void assertVarintRoundTrip(long value) throws Exception { + { + ByteArrayOutputStream rawOutput = new ByteArrayOutputStream(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); + output.writeRawVarint64(value); + output.flush(); + byte[] bytes = rawOutput.toByteArray(); + assertEquals(bytes.length, CodedOutputStream.computeRawVarint64Size(value)); + CodedInputStream input = CodedInputStream.newInstance(new ByteArrayInputStream(bytes)); + assertEquals(value, input.readRawVarint64()); + } + + if (value == (int) value) { + ByteArrayOutputStream rawOutput = new ByteArrayOutputStream(); + CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); + output.writeRawVarint32((int) value); + output.flush(); + byte[] bytes = rawOutput.toByteArray(); + assertEquals(bytes.length, CodedOutputStream.computeRawVarint32Size((int) value)); + CodedInputStream input = CodedInputStream.newInstance(new ByteArrayInputStream(bytes)); + assertEquals(value, input.readRawVarint32()); + } + } + + /** Checks that invariants are maintained for varint round trip input and output. */ + public void testVarintRoundTrips() throws Exception { + assertVarintRoundTrip(0L); + for (int bits = 0; bits < 64; bits++) { + long value = 1L << bits; + assertVarintRoundTrip(value); + assertVarintRoundTrip(value + 1); + assertVarintRoundTrip(value - 1); + assertVarintRoundTrip(-value); + } + } + /** Tests writeRawVarint32() and writeRawVarint64(). */ public void testWriteVarint() throws Exception { assertWriteVarint(bytes(0x00), 0); diff --git a/java/src/test/java/com/google/protobuf/DescriptorsTest.java b/java/src/test/java/com/google/protobuf/DescriptorsTest.java index 30da2487..82ff34af 100644 --- a/java/src/test/java/com/google/protobuf/DescriptorsTest.java +++ b/java/src/test/java/com/google/protobuf/DescriptorsTest.java @@ -274,6 +274,15 @@ public class DescriptorsTest extends TestCase { assertFalse(repeatedField.isRequired()); assertTrue(repeatedField.isRepeated()); } + + public void testFieldDescriptorJsonName() throws Exception { + FieldDescriptor requiredField = TestRequired.getDescriptor().findFieldByName("a"); + FieldDescriptor optionalField = TestAllTypes.getDescriptor().findFieldByName("optional_int32"); + FieldDescriptor repeatedField = TestAllTypes.getDescriptor().findFieldByName("repeated_int32"); + assertEquals("a", requiredField.getJsonName()); + assertEquals("optionalInt32", optionalField.getJsonName()); + assertEquals("repeatedInt32", repeatedField.getJsonName()); + } public void testFieldDescriptorDefault() throws Exception { Descriptor d = TestAllTypes.getDescriptor(); diff --git a/java/src/test/java/com/google/protobuf/DoubleArrayListTest.java b/java/src/test/java/com/google/protobuf/DoubleArrayListTest.java index e7a73d70..d3deaa07 100644 --- a/java/src/test/java/com/google/protobuf/DoubleArrayListTest.java +++ b/java/src/test/java/com/google/protobuf/DoubleArrayListTest.java @@ -310,7 +310,7 @@ public class DoubleArrayListTest extends TestCase { } private void assertImmutable(DoubleArrayList list) { - if (list.contains(1)) { + if (list.contains(1D)) { throw new RuntimeException("Cannot test the immutability of lists that contain 1."); } @@ -413,7 +413,7 @@ public class DoubleArrayListTest extends TestCase { } try { - list.removeAll(Collections.singleton(1)); + list.removeAll(Collections.singleton(1D)); fail(); } catch (UnsupportedOperationException e) { // expected @@ -434,7 +434,7 @@ public class DoubleArrayListTest extends TestCase { } try { - list.retainAll(Collections.singleton(1)); + list.retainAll(Collections.singleton(1D)); fail(); } catch (UnsupportedOperationException e) { // expected diff --git a/java/src/test/java/com/google/protobuf/FloatArrayListTest.java b/java/src/test/java/com/google/protobuf/FloatArrayListTest.java index 8f3e93dc..a5e65424 100644 --- a/java/src/test/java/com/google/protobuf/FloatArrayListTest.java +++ b/java/src/test/java/com/google/protobuf/FloatArrayListTest.java @@ -310,7 +310,7 @@ public class FloatArrayListTest extends TestCase { } private void assertImmutable(FloatArrayList list) { - if (list.contains(1)) { + if (list.contains(1F)) { throw new RuntimeException("Cannot test the immutability of lists that contain 1."); } @@ -413,7 +413,7 @@ public class FloatArrayListTest extends TestCase { } try { - list.removeAll(Collections.singleton(1)); + list.removeAll(Collections.singleton(1F)); fail(); } catch (UnsupportedOperationException e) { // expected @@ -434,7 +434,7 @@ public class FloatArrayListTest extends TestCase { } try { - list.retainAll(Collections.singleton(1)); + list.retainAll(Collections.singleton(1F)); fail(); } catch (UnsupportedOperationException e) { // expected diff --git a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java index 7dfda2ae..c8495633 100644 --- a/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java +++ b/java/src/test/java/com/google/protobuf/LiteralByteStringTest.java @@ -355,7 +355,7 @@ public class LiteralByteStringTest extends TestCase { assertEquals(classUnderTest + " unicode must match", testString, roundTripString); } - public void testCharsetToString() throws UnsupportedEncodingException { + public void testCharsetToString() { String testString = "I love unicode \u1234\u5678 characters"; LiteralByteString unicode = new LiteralByteString(testString.getBytes(Internal.UTF_8)); String roundTripString = unicode.toString(Internal.UTF_8); diff --git a/java/src/test/java/com/google/protobuf/LongArrayListTest.java b/java/src/test/java/com/google/protobuf/LongArrayListTest.java index 3a52ec7f..1bd094f7 100644 --- a/java/src/test/java/com/google/protobuf/LongArrayListTest.java +++ b/java/src/test/java/com/google/protobuf/LongArrayListTest.java @@ -310,7 +310,7 @@ public class LongArrayListTest extends TestCase { } private void assertImmutable(LongArrayList list) { - if (list.contains(1)) { + if (list.contains(1L)) { throw new RuntimeException("Cannot test the immutability of lists that contain 1."); } @@ -413,7 +413,7 @@ public class LongArrayListTest extends TestCase { } try { - list.removeAll(Collections.singleton(1)); + list.removeAll(Collections.singleton(1L)); fail(); } catch (UnsupportedOperationException e) { // expected @@ -434,7 +434,7 @@ public class LongArrayListTest extends TestCase { } try { - list.retainAll(Collections.singleton(1)); + list.retainAll(Collections.singleton(1L)); fail(); } catch (UnsupportedOperationException e) { // expected diff --git a/java/src/test/java/com/google/protobuf/NioByteStringTest.java b/java/src/test/java/com/google/protobuf/NioByteStringTest.java new file mode 100644 index 00000000..0679937f --- /dev/null +++ b/java/src/test/java/com/google/protobuf/NioByteStringTest.java @@ -0,0 +1,546 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package com.google.protobuf; + +import static com.google.protobuf.Internal.UTF_8; + +import junit.framework.TestCase; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * Tests for {@link NioByteString}. + */ +public class NioByteStringTest extends TestCase { + private static final ByteString EMPTY = UnsafeByteStrings.unsafeWrap( + ByteBuffer.wrap(new byte[0])); + private static final String CLASSNAME = NioByteString.class.getSimpleName(); + private static final byte[] BYTES = ByteStringTest.getTestBytes(1234, 11337766L); + private static final int EXPECTED_HASH = new LiteralByteString(BYTES).hashCode(); + private static final ByteBuffer BUFFER = ByteBuffer.wrap(BYTES.clone()); + private static final ByteString TEST_STRING = UnsafeByteStrings.unsafeWrap(BUFFER); + + public void testExpectedType() { + String actualClassName = getActualClassName(TEST_STRING); + assertEquals(CLASSNAME + " should match type exactly", CLASSNAME, actualClassName); + } + + protected String getActualClassName(Object object) { + String actualClassName = object.getClass().getName(); + actualClassName = actualClassName.substring(actualClassName.lastIndexOf('.') + 1); + return actualClassName; + } + + public void testByteAt() { + boolean stillEqual = true; + for (int i = 0; stillEqual && i < BYTES.length; ++i) { + stillEqual = (BYTES[i] == TEST_STRING.byteAt(i)); + } + assertTrue(CLASSNAME + " must capture the right bytes", stillEqual); + } + + public void testByteIterator() { + boolean stillEqual = true; + ByteString.ByteIterator iter = TEST_STRING.iterator(); + for (int i = 0; stillEqual && i < BYTES.length; ++i) { + stillEqual = (iter.hasNext() && BYTES[i] == iter.nextByte()); + } + assertTrue(CLASSNAME + " must capture the right bytes", stillEqual); + assertFalse(CLASSNAME + " must have exhausted the itertor", iter.hasNext()); + + try { + iter.nextByte(); + fail("Should have thrown an exception."); + } catch (NoSuchElementException e) { + // This is success + } + } + + public void testByteIterable() { + boolean stillEqual = true; + int j = 0; + for (byte quantum : TEST_STRING) { + stillEqual = (BYTES[j] == quantum); + ++j; + } + assertTrue(CLASSNAME + " must capture the right bytes as Bytes", stillEqual); + assertEquals(CLASSNAME + " iterable character count", BYTES.length, j); + } + + public void testSize() { + assertEquals(CLASSNAME + " must have the expected size", BYTES.length, + TEST_STRING.size()); + } + + public void testGetTreeDepth() { + assertEquals(CLASSNAME + " must have depth 0", 0, TEST_STRING.getTreeDepth()); + } + + public void testIsBalanced() { + assertTrue(CLASSNAME + " is technically balanced", TEST_STRING.isBalanced()); + } + + public void testCopyTo_ByteArrayOffsetLength() { + int destinationOffset = 50; + int length = 100; + byte[] destination = new byte[destinationOffset + length]; + int sourceOffset = 213; + TEST_STRING.copyTo(destination, sourceOffset, destinationOffset, length); + boolean stillEqual = true; + for (int i = 0; stillEqual && i < length; ++i) { + stillEqual = BYTES[i + sourceOffset] == destination[i + destinationOffset]; + } + assertTrue(CLASSNAME + ".copyTo(4 arg) must give the expected bytes", stillEqual); + } + + public void testCopyTo_ByteArrayOffsetLengthErrors() { + int destinationOffset = 50; + int length = 100; + byte[] destination = new byte[destinationOffset + length]; + + try { + // Copy one too many bytes + TEST_STRING.copyTo(destination, TEST_STRING.size() + 1 - length, + destinationOffset, length); + fail("Should have thrown an exception when copying too many bytes of a " + + CLASSNAME); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal negative sourceOffset + TEST_STRING.copyTo(destination, -1, destinationOffset, length); + fail("Should have thrown an exception when given a negative sourceOffset in " + + CLASSNAME); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal negative destinationOffset + TEST_STRING.copyTo(destination, 0, -1, length); + fail("Should have thrown an exception when given a negative destinationOffset in " + + CLASSNAME); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal negative size + TEST_STRING.copyTo(destination, 0, 0, -1); + fail("Should have thrown an exception when given a negative size in " + + CLASSNAME); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal too-large sourceOffset + TEST_STRING.copyTo(destination, 2 * TEST_STRING.size(), 0, length); + fail("Should have thrown an exception when the destinationOffset is too large in " + + CLASSNAME); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + + try { + // Copy with illegal too-large destinationOffset + TEST_STRING.copyTo(destination, 0, 2 * destination.length, length); + fail("Should have thrown an exception when the destinationOffset is too large in " + + CLASSNAME); + } catch (IndexOutOfBoundsException expected) { + // This is success + } + } + + public void testCopyTo_ByteBuffer() { + // Same length. + ByteBuffer myBuffer = ByteBuffer.allocate(BYTES.length); + TEST_STRING.copyTo(myBuffer); + myBuffer.flip(); + assertEquals(CLASSNAME + ".copyTo(ByteBuffer) must give back the same bytes", + BUFFER, myBuffer); + + // Target buffer bigger than required. + myBuffer = ByteBuffer.allocate(TEST_STRING.size() + 1); + TEST_STRING.copyTo(myBuffer); + myBuffer.flip(); + assertEquals(BUFFER, myBuffer); + + // Target buffer has no space. + myBuffer = ByteBuffer.allocate(0); + try { + TEST_STRING.copyTo(myBuffer); + fail("Should have thrown an exception when target ByteBuffer has insufficient capacity"); + } catch (BufferOverflowException e) { + // Expected. + } + + // Target buffer too small. + myBuffer = ByteBuffer.allocate(1); + try { + TEST_STRING.copyTo(myBuffer); + fail("Should have thrown an exception when target ByteBuffer has insufficient capacity"); + } catch (BufferOverflowException e) { + // Expected. + } + } + + public void testMarkSupported() { + InputStream stream = TEST_STRING.newInput(); + assertTrue(CLASSNAME + ".newInput() must support marking", stream.markSupported()); + } + + public void testMarkAndReset() throws IOException { + int fraction = TEST_STRING.size() / 3; + + InputStream stream = TEST_STRING.newInput(); + stream.mark(TEST_STRING.size()); // First, mark() the end. + + skipFully(stream, fraction); // Skip a large fraction, but not all. + assertEquals( + CLASSNAME + ": after skipping to the 'middle', half the bytes are available", + (TEST_STRING.size() - fraction), stream.available()); + stream.reset(); + assertEquals( + CLASSNAME + ": after resetting, all bytes are available", + TEST_STRING.size(), stream.available()); + + skipFully(stream, TEST_STRING.size()); // Skip to the end. + assertEquals( + CLASSNAME + ": after skipping to the end, no more bytes are available", + 0, stream.available()); + } + + /** + * Discards {@code n} bytes of data from the input stream. This method + * will block until the full amount has been skipped. Does not close the + * stream. + *

Copied from com.google.common.io.ByteStreams to avoid adding dependency. + * + * @param in the input stream to read from + * @param n the number of bytes to skip + * @throws EOFException if this stream reaches the end before skipping all + * the bytes + * @throws IOException if an I/O error occurs, or the stream does not + * support skipping + */ + static void skipFully(InputStream in, long n) throws IOException { + long toSkip = n; + while (n > 0) { + long amt = in.skip(n); + if (amt == 0) { + // Force a blocking read to avoid infinite loop + if (in.read() == -1) { + long skipped = toSkip - n; + throw new EOFException("reached end of stream after skipping " + + skipped + " bytes; " + toSkip + " bytes expected"); + } + n--; + } else { + n -= amt; + } + } + } + + public void testAsReadOnlyByteBuffer() { + ByteBuffer byteBuffer = TEST_STRING.asReadOnlyByteBuffer(); + byte[] roundTripBytes = new byte[BYTES.length]; + assertTrue(byteBuffer.remaining() == BYTES.length); + assertTrue(byteBuffer.isReadOnly()); + byteBuffer.get(roundTripBytes); + assertTrue(CLASSNAME + ".asReadOnlyByteBuffer() must give back the same bytes", + Arrays.equals(BYTES, roundTripBytes)); + } + + public void testAsReadOnlyByteBufferList() { + List byteBuffers = TEST_STRING.asReadOnlyByteBufferList(); + int bytesSeen = 0; + byte[] roundTripBytes = new byte[BYTES.length]; + for (ByteBuffer byteBuffer : byteBuffers) { + int thisLength = byteBuffer.remaining(); + assertTrue(byteBuffer.isReadOnly()); + assertTrue(bytesSeen + thisLength <= BYTES.length); + byteBuffer.get(roundTripBytes, bytesSeen, thisLength); + bytesSeen += thisLength; + } + assertTrue(bytesSeen == BYTES.length); + assertTrue(CLASSNAME + ".asReadOnlyByteBufferTest() must give back the same bytes", + Arrays.equals(BYTES, roundTripBytes)); + } + + public void testToByteArray() { + byte[] roundTripBytes = TEST_STRING.toByteArray(); + assertTrue(CLASSNAME + ".toByteArray() must give back the same bytes", + Arrays.equals(BYTES, roundTripBytes)); + } + + public void testWriteTo() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + TEST_STRING.writeTo(bos); + byte[] roundTripBytes = bos.toByteArray(); + assertTrue(CLASSNAME + ".writeTo() must give back the same bytes", + Arrays.equals(BYTES, roundTripBytes)); + } + + public void testNewOutput() throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteString.Output output = ByteString.newOutput(); + TEST_STRING.writeTo(output); + assertEquals("Output Size returns correct result", + output.size(), TEST_STRING.size()); + output.writeTo(bos); + assertTrue("Output.writeTo() must give back the same bytes", + Arrays.equals(BYTES, bos.toByteArray())); + + // write the output stream to itself! This should cause it to double + output.writeTo(output); + assertEquals("Writing an output stream to itself is successful", + TEST_STRING.concat(TEST_STRING), output.toByteString()); + + output.reset(); + assertEquals("Output.reset() resets the output", 0, output.size()); + assertEquals("Output.reset() resets the output", + EMPTY, output.toByteString()); + } + + public void testToString() { + String testString = "I love unicode \u1234\u5678 characters"; + ByteString unicode = forString(testString); + String roundTripString = unicode.toString(UTF_8); + assertEquals(CLASSNAME + " unicode must match", testString, roundTripString); + } + + public void testCharsetToString() { + String testString = "I love unicode \u1234\u5678 characters"; + ByteString unicode = forString(testString); + String roundTripString = unicode.toString(UTF_8); + assertEquals(CLASSNAME + " unicode must match", testString, roundTripString); + } + + public void testToString_returnsCanonicalEmptyString() { + assertSame(CLASSNAME + " must be the same string references", + EMPTY.toString(UTF_8), + UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(new byte[0])).toString(UTF_8)); + } + + public void testToString_raisesException() { + try { + EMPTY.toString("invalid"); + fail("Should have thrown an exception."); + } catch (UnsupportedEncodingException expected) { + // This is success + } + + try { + TEST_STRING.toString("invalid"); + fail("Should have thrown an exception."); + } catch (UnsupportedEncodingException expected) { + // This is success + } + } + + public void testEquals() { + assertEquals(CLASSNAME + " must not equal null", false, TEST_STRING.equals(null)); + assertEquals(CLASSNAME + " must equal self", TEST_STRING, TEST_STRING); + assertFalse(CLASSNAME + " must not equal the empty string", + TEST_STRING.equals(EMPTY)); + assertEquals(CLASSNAME + " empty strings must be equal", + EMPTY, TEST_STRING.substring(55, 55)); + assertEquals(CLASSNAME + " must equal another string with the same value", + TEST_STRING, UnsafeByteStrings.unsafeWrap(BUFFER)); + + byte[] mungedBytes = mungedBytes(); + assertFalse(CLASSNAME + " must not equal every string with the same length", + TEST_STRING.equals(UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(mungedBytes)))); + } + + public void testEqualsLiteralByteString() { + ByteString literal = ByteString.copyFrom(BYTES); + assertEquals(CLASSNAME + " must equal LiteralByteString with same value", literal, + TEST_STRING); + assertEquals(CLASSNAME + " must equal LiteralByteString with same value", TEST_STRING, + literal); + assertFalse(CLASSNAME + " must not equal the empty string", + TEST_STRING.equals(ByteString.EMPTY)); + assertEquals(CLASSNAME + " empty strings must be equal", + ByteString.EMPTY, TEST_STRING.substring(55, 55)); + + literal = ByteString.copyFrom(mungedBytes()); + assertFalse(CLASSNAME + " must not equal every LiteralByteString with the same length", + TEST_STRING.equals(literal)); + assertFalse(CLASSNAME + " must not equal every LiteralByteString with the same length", + literal.equals(TEST_STRING)); + } + + public void testEqualsRopeByteString() { + ByteString p1 = ByteString.copyFrom(BYTES, 0, 5); + ByteString p2 = ByteString.copyFrom(BYTES, 5, BYTES.length - 5); + ByteString rope = p1.concat(p2); + + assertEquals(CLASSNAME + " must equal RopeByteString with same value", rope, + TEST_STRING); + assertEquals(CLASSNAME + " must equal RopeByteString with same value", TEST_STRING, + rope); + assertFalse(CLASSNAME + " must not equal the empty string", + TEST_STRING.equals(ByteString.EMPTY.concat(ByteString.EMPTY))); + assertEquals(CLASSNAME + " empty strings must be equal", + ByteString.EMPTY.concat(ByteString.EMPTY), TEST_STRING.substring(55, 55)); + + byte[] mungedBytes = mungedBytes(); + p1 = ByteString.copyFrom(mungedBytes, 0, 5); + p2 = ByteString.copyFrom(mungedBytes, 5, mungedBytes.length - 5); + rope = p1.concat(p2); + assertFalse(CLASSNAME + " must not equal every RopeByteString with the same length", + TEST_STRING.equals(rope)); + assertFalse(CLASSNAME + " must not equal every RopeByteString with the same length", + rope.equals(TEST_STRING)); + } + + private byte[] mungedBytes() { + byte[] mungedBytes = new byte[BYTES.length]; + System.arraycopy(BYTES, 0, mungedBytes, 0, BYTES.length); + mungedBytes[mungedBytes.length - 5] = (byte) (mungedBytes[mungedBytes.length - 5] ^ 0xFF); + return mungedBytes; + } + + public void testHashCode() { + int hash = TEST_STRING.hashCode(); + assertEquals(CLASSNAME + " must have expected hashCode", EXPECTED_HASH, hash); + } + + public void testPeekCachedHashCode() { + ByteString newString = UnsafeByteStrings.unsafeWrap(BUFFER); + assertEquals(CLASSNAME + ".peekCachedHashCode() should return zero at first", 0, + newString.peekCachedHashCode()); + newString.hashCode(); + assertEquals(CLASSNAME + ".peekCachedHashCode should return zero at first", + EXPECTED_HASH, newString.peekCachedHashCode()); + } + + public void testPartialHash() { + // partialHash() is more strenuously tested elsewhere by testing hashes of substrings. + // This test would fail if the expected hash were 1. It's not. + int hash = TEST_STRING.partialHash(TEST_STRING.size(), 0, TEST_STRING.size()); + assertEquals(CLASSNAME + ".partialHash() must yield expected hashCode", + EXPECTED_HASH, hash); + } + + public void testNewInput() throws IOException { + InputStream input = TEST_STRING.newInput(); + assertEquals("InputStream.available() returns correct value", + TEST_STRING.size(), input.available()); + boolean stillEqual = true; + for (byte referenceByte : BYTES) { + int expectedInt = (referenceByte & 0xFF); + stillEqual = (expectedInt == input.read()); + } + assertEquals("InputStream.available() returns correct value", + 0, input.available()); + assertTrue(CLASSNAME + " must give the same bytes from the InputStream", stillEqual); + assertEquals(CLASSNAME + " InputStream must now be exhausted", -1, input.read()); + } + + public void testNewInput_skip() throws IOException { + InputStream input = TEST_STRING.newInput(); + int stringSize = TEST_STRING.size(); + int nearEndIndex = stringSize * 2 / 3; + long skipped1 = input.skip(nearEndIndex); + assertEquals("InputStream.skip()", skipped1, nearEndIndex); + assertEquals("InputStream.available()", + stringSize - skipped1, input.available()); + assertTrue("InputStream.mark() is available", input.markSupported()); + input.mark(0); + assertEquals("InputStream.skip(), read()", + TEST_STRING.byteAt(nearEndIndex) & 0xFF, input.read()); + assertEquals("InputStream.available()", + stringSize - skipped1 - 1, input.available()); + long skipped2 = input.skip(stringSize); + assertEquals("InputStream.skip() incomplete", + skipped2, stringSize - skipped1 - 1); + assertEquals("InputStream.skip(), no more input", 0, input.available()); + assertEquals("InputStream.skip(), no more input", -1, input.read()); + input.reset(); + assertEquals("InputStream.reset() succeded", + stringSize - skipped1, input.available()); + assertEquals("InputStream.reset(), read()", + TEST_STRING.byteAt(nearEndIndex) & 0xFF, input.read()); + } + + public void testNewCodedInput() throws IOException { + CodedInputStream cis = TEST_STRING.newCodedInput(); + byte[] roundTripBytes = cis.readRawBytes(BYTES.length); + assertTrue(CLASSNAME + " must give the same bytes back from the CodedInputStream", + Arrays.equals(BYTES, roundTripBytes)); + assertTrue(CLASSNAME + " CodedInputStream must now be exhausted", cis.isAtEnd()); + } + + /** + * Make sure we keep things simple when concatenating with empty. See also + * {@link ByteStringTest#testConcat_empty()}. + */ + public void testConcat_empty() { + assertSame(CLASSNAME + " concatenated with empty must give " + CLASSNAME, + TEST_STRING.concat(EMPTY), TEST_STRING); + assertSame("empty concatenated with " + CLASSNAME + " must give " + CLASSNAME, + EMPTY.concat(TEST_STRING), TEST_STRING); + } + + public void testJavaSerialization() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(out); + oos.writeObject(TEST_STRING); + oos.close(); + byte[] pickled = out.toByteArray(); + InputStream in = new ByteArrayInputStream(pickled); + ObjectInputStream ois = new ObjectInputStream(in); + Object o = ois.readObject(); + assertTrue("Didn't get a ByteString back", o instanceof ByteString); + assertEquals("Should get an equal ByteString back", TEST_STRING, o); + } + + private static ByteString forString(String str) { + return UnsafeByteStrings.unsafeWrap(ByteBuffer.wrap(str.getBytes(UTF_8))); + } +} diff --git a/java/src/test/java/com/google/protobuf/ProtobufArrayListTest.java b/java/src/test/java/com/google/protobuf/ProtobufArrayListTest.java index f9f5e9c7..245c3dee 100644 --- a/java/src/test/java/com/google/protobuf/ProtobufArrayListTest.java +++ b/java/src/test/java/com/google/protobuf/ProtobufArrayListTest.java @@ -243,7 +243,7 @@ public class ProtobufArrayListTest extends TestCase { } try { - list.removeAll(Collections.emptyList()); + list.removeAll(Collections.emptyList()); fail(); } catch (UnsupportedOperationException e) { // expected @@ -264,7 +264,7 @@ public class ProtobufArrayListTest extends TestCase { } try { - list.retainAll(Collections.emptyList()); + list.retainAll(Collections.emptyList()); fail(); } catch (UnsupportedOperationException e) { // expected diff --git a/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java b/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java index cc385599..dc56f2e9 100644 --- a/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java +++ b/java/src/test/java/com/google/protobuf/RopeByteStringSubstringTest.java @@ -96,7 +96,7 @@ public class RopeByteStringSubstringTest extends LiteralByteStringTest { } @Override - public void testCharsetToString() throws UnsupportedEncodingException { + public void testCharsetToString() { String sourceString = "I love unicode \u1234\u5678 characters"; ByteString sourceByteString = ByteString.copyFromUtf8(sourceString); int copies = 250; diff --git a/java/src/test/java/com/google/protobuf/TestUtil.java b/java/src/test/java/com/google/protobuf/TestUtil.java index 792e8665..01acb884 100644 --- a/java/src/test/java/com/google/protobuf/TestUtil.java +++ b/java/src/test/java/com/google/protobuf/TestUtil.java @@ -299,6 +299,16 @@ public final class TestUtil { return builder; } + /** + * Get a {@code TestAllTypesLite.Builder} with all fields set as they would be by + * {@link #setAllFields(TestAllTypesLite.Builder)}. + */ + public static TestAllTypesLite.Builder getAllLiteSetBuilder() { + TestAllTypesLite.Builder builder = TestAllTypesLite.newBuilder(); + setAllFields(builder); + return builder; + } + /** * Get a {@code TestAllExtensions} with all fields set as they would be by * {@link #setAllExtensions(TestAllExtensions.Builder)}. @@ -339,6 +349,150 @@ public final class TestUtil { setPackedExtensions(builder); return builder.build(); } + + /** + * Set every field of {@code builder} to the values expected by + * {@code assertAllFieldsSet()}. + */ + public static void setAllFields(TestAllTypesLite.Builder builder) { + builder.setOptionalInt32 (101); + builder.setOptionalInt64 (102); + builder.setOptionalUint32 (103); + builder.setOptionalUint64 (104); + builder.setOptionalSint32 (105); + builder.setOptionalSint64 (106); + builder.setOptionalFixed32 (107); + builder.setOptionalFixed64 (108); + builder.setOptionalSfixed32(109); + builder.setOptionalSfixed64(110); + builder.setOptionalFloat (111); + builder.setOptionalDouble (112); + builder.setOptionalBool (true); + builder.setOptionalString ("115"); + builder.setOptionalBytes (toBytes("116")); + + builder.setOptionalGroup( + TestAllTypesLite.OptionalGroup.newBuilder().setA(117).build()); + builder.setOptionalNestedMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(118).build()); + builder.setOptionalForeignMessage( + ForeignMessageLite.newBuilder().setC(119).build()); + builder.setOptionalImportMessage( + ImportMessageLite.newBuilder().setD(120).build()); + builder.setOptionalPublicImportMessage( + PublicImportMessageLite.newBuilder().setE(126).build()); + builder.setOptionalLazyMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(127).build()); + + builder.setOptionalNestedEnum (TestAllTypesLite.NestedEnum.BAZ); + builder.setOptionalForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAZ); + builder.setOptionalImportEnum (ImportEnumLite.IMPORT_LITE_BAZ); + + builder.setOptionalStringPiece("124"); + builder.setOptionalCord("125"); + + // ----------------------------------------------------------------- + + builder.addRepeatedInt32 (201); + builder.addRepeatedInt64 (202); + builder.addRepeatedUint32 (203); + builder.addRepeatedUint64 (204); + builder.addRepeatedSint32 (205); + builder.addRepeatedSint64 (206); + builder.addRepeatedFixed32 (207); + builder.addRepeatedFixed64 (208); + builder.addRepeatedSfixed32(209); + builder.addRepeatedSfixed64(210); + builder.addRepeatedFloat (211); + builder.addRepeatedDouble (212); + builder.addRepeatedBool (true); + builder.addRepeatedString ("215"); + builder.addRepeatedBytes (toBytes("216")); + + builder.addRepeatedGroup( + TestAllTypesLite.RepeatedGroup.newBuilder().setA(217).build()); + builder.addRepeatedNestedMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(218).build()); + builder.addRepeatedForeignMessage( + ForeignMessageLite.newBuilder().setC(219).build()); + builder.addRepeatedImportMessage( + ImportMessageLite.newBuilder().setD(220).build()); + builder.addRepeatedLazyMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(227).build()); + + builder.addRepeatedNestedEnum (TestAllTypesLite.NestedEnum.BAR); + builder.addRepeatedForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAR); + builder.addRepeatedImportEnum (ImportEnumLite.IMPORT_LITE_BAR); + + builder.addRepeatedStringPiece("224"); + builder.addRepeatedCord("225"); + + // Add a second one of each field. + builder.addRepeatedInt32 (301); + builder.addRepeatedInt64 (302); + builder.addRepeatedUint32 (303); + builder.addRepeatedUint64 (304); + builder.addRepeatedSint32 (305); + builder.addRepeatedSint64 (306); + builder.addRepeatedFixed32 (307); + builder.addRepeatedFixed64 (308); + builder.addRepeatedSfixed32(309); + builder.addRepeatedSfixed64(310); + builder.addRepeatedFloat (311); + builder.addRepeatedDouble (312); + builder.addRepeatedBool (false); + builder.addRepeatedString ("315"); + builder.addRepeatedBytes (toBytes("316")); + + builder.addRepeatedGroup( + TestAllTypesLite.RepeatedGroup.newBuilder().setA(317).build()); + builder.addRepeatedNestedMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(318).build()); + builder.addRepeatedForeignMessage( + ForeignMessageLite.newBuilder().setC(319).build()); + builder.addRepeatedImportMessage( + ImportMessageLite.newBuilder().setD(320).build()); + builder.addRepeatedLazyMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(327).build()); + + builder.addRepeatedNestedEnum (TestAllTypesLite.NestedEnum.BAZ); + builder.addRepeatedForeignEnum(ForeignEnumLite.FOREIGN_LITE_BAZ); + builder.addRepeatedImportEnum (ImportEnumLite.IMPORT_LITE_BAZ); + + builder.addRepeatedStringPiece("324"); + builder.addRepeatedCord("325"); + + // ----------------------------------------------------------------- + + builder.setDefaultInt32 (401); + builder.setDefaultInt64 (402); + builder.setDefaultUint32 (403); + builder.setDefaultUint64 (404); + builder.setDefaultSint32 (405); + builder.setDefaultSint64 (406); + builder.setDefaultFixed32 (407); + builder.setDefaultFixed64 (408); + builder.setDefaultSfixed32(409); + builder.setDefaultSfixed64(410); + builder.setDefaultFloat (411); + builder.setDefaultDouble (412); + builder.setDefaultBool (false); + builder.setDefaultString ("415"); + builder.setDefaultBytes (toBytes("416")); + + builder.setDefaultNestedEnum (TestAllTypesLite.NestedEnum.FOO); + builder.setDefaultForeignEnum(ForeignEnumLite.FOREIGN_LITE_FOO); + builder.setDefaultImportEnum (ImportEnumLite.IMPORT_LITE_FOO); + + builder.setDefaultStringPiece("424"); + builder.setDefaultCord("425"); + + builder.setOneofUint32(601); + builder.setOneofNestedMessage( + TestAllTypesLite.NestedMessage.newBuilder().setBb(602).build()); + builder.setOneofString("603"); + builder.setOneofBytes(toBytes("604")); + } /** * Set every field of {@code message} to the values expected by diff --git a/java/src/test/java/com/google/protobuf/map_test.proto b/java/src/test/java/com/google/protobuf/map_test.proto index 2f7709be..2280ac03 100644 --- a/java/src/test/java/com/google/protobuf/map_test.proto +++ b/java/src/test/java/com/google/protobuf/map_test.proto @@ -36,7 +36,6 @@ option java_package = "map_test"; option java_outer_classname = "MapTestProto"; option java_generate_equals_and_hash = true; - message TestMap { message MessageValue { int32 value = 1; diff --git a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto index 8c37c03c..2b1f65e4 100644 --- a/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto +++ b/java/src/test/java/com/google/protobuf/test_bad_identifiers.proto @@ -45,7 +45,6 @@ option java_package = "com.google.protobuf"; option java_outer_classname = "TestBadIdentifiersProto"; option java_generate_equals_and_hash = true; - message TestMessage { optional string cached_size = 1; optional string serialized_size = 2; diff --git a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java index 7bf87858..535be0fa 100644 --- a/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java +++ b/java/util/src/main/java/com/google/protobuf/util/FieldMaskUtil.java @@ -30,6 +30,9 @@ package com.google.protobuf.util; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.primitives.Ints; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.FieldMask; @@ -37,7 +40,6 @@ import com.google.protobuf.Internal; import com.google.protobuf.Message; import java.util.Arrays; -import java.util.List; /** * Utility helper functions to work with {@link com.google.protobuf.FieldMask}. @@ -53,6 +55,7 @@ public class FieldMaskUtil { * Converts a FieldMask to a string. */ public static String toString(FieldMask fieldMask) { + // TODO(xiaofeng): Consider using com.google.common.base.Joiner here instead. StringBuilder result = new StringBuilder(); boolean first = true; for (String value : fieldMask.getPathsList()) { @@ -74,6 +77,7 @@ public class FieldMaskUtil { * Parses from a string to a FieldMask. */ public static FieldMask fromString(String value) { + // TODO(xiaofeng): Consider using com.google.common.base.Splitter here instead. return fromStringList( null, Arrays.asList(value.split(FIELD_PATH_SEPARATOR_REGEX))); } @@ -83,8 +87,8 @@ public class FieldMaskUtil { * * @throws IllegalArgumentException if any of the field path is invalid. */ - public static FieldMask fromString(Class type, String value) - throws IllegalArgumentException { + public static FieldMask fromString(Class type, String value) { + // TODO(xiaofeng): Consider using com.google.common.base.Splitter here instead. return fromStringList( type, Arrays.asList(value.split(FIELD_PATH_SEPARATOR_REGEX))); } @@ -94,9 +98,9 @@ public class FieldMaskUtil { * * @throws IllegalArgumentException if any of the field path is not valid. */ + // TODO(xiaofeng): Consider renaming fromStrings() public static FieldMask fromStringList( - Class type, List paths) - throws IllegalArgumentException { + Class type, Iterable paths) { FieldMask.Builder builder = FieldMask.newBuilder(); for (String path : paths) { if (path.isEmpty()) { @@ -112,16 +116,75 @@ public class FieldMaskUtil { return builder.build(); } + /** + * Constructs a FieldMask from the passed field numbers. + * + * @throws IllegalArugmentException if any of the fields are invalid for the message. + */ + public static FieldMask fromFieldNumbers(Class type, int... fieldNumbers) { + return fromFieldNumbers(type, Ints.asList(fieldNumbers)); + } + + /** + * Constructs a FieldMask from the passed field numbers. + * + * @throws IllegalArugmentException if any of the fields are invalid for the message. + */ + public static FieldMask fromFieldNumbers( + Class type, Iterable fieldNumbers) { + Descriptor descriptor = Internal.getDefaultInstance(type).getDescriptorForType(); + + FieldMask.Builder builder = FieldMask.newBuilder(); + for (Integer fieldNumber : fieldNumbers) { + FieldDescriptor field = descriptor.findFieldByNumber(fieldNumber); + checkArgument( + field != null, + String.format("%s is not a valid field number for %s.", fieldNumber, type)); + builder.addPaths(field.getName()); + } + return builder.build(); + } + + /** + * Checks whether paths in a given fields mask are valid. + */ + public static boolean isValid(Class type, FieldMask fieldMask) { + Descriptor descriptor = + Internal.getDefaultInstance(type).getDescriptorForType(); + + return isValid(descriptor, fieldMask); + } + + /** + * Checks whether paths in a given fields mask are valid. + */ + public static boolean isValid(Descriptor descriptor, FieldMask fieldMask) { + for (String path : fieldMask.getPathsList()) { + if (!isValid(descriptor, path)) { + return false; + } + } + return true; + } + /** * Checks whether a given field path is valid. */ public static boolean isValid(Class type, String path) { + Descriptor descriptor = + Internal.getDefaultInstance(type).getDescriptorForType(); + + return isValid(descriptor, path); + } + + /** + * Checks whether paths in a given fields mask are valid. + */ + public static boolean isValid(Descriptor descriptor, String path) { String[] parts = path.split(FIELD_SEPARATOR_REGEX); if (parts.length == 0) { return false; } - Descriptor descriptor = - Internal.getDefaultInstance(type).getDescriptorForType(); for (String name : parts) { if (descriptor == null) { return false; @@ -171,7 +234,7 @@ public class FieldMaskUtil { /** * Options to customize merging behavior. */ - public static class MergeOptions { + public static final class MergeOptions { private boolean replaceMessageFields = false; private boolean replaceRepeatedFields = false; diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index c9a39153..d13ff0ed 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -78,6 +78,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.logging.Logger; /** @@ -99,7 +100,7 @@ public class JsonFormat { * Creates a {@link Printer} with default configurations. */ public static Printer printer() { - return new Printer(TypeRegistry.getEmptyTypeRegistry()); + return new Printer(TypeRegistry.getEmptyTypeRegistry(), false, false); } /** @@ -107,9 +108,16 @@ public class JsonFormat { */ public static class Printer { private final TypeRegistry registry; - - private Printer(TypeRegistry registry) { + private final boolean includingDefaultValueFields; + private final boolean preservingProtoFieldNames; + + private Printer( + TypeRegistry registry, + boolean includingDefaultValueFields, + boolean preservingProtoFieldNames) { this.registry = registry; + this.includingDefaultValueFields = includingDefaultValueFields; + this.preservingProtoFieldNames = preservingProtoFieldNames; } /** @@ -122,7 +130,27 @@ public class JsonFormat { if (this.registry != TypeRegistry.getEmptyTypeRegistry()) { throw new IllegalArgumentException("Only one registry is allowed."); } - return new Printer(registry); + return new Printer(registry, includingDefaultValueFields, preservingProtoFieldNames); + } + + /** + * Creates a new {@link Printer} that will also print fields set to their + * defaults. Empty repeated fields and map fields will be printed as well. + * The new Printer clones all other configurations from the current + * {@link Printer}. + */ + public Printer includingDefaultValueFields() { + return new Printer(registry, true, preservingProtoFieldNames); + } + + /** + * Creates a new {@link Printer} that is configured to use the original proto + * field names as defined in the .proto file rather than converting them to + * lowerCamelCase. The new Printer clones all other configurations from the + * current {@link Printer}. + */ + public Printer preservingProtoFieldNames() { + return new Printer(registry, includingDefaultValueFields, true); } /** @@ -136,7 +164,8 @@ public class JsonFormat { throws IOException { // TODO(xiaofeng): Investigate the allocation overhead and optimize for // mobile. - new PrinterImpl(registry, output).print(message); + new PrinterImpl(registry, includingDefaultValueFields, preservingProtoFieldNames, output) + .print(message); } /** @@ -298,7 +327,7 @@ public class JsonFormat { private void addFile(FileDescriptor file) { // Skip the file if it's already added. - if (files.contains(file.getName())) { + if (!files.add(file.getFullName())) { return; } for (FileDescriptor dependency : file.getDependencies()) { @@ -397,6 +426,8 @@ public class JsonFormat { */ private static final class PrinterImpl { private final TypeRegistry registry; + private final boolean includingDefaultValueFields; + private final boolean preservingProtoFieldNames; private final TextGenerator generator; // We use Gson to help handle string escapes. private final Gson gson; @@ -405,8 +436,14 @@ public class JsonFormat { private static final Gson DEFAULT_GSON = new Gson(); } - PrinterImpl(TypeRegistry registry, Appendable jsonOutput) { + PrinterImpl( + TypeRegistry registry, + boolean includingDefaultValueFields, + boolean preservingProtoFieldNames, + Appendable jsonOutput) { this.registry = registry; + this.includingDefaultValueFields = includingDefaultValueFields; + this.preservingProtoFieldNames = preservingProtoFieldNames; this.generator = new TextGenerator(jsonOutput); this.gson = GsonHolder.DEFAULT_GSON; } @@ -647,13 +684,23 @@ public class JsonFormat { generator.print("\"@type\": " + gson.toJson(typeUrl)); printedField = true; } - for (Map.Entry field - : message.getAllFields().entrySet()) { - // Skip unknown enum fields. - if (field.getValue() instanceof EnumValueDescriptor - && ((EnumValueDescriptor) field.getValue()).getIndex() == -1) { - continue; + Map fieldsToPrint = null; + if (includingDefaultValueFields) { + fieldsToPrint = new TreeMap(); + for (FieldDescriptor field : message.getDescriptorForType().getFields()) { + if (field.isOptional() + && field.getJavaType() == FieldDescriptor.JavaType.MESSAGE + && !message.hasField(field)) { + // Always skip empty optional message fields. If not we will recurse indefinitely if + // a message has itself as a sub-field. + continue; + } + fieldsToPrint.put(field, message.getField(field)); } + } else { + fieldsToPrint = message.getAllFields(); + } + for (Map.Entry field : fieldsToPrint.entrySet()) { if (printedField) { // Add line-endings for the previous field. generator.print(",\n"); @@ -673,7 +720,11 @@ public class JsonFormat { private void printField(FieldDescriptor field, Object value) throws IOException { - generator.print("\"" + fieldNameToCamelName(field.getName()) + "\": "); + if (preservingProtoFieldNames) { + generator.print("\"" + field.getName() + "\": "); + } else { + generator.print("\"" + field.getJsonName() + "\": "); + } if (field.isMapField()) { printMapFieldValue(field, value); } else if (field.isRepeated()) { @@ -689,11 +740,6 @@ public class JsonFormat { generator.print("["); boolean printedElement = false; for (Object element : (List) value) { - // Skip unknown enum entries. - if (element instanceof EnumValueDescriptor - && ((EnumValueDescriptor) element).getIndex() == -1) { - continue; - } if (printedElement) { generator.print(", "); } else { @@ -720,11 +766,6 @@ public class JsonFormat { Message entry = (Message) element; Object entryKey = entry.getField(keyField); Object entryValue = entry.getField(valueField); - // Skip unknown enum entries. - if (entryValue instanceof EnumValueDescriptor - && ((EnumValueDescriptor) entryValue).getIndex() == -1) { - continue; - } if (printedElement) { generator.print(",\n"); } else { @@ -871,8 +912,13 @@ public class JsonFormat { generator.print("\""); } } else { - generator.print( - "\"" + ((EnumValueDescriptor) value).getName() + "\""); + if (((EnumValueDescriptor) value).getIndex() == -1) { + generator.print( + String.valueOf(((EnumValueDescriptor) value).getNumber())); + } else { + generator.print( + "\"" + ((EnumValueDescriptor) value).getName() + "\""); + } } break; @@ -916,38 +962,6 @@ public class JsonFormat { } return parts[1]; } - - private static String fieldNameToCamelName(String name) { - StringBuilder result = new StringBuilder(name.length()); - boolean isNextUpperCase = false; - for (int i = 0; i < name.length(); i++) { - Character ch = name.charAt(i); - if (Character.isLowerCase(ch)) { - if (isNextUpperCase) { - result.append(Character.toUpperCase(ch)); - } else { - result.append(ch); - } - isNextUpperCase = false; - } else if (Character.isUpperCase(ch)) { - if (i == 0 && !isNextUpperCase) { - // Force first letter to lower-case unless explicitly told to - // capitalize it. - result.append(Character.toLowerCase(ch)); - } else { - // Capital letters after the first are left as-is. - result.append(ch); - } - isNextUpperCase = false; - } else if (Character.isDigit(ch)) { - result.append(ch); - isNextUpperCase = true; - } else { - isNextUpperCase = true; - } - } - return result.toString(); - } private static class ParserImpl { private final TypeRegistry registry; @@ -1085,7 +1099,8 @@ public class JsonFormat { Map fieldNameMap = new HashMap(); for (FieldDescriptor field : descriptor.getFields()) { - fieldNameMap.put(fieldNameToCamelName(field.getName()), field); + fieldNameMap.put(field.getName(), field); + fieldNameMap.put(field.getJsonName(), field); } fieldNameMaps.put(descriptor, fieldNameMap); return fieldNameMap; @@ -1244,7 +1259,25 @@ public class JsonFormat { private void mergeField(FieldDescriptor field, JsonElement json, Message.Builder builder) throws InvalidProtocolBufferException { - if (json instanceof JsonNull) { + if (field.isRepeated()) { + if (builder.getRepeatedFieldCount(field) > 0) { + throw new InvalidProtocolBufferException( + "Field " + field.getFullName() + " has already been set."); + } + } else { + if (builder.hasField(field)) { + throw new InvalidProtocolBufferException( + "Field " + field.getFullName() + " has already been set."); + } + if (field.getContainingOneof() != null + && builder.getOneofFieldDescriptor(field.getContainingOneof()) != null) { + FieldDescriptor other = builder.getOneofFieldDescriptor(field.getContainingOneof()); + throw new InvalidProtocolBufferException( + "Cannot set field " + field.getFullName() + " because another field " + + other.getFullName() + " belonging to the same oneof has already been set "); + } + } + if (field.isRepeated() && json instanceof JsonNull) { // We allow "null" as value for all field types and treat it as if the // field is not present. return; @@ -1282,7 +1315,8 @@ public class JsonFormat { Object value = parseFieldValue( valueField, entry.getValue(), entryBuilder); if (value == null) { - value = getDefaultValue(valueField, entryBuilder); + throw new InvalidProtocolBufferException( + "Map value cannot be null."); } entryBuilder.setField(keyField, key); entryBuilder.setField(valueField, value); @@ -1341,7 +1375,8 @@ public class JsonFormat { for (int i = 0; i < array.size(); ++i) { Object value = parseFieldValue(field, array.get(i), builder); if (value == null) { - value = getDefaultValue(field, builder); + throw new InvalidProtocolBufferException( + "Repeated field elements cannot be null"); } builder.addRepeatedField(field, value); } @@ -1351,6 +1386,15 @@ public class JsonFormat { throws InvalidProtocolBufferException { try { return Integer.parseInt(json.getAsString()); + } catch (Exception e) { + // Fall through. + } + // JSON doesn't distinguish between integer values and floating point values so "1" and + // "1.000" are treated as equal in JSON. For this reason we accept floating point values for + // integer fields as well as long as it actually is an integer (i.e., round(value) == value). + try { + BigDecimal value = new BigDecimal(json.getAsString()); + return value.intValueExact(); } catch (Exception e) { throw new InvalidProtocolBufferException("Not an int32 value: " + json); } @@ -1361,7 +1405,16 @@ public class JsonFormat { try { return Long.parseLong(json.getAsString()); } catch (Exception e) { - throw new InvalidProtocolBufferException("Not an int64 value: " + json); + // Fall through. + } + // JSON doesn't distinguish between integer values and floating point values so "1" and + // "1.000" are treated as equal in JSON. For this reason we accept floating point values for + // integer fields as well as long as it actually is an integer (i.e., round(value) == value). + try { + BigDecimal value = new BigDecimal(json.getAsString()); + return value.longValueExact(); + } catch (Exception e) { + throw new InvalidProtocolBufferException("Not an int32 value: " + json); } } @@ -1376,6 +1429,21 @@ public class JsonFormat { return (int) result; } catch (InvalidProtocolBufferException e) { throw e; + } catch (Exception e) { + // Fall through. + } + // JSON doesn't distinguish between integer values and floating point values so "1" and + // "1.000" are treated as equal in JSON. For this reason we accept floating point values for + // integer fields as well as long as it actually is an integer (i.e., round(value) == value). + try { + BigDecimal decimalValue = new BigDecimal(json.getAsString()); + BigInteger value = decimalValue.toBigIntegerExact(); + if (value.signum() < 0 || value.compareTo(new BigInteger("FFFFFFFF", 16)) > 0) { + throw new InvalidProtocolBufferException("Out of range uint32 value: " + json); + } + return value.intValue(); + } catch (InvalidProtocolBufferException e) { + throw e; } catch (Exception e) { throw new InvalidProtocolBufferException( "Not an uint32 value: " + json); @@ -1388,7 +1456,8 @@ public class JsonFormat { private long parseUint64(JsonElement json) throws InvalidProtocolBufferException { try { - BigInteger value = new BigInteger(json.getAsString()); + BigDecimal decimalValue = new BigDecimal(json.getAsString()); + BigInteger value = decimalValue.toBigIntegerExact(); if (value.compareTo(BigInteger.ZERO) < 0 || value.compareTo(MAX_UINT64) > 0) { throw new InvalidProtocolBufferException( @@ -1488,7 +1557,12 @@ public class JsonFormat { return json.getAsString(); } - private ByteString parseBytes(JsonElement json) { + private ByteString parseBytes(JsonElement json) throws InvalidProtocolBufferException { + String encoded = json.getAsString(); + if (encoded.length() % 4 != 0) { + throw new InvalidProtocolBufferException( + "Bytes field is not encoded in standard BASE64 with paddings: " + encoded); + } return ByteString.copyFrom( BaseEncoding.base64().decode(json.getAsString())); } @@ -1498,9 +1572,25 @@ public class JsonFormat { String value = json.getAsString(); EnumValueDescriptor result = enumDescriptor.findValueByName(value); if (result == null) { - throw new InvalidProtocolBufferException( - "Invalid enum value: " + value + " for enum type: " - + enumDescriptor.getFullName()); + // Try to interpret the value as a number. + try { + int numericValue = parseInt32(json); + if (enumDescriptor.getFile().getSyntax() == FileDescriptor.Syntax.PROTO3) { + result = enumDescriptor.findValueByNumberCreatingIfUnknown(numericValue); + } else { + result = enumDescriptor.findValueByNumber(numericValue); + } + } catch (InvalidProtocolBufferException e) { + // Fall through. This exception is about invalid int32 value we get from parseInt32() but + // that's not the exception we want the user to see. Since result == null, we will throw + // an exception later. + } + + if (result == null) { + throw new InvalidProtocolBufferException( + "Invalid enum value: " + value + " for enum type: " + + enumDescriptor.getFullName()); + } } return result; } diff --git a/java/util/src/main/java/com/google/protobuf/util/TimeUtil.java b/java/util/src/main/java/com/google/protobuf/util/TimeUtil.java index 6e4b7c03..3033182a 100644 --- a/java/util/src/main/java/com/google/protobuf/util/TimeUtil.java +++ b/java/util/src/main/java/com/google/protobuf/util/TimeUtil.java @@ -58,8 +58,12 @@ public class TimeUtil { private static final long MILLIS_PER_SECOND = 1000; private static final long MICROS_PER_SECOND = 1000000; - private static final SimpleDateFormat timestampFormat = - createTimestampFormat(); + private static final ThreadLocal timestampFormat = + new ThreadLocal() { + protected SimpleDateFormat initialValue() { + return createTimestampFormat(); + } + }; private static SimpleDateFormat createTimestampFormat() { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); @@ -96,7 +100,7 @@ public class TimeUtil { throw new IllegalArgumentException("Timestamp is out of range."); } Date date = new Date(timestamp.getSeconds() * MILLIS_PER_SECOND); - result.append(timestampFormat.format(date)); + result.append(timestampFormat.get().format(date)); // Format the nanos part. if (timestamp.getNanos() < 0 || timestamp.getNanos() >= NANOS_PER_SECOND) { throw new IllegalArgumentException("Timestamp has invalid nanos value."); @@ -147,7 +151,7 @@ public class TimeUtil { secondValue = timeValue.substring(0, pointPosition); nanoValue = timeValue.substring(pointPosition + 1); } - Date date = timestampFormat.parse(secondValue); + Date date = timestampFormat.get().parse(secondValue); long seconds = date.getTime() / MILLIS_PER_SECOND; int nanos = nanoValue.isEmpty() ? 0 : parseNanos(nanoValue); // Parse timezone offsets. diff --git a/java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java b/java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java index 67fbe0b1..a312fc33 100644 --- a/java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/FieldMaskUtilTest.java @@ -52,6 +52,21 @@ public class FieldMaskUtilTest extends TestCase { assertFalse(FieldMaskUtil.isValid( NestedTestAllTypes.class, "payload.nonexist")); + assertTrue(FieldMaskUtil.isValid( + NestedTestAllTypes.class, FieldMaskUtil.fromString("payload"))); + assertFalse(FieldMaskUtil.isValid( + NestedTestAllTypes.class, FieldMaskUtil.fromString("nonexist"))); + assertFalse(FieldMaskUtil.isValid( + NestedTestAllTypes.class, FieldMaskUtil.fromString("payload,nonexist"))); + + assertTrue(FieldMaskUtil.isValid(NestedTestAllTypes.getDescriptor(), "payload")); + assertFalse(FieldMaskUtil.isValid(NestedTestAllTypes.getDescriptor(), "nonexist")); + + assertTrue(FieldMaskUtil.isValid( + NestedTestAllTypes.getDescriptor(), FieldMaskUtil.fromString("payload"))); + assertFalse(FieldMaskUtil.isValid( + NestedTestAllTypes.getDescriptor(), FieldMaskUtil.fromString("nonexist"))); + assertTrue(FieldMaskUtil.isValid( NestedTestAllTypes.class, "payload.optional_nested_message.bb")); // Repeated fields cannot have sub-paths. @@ -74,7 +89,7 @@ public class FieldMaskUtilTest extends TestCase { addPaths("bar").addPaths("").build(); assertEquals("foo,bar", FieldMaskUtil.toString(mask)); } - + public void testFromString() throws Exception { FieldMask mask = FieldMaskUtil.fromString(""); assertEquals(0, mask.getPathsCount()); @@ -85,16 +100,16 @@ public class FieldMaskUtilTest extends TestCase { assertEquals(2, mask.getPathsCount()); assertEquals("foo", mask.getPaths(0)); assertEquals("bar.baz", mask.getPaths(1)); - + // Empty field paths are ignore. mask = FieldMaskUtil.fromString(",foo,,bar,"); assertEquals(2, mask.getPathsCount()); assertEquals("foo", mask.getPaths(0)); assertEquals("bar", mask.getPaths(1)); - + // Check whether the field paths are valid if a class parameter is provided. mask = FieldMaskUtil.fromString(NestedTestAllTypes.class, ",payload"); - + try { mask = FieldMaskUtil.fromString( NestedTestAllTypes.class, "payload,nonexist"); @@ -103,6 +118,31 @@ public class FieldMaskUtilTest extends TestCase { // Expected. } } + + public void testFromFieldNumbers() throws Exception { + FieldMask mask = FieldMaskUtil.fromFieldNumbers(TestAllTypes.class); + assertEquals(0, mask.getPathsCount()); + mask = + FieldMaskUtil.fromFieldNumbers( + TestAllTypes.class, TestAllTypes.OPTIONAL_INT32_FIELD_NUMBER); + assertEquals(1, mask.getPathsCount()); + assertEquals("optional_int32", mask.getPaths(0)); + mask = + FieldMaskUtil.fromFieldNumbers( + TestAllTypes.class, + TestAllTypes.OPTIONAL_INT32_FIELD_NUMBER, + TestAllTypes.OPTIONAL_INT64_FIELD_NUMBER); + assertEquals(2, mask.getPathsCount()); + assertEquals("optional_int32", mask.getPaths(0)); + assertEquals("optional_int64", mask.getPaths(1)); + + try { + int invalidFieldNumber = 1000; + mask = FieldMaskUtil.fromFieldNumbers(TestAllTypes.class, invalidFieldNumber); + fail("Exception is expected."); + } catch (IllegalArgumentException expected) { + } + } public void testUnion() throws Exception { // Only test a simple case here and expect diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java index ddf5ad2a..c0eb0330 100644 --- a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java @@ -51,9 +51,11 @@ import com.google.protobuf.util.JsonTestProto.TestAllTypes; import com.google.protobuf.util.JsonTestProto.TestAllTypes.NestedEnum; import com.google.protobuf.util.JsonTestProto.TestAllTypes.NestedMessage; import com.google.protobuf.util.JsonTestProto.TestAny; +import com.google.protobuf.util.JsonTestProto.TestCustomJsonName; import com.google.protobuf.util.JsonTestProto.TestDuration; import com.google.protobuf.util.JsonTestProto.TestFieldMask; import com.google.protobuf.util.JsonTestProto.TestMap; +import com.google.protobuf.util.JsonTestProto.TestOneof; import com.google.protobuf.util.JsonTestProto.TestStruct; import com.google.protobuf.util.JsonTestProto.TestTimestamp; import com.google.protobuf.util.JsonTestProto.TestWrappers; @@ -196,9 +198,6 @@ public class JsonFormatTest extends TestCase { } public void testUnknownEnumValues() throws Exception { - // Unknown enum values will be dropped. - // TODO(xiaofeng): We may want to revisit this (whether we should omit - // unknown enum values). TestAllTypes message = TestAllTypes.newBuilder() .setOptionalNestedEnumValue(12345) .addRepeatedNestedEnumValue(12345) @@ -206,8 +205,10 @@ public class JsonFormatTest extends TestCase { .build(); assertEquals( "{\n" - + " \"repeatedNestedEnum\": [\"FOO\"]\n" + + " \"optionalNestedEnum\": 12345,\n" + + " \"repeatedNestedEnum\": [12345, \"FOO\"]\n" + "}", toJsonString(message)); + assertRoundTripEquals(message); TestMap.Builder mapBuilder = TestMap.newBuilder(); mapBuilder.getMutableInt32ToEnumMapValue().put(1, 0); @@ -216,9 +217,11 @@ public class JsonFormatTest extends TestCase { assertEquals( "{\n" + " \"int32ToEnumMap\": {\n" - + " \"1\": \"FOO\"\n" + + " \"1\": \"FOO\",\n" + + " \"2\": 12345\n" + " }\n" + "}", toJsonString(mapMessage)); + assertRoundTripEquals(mapMessage); } public void testSpecialFloatValues() throws Exception { @@ -263,6 +266,35 @@ public class JsonFormatTest extends TestCase { assertEquals(true, message.getOptionalBool()); } + public void testParserAcceptFloatingPointValueForIntegerField() throws Exception { + // Test that numeric values like "1.000", "1e5" will also be accepted. + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + mergeFromJson( + "{\n" + + " \"repeatedInt32\": [1.000, 1e5, \"1.000\", \"1e5\"],\n" + + " \"repeatedUint32\": [1.000, 1e5, \"1.000\", \"1e5\"],\n" + + " \"repeatedInt64\": [1.000, 1e5, \"1.000\", \"1e5\"],\n" + + " \"repeatedUint64\": [1.000, 1e5, \"1.000\", \"1e5\"]\n" + + "}", builder); + int[] expectedValues = new int[]{1, 100000, 1, 100000}; + assertEquals(4, builder.getRepeatedInt32Count()); + assertEquals(4, builder.getRepeatedUint32Count()); + assertEquals(4, builder.getRepeatedInt64Count()); + assertEquals(4, builder.getRepeatedUint64Count()); + for (int i = 0; i < 4; ++i) { + assertEquals(expectedValues[i], builder.getRepeatedInt32(i)); + assertEquals(expectedValues[i], builder.getRepeatedUint32(i)); + assertEquals(expectedValues[i], builder.getRepeatedInt64(i)); + assertEquals(expectedValues[i], builder.getRepeatedUint64(i)); + } + + // Non-integers will still be rejected. + assertRejects("optionalInt32", "1.5"); + assertRejects("optionalUint32", "1.5"); + assertRejects("optionalInt64", "1.5"); + assertRejects("optionalUint64", "1.5"); + } + private void assertRejects(String name, String value) { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); try { @@ -285,6 +317,7 @@ public class JsonFormatTest extends TestCase { TestAllTypes.Builder builder = TestAllTypes.newBuilder(); // Both numeric form and string form are accepted. mergeFromJson("{\"" + name + "\":" + value + "}", builder); + builder.clear(); mergeFromJson("{\"" + name + "\":\"" + value + "\"}", builder); } @@ -370,84 +403,74 @@ public class JsonFormatTest extends TestCase { TestAllTypes message = builder.build(); assertEquals(TestAllTypes.getDefaultInstance(), message); - // Repeated field elements can also be null. - builder = TestAllTypes.newBuilder(); - mergeFromJson( - "{\n" - + " \"repeatedInt32\": [null, null],\n" - + " \"repeatedInt64\": [null, null],\n" - + " \"repeatedUint32\": [null, null],\n" - + " \"repeatedUint64\": [null, null],\n" - + " \"repeatedSint32\": [null, null],\n" - + " \"repeatedSint64\": [null, null],\n" - + " \"repeatedFixed32\": [null, null],\n" - + " \"repeatedFixed64\": [null, null],\n" - + " \"repeatedSfixed32\": [null, null],\n" - + " \"repeatedSfixed64\": [null, null],\n" - + " \"repeatedFloat\": [null, null],\n" - + " \"repeatedDouble\": [null, null],\n" - + " \"repeatedBool\": [null, null],\n" - + " \"repeatedString\": [null, null],\n" - + " \"repeatedBytes\": [null, null],\n" - + " \"repeatedNestedMessage\": [null, null],\n" - + " \"repeatedNestedEnum\": [null, null]\n" - + "}", builder); - message = builder.build(); - // "null" elements will be parsed to default values. - assertEquals(2, message.getRepeatedInt32Count()); - assertEquals(0, message.getRepeatedInt32(0)); - assertEquals(0, message.getRepeatedInt32(1)); - assertEquals(2, message.getRepeatedInt32Count()); - assertEquals(0, message.getRepeatedInt32(0)); - assertEquals(0, message.getRepeatedInt32(1)); - assertEquals(2, message.getRepeatedInt64Count()); - assertEquals(0, message.getRepeatedInt64(0)); - assertEquals(0, message.getRepeatedInt64(1)); - assertEquals(2, message.getRepeatedUint32Count()); - assertEquals(0, message.getRepeatedUint32(0)); - assertEquals(0, message.getRepeatedUint32(1)); - assertEquals(2, message.getRepeatedUint64Count()); - assertEquals(0, message.getRepeatedUint64(0)); - assertEquals(0, message.getRepeatedUint64(1)); - assertEquals(2, message.getRepeatedSint32Count()); - assertEquals(0, message.getRepeatedSint32(0)); - assertEquals(0, message.getRepeatedSint32(1)); - assertEquals(2, message.getRepeatedSint64Count()); - assertEquals(0, message.getRepeatedSint64(0)); - assertEquals(0, message.getRepeatedSint64(1)); - assertEquals(2, message.getRepeatedFixed32Count()); - assertEquals(0, message.getRepeatedFixed32(0)); - assertEquals(0, message.getRepeatedFixed32(1)); - assertEquals(2, message.getRepeatedFixed64Count()); - assertEquals(0, message.getRepeatedFixed64(0)); - assertEquals(0, message.getRepeatedFixed64(1)); - assertEquals(2, message.getRepeatedSfixed32Count()); - assertEquals(0, message.getRepeatedSfixed32(0)); - assertEquals(0, message.getRepeatedSfixed32(1)); - assertEquals(2, message.getRepeatedSfixed64Count()); - assertEquals(0, message.getRepeatedSfixed64(0)); - assertEquals(0, message.getRepeatedSfixed64(1)); - assertEquals(2, message.getRepeatedFloatCount()); - assertEquals(0f, message.getRepeatedFloat(0)); - assertEquals(0f, message.getRepeatedFloat(1)); - assertEquals(2, message.getRepeatedDoubleCount()); - assertEquals(0.0, message.getRepeatedDouble(0)); - assertEquals(0.0, message.getRepeatedDouble(1)); - assertEquals(2, message.getRepeatedBoolCount()); - assertFalse(message.getRepeatedBool(0)); - assertFalse(message.getRepeatedBool(1)); - assertEquals(2, message.getRepeatedStringCount()); - assertTrue(message.getRepeatedString(0).isEmpty()); - assertTrue(message.getRepeatedString(1).isEmpty()); - assertEquals(2, message.getRepeatedBytesCount()); - assertTrue(message.getRepeatedBytes(0).isEmpty()); - assertTrue(message.getRepeatedBytes(1).isEmpty()); - assertEquals(2, message.getRepeatedNestedMessageCount()); - assertEquals(NestedMessage.getDefaultInstance(), message.getRepeatedNestedMessage(0)); - assertEquals(NestedMessage.getDefaultInstance(), message.getRepeatedNestedMessage(1)); - assertEquals(2, message.getRepeatedNestedEnumCount()); - assertEquals(0, message.getRepeatedNestedEnumValue(0)); - assertEquals(0, message.getRepeatedNestedEnumValue(1)); + // Repeated field elements cannot be null. + try { + builder = TestAllTypes.newBuilder(); + mergeFromJson( + "{\n" + + " \"repeatedInt32\": [null, null],\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } + + try { + builder = TestAllTypes.newBuilder(); + mergeFromJson( + "{\n" + + " \"repeatedNestedMessage\": [null, null],\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } + } + + public void testParserRejectDuplicatedFields() throws Exception { + // TODO(xiaofeng): The parser we are currently using (GSON) will accept and keep the last + // one if multiple entries have the same name. This is not the desired behavior but it can + // only be fixed by using our own parser. Here we only test the cases where the names are + // different but still referring to the same field. + + // Duplicated optional fields. + try { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + mergeFromJson( + "{\n" + + " \"optionalNestedMessage\": {},\n" + + " \"optional_nested_message\": {}\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } + + // Duplicated repeated fields. + try { + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + mergeFromJson( + "{\n" + + " \"repeatedNestedMessage\": [null, null],\n" + + " \"repeated_nested_message\": [null, null]\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } + + // Duplicated oneof fields. + try { + TestOneof.Builder builder = TestOneof.newBuilder(); + mergeFromJson( + "{\n" + + " \"oneofInt32\": 1,\n" + + " \"oneof_int32\": 2\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } } public void testMapFields() throws Exception { @@ -592,18 +615,30 @@ public class JsonFormatTest extends TestCase { assertRoundTripEquals(message); } - public void testMapNullValueIsDefault() throws Exception { - TestMap.Builder builder = TestMap.newBuilder(); - mergeFromJson( - "{\n" - + " \"int32ToInt32Map\": {\"1\": null},\n" - + " \"int32ToMessageMap\": {\"2\": null}\n" - + "}", builder); - TestMap message = builder.build(); - assertTrue(message.getInt32ToInt32Map().containsKey(1)); - assertEquals(0, message.getInt32ToInt32Map().get(1).intValue()); - assertTrue(message.getInt32ToMessageMap().containsKey(2)); - assertEquals(0, message.getInt32ToMessageMap().get(2).getValue()); + public void testMapNullValueIsRejected() throws Exception { + try { + TestMap.Builder builder = TestMap.newBuilder(); + mergeFromJson( + "{\n" + + " \"int32ToInt32Map\": {null: 1},\n" + + " \"int32ToMessageMap\": {null: 2}\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } + + try { + TestMap.Builder builder = TestMap.newBuilder(); + mergeFromJson( + "{\n" + + " \"int32ToInt32Map\": {\"1\": null},\n" + + " \"int32ToMessageMap\": {\"2\": null}\n" + + "}", builder); + fail(); + } catch (InvalidProtocolBufferException e) { + // Exception expected. + } } public void testParserAcceptNonQuotedObjectKey() throws Exception { @@ -743,6 +778,15 @@ public class JsonFormatTest extends TestCase { + " }\n" + "}", toJsonString(message)); assertRoundTripEquals(message); + + builder = TestStruct.newBuilder(); + builder.setValue(Value.newBuilder().setNullValueValue(0).build()); + message = builder.build(); + assertEquals( + "{\n" + + " \"value\": null\n" + + "}", toJsonString(message)); + assertRoundTripEquals(message); } public void testAnyFields() throws Exception { @@ -891,6 +935,15 @@ public class JsonFormatTest extends TestCase { + " }\n" + "}", printer.print(anyMessage)); assertRoundTripEquals(anyMessage, registry); + Value.Builder valueBuilder = Value.newBuilder(); + valueBuilder.setNumberValue(1); + anyMessage = Any.pack(valueBuilder.build()); + assertEquals( + "{\n" + + " \"@type\": \"type.googleapis.com/google.protobuf.Value\",\n" + + " \"value\": 1.0\n" + + "}", printer.print(anyMessage)); + assertRoundTripEquals(anyMessage, registry); } public void testParserMissingTypeUrl() throws Exception { @@ -949,16 +1002,9 @@ public class JsonFormatTest extends TestCase { } public void testParserRejectInvalidBase64() throws Exception { - try { - TestAllTypes.Builder builder = TestAllTypes.newBuilder(); - mergeFromJson( - "{\n" - + " \"optionalBytes\": \"!@#$\"\n" - + "}", builder); - fail("Exception is expected."); - } catch (InvalidProtocolBufferException e) { - // Expected. - } + assertRejects("optionalBytes", "!@#$"); + // We use standard BASE64 with paddings. + assertRejects("optionalBytes", "AQI"); } public void testParserRejectInvalidEnumValue() throws Exception { @@ -973,4 +1019,138 @@ public class JsonFormatTest extends TestCase { // Expected. } } + + public void testCustomJsonName() throws Exception { + TestCustomJsonName message = TestCustomJsonName.newBuilder().setValue(12345).build(); + assertEquals("{\n" + " \"@value\": 12345\n" + "}", JsonFormat.printer().print(message)); + assertRoundTripEquals(message); + } + + public void testIncludingDefaultValueFields() throws Exception { + TestAllTypes message = TestAllTypes.getDefaultInstance(); + assertEquals("{\n}", JsonFormat.printer().print(message)); + assertEquals( + "{\n" + + " \"optionalInt32\": 0,\n" + + " \"optionalInt64\": \"0\",\n" + + " \"optionalUint32\": 0,\n" + + " \"optionalUint64\": \"0\",\n" + + " \"optionalSint32\": 0,\n" + + " \"optionalSint64\": \"0\",\n" + + " \"optionalFixed32\": 0,\n" + + " \"optionalFixed64\": \"0\",\n" + + " \"optionalSfixed32\": 0,\n" + + " \"optionalSfixed64\": \"0\",\n" + + " \"optionalFloat\": 0.0,\n" + + " \"optionalDouble\": 0.0,\n" + + " \"optionalBool\": false,\n" + + " \"optionalString\": \"\",\n" + + " \"optionalBytes\": \"\",\n" + + " \"optionalNestedEnum\": \"FOO\",\n" + + " \"repeatedInt32\": [],\n" + + " \"repeatedInt64\": [],\n" + + " \"repeatedUint32\": [],\n" + + " \"repeatedUint64\": [],\n" + + " \"repeatedSint32\": [],\n" + + " \"repeatedSint64\": [],\n" + + " \"repeatedFixed32\": [],\n" + + " \"repeatedFixed64\": [],\n" + + " \"repeatedSfixed32\": [],\n" + + " \"repeatedSfixed64\": [],\n" + + " \"repeatedFloat\": [],\n" + + " \"repeatedDouble\": [],\n" + + " \"repeatedBool\": [],\n" + + " \"repeatedString\": [],\n" + + " \"repeatedBytes\": [],\n" + + " \"repeatedNestedMessage\": [],\n" + + " \"repeatedNestedEnum\": []\n" + + "}", + JsonFormat.printer().includingDefaultValueFields().print(message)); + + TestMap mapMessage = TestMap.getDefaultInstance(); + assertEquals("{\n}", JsonFormat.printer().print(mapMessage)); + assertEquals( + "{\n" + + " \"int32ToInt32Map\": {\n" + + " },\n" + + " \"int64ToInt32Map\": {\n" + + " },\n" + + " \"uint32ToInt32Map\": {\n" + + " },\n" + + " \"uint64ToInt32Map\": {\n" + + " },\n" + + " \"sint32ToInt32Map\": {\n" + + " },\n" + + " \"sint64ToInt32Map\": {\n" + + " },\n" + + " \"fixed32ToInt32Map\": {\n" + + " },\n" + + " \"fixed64ToInt32Map\": {\n" + + " },\n" + + " \"sfixed32ToInt32Map\": {\n" + + " },\n" + + " \"sfixed64ToInt32Map\": {\n" + + " },\n" + + " \"boolToInt32Map\": {\n" + + " },\n" + + " \"stringToInt32Map\": {\n" + + " },\n" + + " \"int32ToInt64Map\": {\n" + + " },\n" + + " \"int32ToUint32Map\": {\n" + + " },\n" + + " \"int32ToUint64Map\": {\n" + + " },\n" + + " \"int32ToSint32Map\": {\n" + + " },\n" + + " \"int32ToSint64Map\": {\n" + + " },\n" + + " \"int32ToFixed32Map\": {\n" + + " },\n" + + " \"int32ToFixed64Map\": {\n" + + " },\n" + + " \"int32ToSfixed32Map\": {\n" + + " },\n" + + " \"int32ToSfixed64Map\": {\n" + + " },\n" + + " \"int32ToFloatMap\": {\n" + + " },\n" + + " \"int32ToDoubleMap\": {\n" + + " },\n" + + " \"int32ToBoolMap\": {\n" + + " },\n" + + " \"int32ToStringMap\": {\n" + + " },\n" + + " \"int32ToBytesMap\": {\n" + + " },\n" + + " \"int32ToMessageMap\": {\n" + + " },\n" + + " \"int32ToEnumMap\": {\n" + + " }\n" + + "}", + JsonFormat.printer().includingDefaultValueFields().print(mapMessage)); + } + + public void testPreservingProtoFieldNames() throws Exception { + TestAllTypes message = TestAllTypes.newBuilder().setOptionalInt32(12345).build(); + assertEquals("{\n" + " \"optionalInt32\": 12345\n" + "}", JsonFormat.printer().print(message)); + assertEquals( + "{\n" + " \"optional_int32\": 12345\n" + "}", + JsonFormat.printer().preservingProtoFieldNames().print(message)); + + // The json_name field option is ignored when configured to use original proto field names. + TestCustomJsonName messageWithCustomJsonName = + TestCustomJsonName.newBuilder().setValue(12345).build(); + assertEquals( + "{\n" + " \"value\": 12345\n" + "}", + JsonFormat.printer().preservingProtoFieldNames().print(messageWithCustomJsonName)); + + // Parsers accept both original proto field names and lowerCamelCase names. + TestAllTypes.Builder builder = TestAllTypes.newBuilder(); + JsonFormat.parser().merge("{\"optionalInt32\": 12345}", builder); + assertEquals(12345, builder.getOptionalInt32()); + builder.clear(); + JsonFormat.parser().merge("{\"optional_int32\": 54321}", builder); + assertEquals(54321, builder.getOptionalInt32()); + } } diff --git a/java/util/src/test/java/com/google/protobuf/util/TimeUtilTest.java b/java/util/src/test/java/com/google/protobuf/util/TimeUtilTest.java index fe5617e1..4c31b2b3 100644 --- a/java/util/src/test/java/com/google/protobuf/util/TimeUtilTest.java +++ b/java/util/src/test/java/com/google/protobuf/util/TimeUtilTest.java @@ -38,6 +38,8 @@ import junit.framework.TestCase; import org.junit.Assert; import java.text.ParseException; +import java.util.ArrayList; +import java.util.List; /** Unit tests for {@link TimeUtil}. */ public class TimeUtilTest extends TestCase { @@ -76,6 +78,71 @@ public class TimeUtilTest extends TestCase { assertEquals("1970-01-01T08:00:00.010Z", TimeUtil.toString(value)); } + private volatile boolean stopParsingThreads = false; + private volatile String errorMessage = ""; + + private class ParseTimestampThread extends Thread { + private final String[] strings; + private final Timestamp[] values; + public ParseTimestampThread(String[] strings, Timestamp[] values) { + this.strings = strings; + this.values = values; + } + + @Override + public void run() { + int index = 0; + while (!stopParsingThreads) { + Timestamp result; + try { + result = TimeUtil.parseTimestamp(strings[index]); + } catch (ParseException e) { + errorMessage = "Failed to parse timestamp: " + strings[index]; + break; + } + if (result.getSeconds() != values[index].getSeconds() + || result.getNanos() != values[index].getNanos()) { + errorMessage = "Actual result: " + result.toString() + ", expected: " + + values[index].toString(); + break; + } + index = (index + 1) % strings.length; + } + } + } + + public void testTimestampConcurrentParsing() throws Exception { + String[] timestampStrings = new String[]{ + "0001-01-01T00:00:00Z", + "9999-12-31T23:59:59.999999999Z", + "1970-01-01T00:00:00Z", + "1969-12-31T23:59:59.999Z", + }; + Timestamp[] timestampValues = new Timestamp[timestampStrings.length]; + for (int i = 0; i < timestampStrings.length; i++) { + timestampValues[i] = TimeUtil.parseTimestamp(timestampStrings[i]); + } + + final int THREAD_COUNT = 16; + final int RUNNING_TIME = 5000; // in milliseconds. + final List threads = new ArrayList(); + + stopParsingThreads = false; + errorMessage = ""; + for (int i = 0; i < THREAD_COUNT; i++) { + Thread thread = new ParseTimestampThread( + timestampStrings, timestampValues); + thread.start(); + threads.add(thread); + } + Thread.sleep(RUNNING_TIME); + stopParsingThreads = true; + for (Thread thread : threads) { + thread.join(); + } + Assert.assertEquals("", errorMessage); + } + public void testTimetampInvalidFormat() throws Exception { try { // Value too small. diff --git a/java/util/src/test/java/com/google/protobuf/util/json_test.proto b/java/util/src/test/java/com/google/protobuf/util/json_test.proto index b2753af6..023ec2ca 100644 --- a/java/util/src/test/java/com/google/protobuf/util/json_test.proto +++ b/java/util/src/test/java/com/google/protobuf/util/json_test.proto @@ -65,7 +65,7 @@ message TestAllTypes { float optional_float = 11; double optional_double = 12; bool optional_bool = 13; - string optional_string = 14; + string optional_string = 14 [enforce_utf8 = false]; bytes optional_bytes = 15; NestedMessage optional_nested_message = 18; NestedEnum optional_nested_enum = 21; @@ -84,12 +84,19 @@ message TestAllTypes { repeated float repeated_float = 41; repeated double repeated_double = 42; repeated bool repeated_bool = 43; - repeated string repeated_string = 44; + repeated string repeated_string = 44 [enforce_utf8 = false]; repeated bytes repeated_bytes = 45; repeated NestedMessage repeated_nested_message = 48; repeated NestedEnum repeated_nested_enum = 51; } +message TestOneof { + oneof oneof_field { + int32 oneof_int32 = 1; + TestAllTypes.NestedMessage oneof_nested_message = 2; + } +} + message TestMap { // Instead of testing all combinations (too many), we only make sure all // valid types have been used at least in one field as key and in one @@ -105,7 +112,7 @@ message TestMap { map sfixed32_to_int32_map = 9; map sfixed64_to_int32_map = 10; map bool_to_int32_map = 11; - map string_to_int32_map = 12; + map string_to_int32_map = 12 [enforce_utf8 = false]; map int32_to_int64_map = 101; map int32_to_uint32_map = 102; @@ -119,7 +126,7 @@ message TestMap { map int32_to_float_map = 110; map int32_to_double_map = 111; map int32_to_bool_map = 112; - map int32_to_string_map = 113; + map int32_to_string_map = 113 [enforce_utf8 = false]; map int32_to_bytes_map = 114; map int32_to_message_map = 115; map int32_to_enum_map = 116; @@ -151,8 +158,13 @@ message TestFieldMask { message TestStruct { google.protobuf.Struct struct_value = 1; + google.protobuf.Value value = 2; } message TestAny { google.protobuf.Any any_value = 1; } + +message TestCustomJsonName { + int32 value = 1 [json_name = "@value"]; +} diff --git a/js/binary/arith.js b/js/binary/arith.js new file mode 100644 index 00000000..70257de7 --- /dev/null +++ b/js/binary/arith.js @@ -0,0 +1,413 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview This file contains helper code used by jspb.utils to + * handle 64-bit integer conversion to/from strings. + * + * @author cfallin@google.com (Chris Fallin) + * + * TODO(haberman): move this to javascript/closure/math? + */ + +goog.provide('jspb.arith.Int64'); +goog.provide('jspb.arith.UInt64'); + +/** + * UInt64 implements some 64-bit arithmetic routines necessary for properly + * handling 64-bit integer fields. It implements lossless integer arithmetic on + * top of JavaScript's number type, which has only 53 bits of precision, by + * representing 64-bit integers as two 32-bit halves. + * + * @param {number} lo The low 32 bits. + * @param {number} hi The high 32 bits. + * @constructor + */ +jspb.arith.UInt64 = function(lo, hi) { + /** + * The low 32 bits. + * @public {number} + */ + this.lo = lo; + /** + * The high 32 bits. + * @public {number} + */ + this.hi = hi; +}; + + +/** + * Compare two 64-bit numbers. Returns -1 if the first is + * less, +1 if the first is greater, or 0 if both are equal. + * @param {!jspb.arith.UInt64} other + * @return {number} + */ +jspb.arith.UInt64.prototype.cmp = function(other) { + if (this.hi < other.hi || (this.hi == other.hi && this.lo < other.lo)) { + return -1; + } else if (this.hi == other.hi && this.lo == other.lo) { + return 0; + } else { + return 1; + } +}; + + +/** + * Right-shift this number by one bit. + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.prototype.rightShift = function() { + var hi = this.hi >>> 1; + var lo = (this.lo >>> 1) | ((this.hi & 1) << 31); + return new jspb.arith.UInt64(lo >>> 0, hi >>> 0); +}; + + +/** + * Left-shift this number by one bit. + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.prototype.leftShift = function() { + var lo = this.lo << 1; + var hi = (this.hi << 1) | (this.lo >>> 31); + return new jspb.arith.UInt64(lo >>> 0, hi >>> 0); +}; + + +/** + * Test the MSB. + * @return {boolean} + */ +jspb.arith.UInt64.prototype.msb = function() { + return !!(this.hi & 0x80000000); +}; + + +/** + * Test the LSB. + * @return {boolean} + */ +jspb.arith.UInt64.prototype.lsb = function() { + return !!(this.lo & 1); +}; + + +/** + * Test whether this number is zero. + * @return {boolean} + */ +jspb.arith.UInt64.prototype.zero = function() { + return this.lo == 0 && this.hi == 0; +}; + + +/** + * Add two 64-bit numbers to produce a 64-bit number. + * @param {!jspb.arith.UInt64} other + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.prototype.add = function(other) { + var lo = ((this.lo + other.lo) & 0xffffffff) >>> 0; + var hi = + (((this.hi + other.hi) & 0xffffffff) >>> 0) + + (((this.lo + other.lo) >= 0x100000000) ? 1 : 0); + return new jspb.arith.UInt64(lo >>> 0, hi >>> 0); +}; + + +/** + * Subtract two 64-bit numbers to produce a 64-bit number. + * @param {!jspb.arith.UInt64} other + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.prototype.sub = function(other) { + var lo = ((this.lo - other.lo) & 0xffffffff) >>> 0; + var hi = + (((this.hi - other.hi) & 0xffffffff) >>> 0) - + (((this.lo - other.lo) < 0) ? 1 : 0); + return new jspb.arith.UInt64(lo >>> 0, hi >>> 0); +}; + + +/** + * Multiply two 32-bit numbers to produce a 64-bit number. + * @param {number} a The first integer: must be in [0, 2^32-1). + * @param {number} b The second integer: must be in [0, 2^32-1). + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.mul32x32 = function(a, b) { + // Directly multiplying two 32-bit numbers may produce up to 64 bits of + // precision, thus losing precision because of the 53-bit mantissa of + // JavaScript numbers. So we multiply with 16-bit digits (radix 65536) + // instead. + var aLow = (a & 0xffff); + var aHigh = (a >>> 16); + var bLow = (b & 0xffff); + var bHigh = (b >>> 16); + var productLow = + // 32-bit result, result bits 0-31, take all 32 bits + (aLow * bLow) + + // 32-bit result, result bits 16-47, take bottom 16 as our top 16 + ((aLow * bHigh) & 0xffff) * 0x10000 + + // 32-bit result, result bits 16-47, take bottom 16 as our top 16 + ((aHigh * bLow) & 0xffff) * 0x10000; + var productHigh = + // 32-bit result, result bits 32-63, take all 32 bits + (aHigh * bHigh) + + // 32-bit result, result bits 16-47, take top 16 as our bottom 16 + ((aLow * bHigh) >>> 16) + + // 32-bit result, result bits 16-47, take top 16 as our bottom 16 + ((aHigh * bLow) >>> 16); + + // Carry. Note that we actually have up to *two* carries due to addition of + // three terms. + while (productLow >= 0x100000000) { + productLow -= 0x100000000; + productHigh += 1; + } + + return new jspb.arith.UInt64(productLow >>> 0, productHigh >>> 0); +}; + + +/** + * Multiply this number by a 32-bit number, producing a 96-bit number, then + * truncate the top 32 bits. + * @param {number} a The multiplier. + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.prototype.mul = function(a) { + // Produce two parts: at bits 0-63, and 32-95. + var lo = jspb.arith.UInt64.mul32x32(this.lo, a); + var hi = jspb.arith.UInt64.mul32x32(this.hi, a); + // Left-shift hi by 32 bits, truncating its top bits. The parts will then be + // aligned for addition. + hi.hi = hi.lo; + hi.lo = 0; + return lo.add(hi); +}; + + +/** + * Divide a 64-bit number by a 32-bit number to produce a + * 64-bit quotient and a 32-bit remainder. + * @param {number} _divisor + * @return {Array.} array of [quotient, remainder], + * unless divisor is 0, in which case an empty array is returned. + */ +jspb.arith.UInt64.prototype.div = function(_divisor) { + if (_divisor == 0) { + return []; + } + + // We perform long division using a radix-2 algorithm, for simplicity (i.e., + // one bit at a time). TODO: optimize to a radix-2^32 algorithm, taking care + // to get the variable shifts right. + var quotient = new jspb.arith.UInt64(0, 0); + var remainder = new jspb.arith.UInt64(this.lo, this.hi); + var divisor = new jspb.arith.UInt64(_divisor, 0); + var unit = new jspb.arith.UInt64(1, 0); + + // Left-shift the divisor and unit until the high bit of divisor is set. + while (!divisor.msb()) { + divisor = divisor.leftShift(); + unit = unit.leftShift(); + } + + // Perform long division one bit at a time. + while (!unit.zero()) { + // If divisor < remainder, add unit to quotient and subtract divisor from + // remainder. + if (divisor.cmp(remainder) <= 0) { + quotient = quotient.add(unit); + remainder = remainder.sub(divisor); + } + // Right-shift the divisor and unit. + divisor = divisor.rightShift(); + unit = unit.rightShift(); + } + + return [quotient, remainder]; +}; + + +/** + * Convert a 64-bit number to a string. + * @return {string} + * @override + */ +jspb.arith.UInt64.prototype.toString = function() { + var result = ''; + var num = this; + while (!num.zero()) { + var divResult = num.div(10); + var quotient = divResult[0], remainder = divResult[1]; + result = remainder.lo + result; + num = quotient; + } + if (result == '') { + result = '0'; + } + return result; +}; + + +/** + * Parse a string into a 64-bit number. Returns `null` on a parse error. + * @param {string} s + * @return {?jspb.arith.UInt64} + */ +jspb.arith.UInt64.fromString = function(s) { + var result = new jspb.arith.UInt64(0, 0); + // optimization: reuse this instance for each digit. + var digit64 = new jspb.arith.UInt64(0, 0); + for (var i = 0; i < s.length; i++) { + if (s[i] < '0' || s[i] > '9') { + return null; + } + var digit = parseInt(s[i], 10); + digit64.lo = digit; + result = result.mul(10).add(digit64); + } + return result; +}; + + +/** + * Make a copy of the uint64. + * @return {!jspb.arith.UInt64} + */ +jspb.arith.UInt64.prototype.clone = function() { + return new jspb.arith.UInt64(this.lo, this.hi); +}; + + +/** + * Int64 is like UInt64, but modifies string conversions to interpret the stored + * 64-bit value as a twos-complement-signed integer. It does *not* support the + * full range of operations that UInt64 does: only add, subtract, and string + * conversions. + * + * N.B. that multiply and divide routines are *NOT* supported. They will throw + * exceptions. (They are not necessary to implement string conversions, which + * are the only operations we really need in jspb.) + * + * @param {number} lo The low 32 bits. + * @param {number} hi The high 32 bits. + * @constructor + */ +jspb.arith.Int64 = function(lo, hi) { + /** + * The low 32 bits. + * @public {number} + */ + this.lo = lo; + /** + * The high 32 bits. + * @public {number} + */ + this.hi = hi; +}; + + +/** + * Add two 64-bit numbers to produce a 64-bit number. + * @param {!jspb.arith.Int64} other + * @return {!jspb.arith.Int64} + */ +jspb.arith.Int64.prototype.add = function(other) { + var lo = ((this.lo + other.lo) & 0xffffffff) >>> 0; + var hi = + (((this.hi + other.hi) & 0xffffffff) >>> 0) + + (((this.lo + other.lo) >= 0x100000000) ? 1 : 0); + return new jspb.arith.Int64(lo >>> 0, hi >>> 0); +}; + + +/** + * Subtract two 64-bit numbers to produce a 64-bit number. + * @param {!jspb.arith.Int64} other + * @return {!jspb.arith.Int64} + */ +jspb.arith.Int64.prototype.sub = function(other) { + var lo = ((this.lo - other.lo) & 0xffffffff) >>> 0; + var hi = + (((this.hi - other.hi) & 0xffffffff) >>> 0) - + (((this.lo - other.lo) < 0) ? 1 : 0); + return new jspb.arith.Int64(lo >>> 0, hi >>> 0); +}; + + +/** + * Make a copy of the int64. + * @return {!jspb.arith.Int64} + */ +jspb.arith.Int64.prototype.clone = function() { + return new jspb.arith.Int64(this.lo, this.hi); +}; + + +/** + * Convert a 64-bit number to a string. + * @return {string} + * @override + */ +jspb.arith.Int64.prototype.toString = function() { + // If the number is negative, find its twos-complement inverse. + var sign = (this.hi & 0x80000000) != 0; + var num = new jspb.arith.UInt64(this.lo, this.hi); + if (sign) { + num = new jspb.arith.UInt64(0, 0).sub(num); + } + return (sign ? '-' : '') + num.toString(); +}; + + +/** + * Parse a string into a 64-bit number. Returns `null` on a parse error. + * @param {string} s + * @return {?jspb.arith.Int64} + */ +jspb.arith.Int64.fromString = function(s) { + var hasNegative = (s.length > 0 && s[0] == '-'); + if (hasNegative) { + s = s.substring(1); + } + var num = jspb.arith.UInt64.fromString(s); + if (num === null) { + return null; + } + if (hasNegative) { + num = new jspb.arith.UInt64(0, 0).sub(num); + } + return new jspb.arith.Int64(num.lo, num.hi); +}; diff --git a/js/binary/arith_test.js b/js/binary/arith_test.js new file mode 100644 index 00000000..89796bf7 --- /dev/null +++ b/js/binary/arith_test.js @@ -0,0 +1,355 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Test cases for Int64-manipulation functions. + * + * Test suite is written using Jasmine -- see http://jasmine.github.io/ + * + * @author cfallin@google.com (Chris Fallin) + */ + +goog.require('goog.testing.asserts'); +goog.require('jspb.arith.Int64'); +goog.require('jspb.arith.UInt64'); + + +describe('binaryArithTest', function() { + /** + * Tests comparison operations. + */ + it('testCompare', function() { + var a = new jspb.arith.UInt64(1234, 5678); + var b = new jspb.arith.UInt64(1234, 5678); + assertEquals(a.cmp(b), 0); + assertEquals(b.cmp(a), 0); + b.lo -= 1; + assertEquals(a.cmp(b), 1); + assertEquals(b.cmp(a), -1); + b.lo += 2; + assertEquals(a.cmp(b), -1); + assertEquals(b.cmp(a), 1); + b.lo = a.lo; + b.hi = a.hi - 1; + assertEquals(a.cmp(b), 1); + assertEquals(b.cmp(a), -1); + + assertEquals(a.zero(), false); + assertEquals(a.msb(), false); + assertEquals(a.lsb(), false); + a.hi = 0; + a.lo = 0; + assertEquals(a.zero(), true); + a.hi = 0x80000000; + assertEquals(a.zero(), false); + assertEquals(a.msb(), true); + a.lo = 0x00000001; + assertEquals(a.lsb(), true); + }); + + + /** + * Tests shifts. + */ + it('testShifts', function() { + var a = new jspb.arith.UInt64(1, 0); + assertEquals(a.lo, 1); + assertEquals(a.hi, 0); + var orig = a; + a = a.leftShift(); + assertEquals(orig.lo, 1); // original unmodified. + assertEquals(orig.hi, 0); + assertEquals(a.lo, 2); + assertEquals(a.hi, 0); + a = a.leftShift(); + assertEquals(a.lo, 4); + assertEquals(a.hi, 0); + for (var i = 0; i < 29; i++) { + a = a.leftShift(); + } + assertEquals(a.lo, 0x80000000); + assertEquals(a.hi, 0); + a = a.leftShift(); + assertEquals(a.lo, 0); + assertEquals(a.hi, 1); + a = a.leftShift(); + assertEquals(a.lo, 0); + assertEquals(a.hi, 2); + a = a.rightShift(); + a = a.rightShift(); + assertEquals(a.lo, 0x80000000); + assertEquals(a.hi, 0); + a = a.rightShift(); + assertEquals(a.lo, 0x40000000); + assertEquals(a.hi, 0); + }); + + + /** + * Tests additions. + */ + it('testAdd', function() { + var a = new jspb.arith.UInt64(/* lo = */ 0x89abcdef, + /* hi = */ 0x01234567); + var b = new jspb.arith.UInt64(/* lo = */ 0xff52ab91, + /* hi = */ 0x92fa2123); + // Addition with carry. + var c = a.add(b); + assertEquals(a.lo, 0x89abcdef); // originals unmodified. + assertEquals(a.hi, 0x01234567); + assertEquals(b.lo, 0xff52ab91); + assertEquals(b.hi, 0x92fa2123); + assertEquals(c.lo, 0x88fe7980); + assertEquals(c.hi, 0x941d668b); + + // Simple addition without carry. + a.lo = 2; + a.hi = 0; + b.lo = 3; + b.hi = 0; + c = a.add(b); + assertEquals(c.lo, 5); + assertEquals(c.hi, 0); + }); + + + /** + * Test subtractions. + */ + it('testSub', function() { + var kLength = 10; + var hiValues = [0x1682ef32, + 0x583902f7, + 0xb62f5955, + 0x6ea99bbf, + 0x25a39c20, + 0x0700a08b, + 0x00f7304d, + 0x91a5b5af, + 0x89077fd2, + 0xe09e347c]; + var loValues = [0xe1538b18, + 0xbeacd556, + 0x74100758, + 0x96e3cb26, + 0x56c37c3f, + 0xe00b3f7d, + 0x859f25d7, + 0xc2ee614a, + 0xe1d21cd7, + 0x30aae6a4]; + for (var i = 0; i < kLength; i++) { + for (var j = 0; j < kLength; j++) { + var a = new jspb.arith.UInt64(loValues[i], hiValues[j]); + var b = new jspb.arith.UInt64(loValues[j], hiValues[i]); + var c = a.add(b).sub(b); + assertEquals(c.hi, a.hi); + assertEquals(c.lo, a.lo); + } + } + }); + + + /** + * Tests 32-by-32 multiplication. + */ + it('testMul32x32', function() { + var testData = [ + // a b low(a*b) high(a*b) + [0xc0abe2f8, 0x1607898a, 0x5de711b0, 0x109471b8], + [0x915eb3cb, 0x4fb66d0e, 0xbd0d441a, 0x2d43d0bc], + [0xfe4efe70, 0x80b48c37, 0xbcddea10, 0x7fdada0c], + [0xe222fd4a, 0xe43d524a, 0xd5e0eb64, 0xc99d549c], + [0xd171f469, 0xb94ebd01, 0x4be17969, 0x979bc4fa], + [0x829cc1df, 0xe2598b38, 0xf4157dc8, 0x737c12ad], + [0xf10c3767, 0x8382881e, 0x942b3612, 0x7bd428b8], + [0xb0f6dd24, 0x232597e1, 0x079c98a4, 0x184bbce7], + [0xfcdb05a7, 0x902f55bc, 0x636199a4, 0x8e69f412], + [0x0dd0bfa9, 0x916e27b1, 0x6e2542d9, 0x07d92e65] + ]; + + for (var i = 0; i < testData.length; i++) { + var a = testData[i][0] >>> 0; + var b = testData[i][1] >>> 0; + var cLow = testData[i][2] >>> 0; + var cHigh = testData[i][3] >>> 0; + var c = jspb.arith.UInt64.mul32x32(a, b); + assertEquals(c.lo, cLow); + assertEquals(c.hi, cHigh); + } + }); + + + /** + * Tests 64-by-32 multiplication. + */ + it('testMul', function() { + // 64x32 bits produces 96 bits of product. The multiplication function under + // test truncates the top 32 bits, so we compare against a 64-bit expected + // product. + var testData = [ + // low(a) high(a) low(a*b) high(a*b) + [0xec10955b, 0x360eb168, 0x4b7f3f5b, 0xbfcb7c59, 0x9517da5f], + [0x42b000fc, 0x9d101642, 0x6fa1ab72, 0x2584c438, 0x6a9e6d2b], + [0xf42d4fb4, 0xae366403, 0xa65a1000, 0x92434000, 0x1ff978df], + [0x17e2f56b, 0x25487693, 0xf13f98c7, 0x73794e2d, 0xa96b0c6a], + [0x492f241f, 0x76c0eb67, 0x7377ac44, 0xd4336c3c, 0xfc4b1ebe], + [0xd6b92321, 0xe184fa48, 0xd6e76904, 0x93141584, 0xcbf44da1], + [0x4bf007ea, 0x968c0a9e, 0xf5e4026a, 0x4fdb1ae4, 0x61b9fb7d], + [0x10a83be7, 0x2d685ba6, 0xc9e5fb7f, 0x2ad43499, 0x3742473d], + [0x2f261829, 0x1aca681a, 0x3d3494e3, 0x8213205b, 0x283719f8], + [0xe4f2ce21, 0x2e74b7bd, 0xd801b38b, 0xbc17feeb, 0xc6c44e0f] + ]; + + for (var i = 0; i < testData.length; i++) { + var a = new jspb.arith.UInt64(testData[i][0], testData[i][1]); + var prod = a.mul(testData[i][2]); + assertEquals(prod.lo, testData[i][3]); + assertEquals(prod.hi, testData[i][4]); + } + }); + + + /** + * Tests 64-div-by-32 division. + */ + it('testDiv', function() { + // Compute a/b, yielding quot = a/b and rem = a%b. + var testData = [ + // --- divisors in (0, 2^32-1) to test full divisor range + // low(a) high(a) b low(quot) high(quot) rem + [0x712443f1, 0xe85cefcc, 0xc1a7050b, 0x332c79ad, 0x00000001, 0x92ffa882], + [0x11912915, 0xb2699eb5, 0x30467cbe, 0xb21b4be4, 0x00000003, 0x283465dd], + [0x0d917982, 0x201f2a6e, 0x3f35bf03, 0x8217c8e4, 0x00000000, 0x153402d6], + [0xa072c108, 0x74020c96, 0xc60568fd, 0x95f9613e, 0x00000000, 0x3f4676c2], + [0xd845d5d8, 0xcdd235c4, 0x20426475, 0x6154e78b, 0x00000006, 0x202fb751], + [0xa4dbf71f, 0x9e90465e, 0xf08e022f, 0xa8be947f, 0x00000000, 0xbe43b5ce], + [0x3dbe627f, 0xa791f4b9, 0x28a5bd89, 0x1f5dfe93, 0x00000004, 0x02bf9ed4], + [0x5c1c53ee, 0xccf5102e, 0x198576e7, 0x07e3ae31, 0x00000008, 0x02ea8fb7], + [0xfef1e581, 0x04714067, 0xca6540c1, 0x059e73ec, 0x00000000, 0x31658095], + [0x1e2dd90c, 0x13dd6667, 0x8b2184c3, 0x248d1a42, 0x00000000, 0x4ca6d0c6], + // --- divisors in (0, 2^16-1) to test larger quotient high-words + // low(a) high(a) b low(quot) high(quot) rem + [0x86722b47, 0x2cd57c9a, 0x00003123, 0x2ae41b7a, 0x0000e995, 0x00000f99], + [0x1dd7884c, 0xf5e839bc, 0x00009eeb, 0x5c886242, 0x00018c21, 0x000099b6], + [0x5c53d625, 0x899fc7e5, 0x000087d7, 0xd625007a, 0x0001035c, 0x000019af], + [0x6932d932, 0x9d0a5488, 0x000051fb, 0x9d976143, 0x0001ea63, 0x00004981], + [0x4d18bb85, 0x0c92fb31, 0x00001d9f, 0x03265ab4, 0x00006cac, 0x000001b9], + [0xbe756768, 0xdea67ccb, 0x00008a03, 0x58add442, 0x00019cff, 0x000056a2], + [0xe2466f9a, 0x2521f114, 0x0000c350, 0xa0c0860d, 0x000030ab, 0x0000a48a], + [0xf00ddad1, 0xe2f5446a, 0x00002cfc, 0x762697a6, 0x00050b96, 0x00000b69], + [0xa879152a, 0x0a70e0a5, 0x00007cdf, 0xb44151b3, 0x00001567, 0x0000363d], + [0x7179a74c, 0x46083fff, 0x0000253c, 0x4d39ba6e, 0x0001e17f, 0x00000f84] + ]; + + for (var i = 0; i < testData.length; i++) { + var a = new jspb.arith.UInt64(testData[i][0], testData[i][1]); + var result = a.div(testData[i][2]); + var quotient = result[0]; + var remainder = result[1]; + assertEquals(quotient.lo, testData[i][3]); + assertEquals(quotient.hi, testData[i][4]); + assertEquals(remainder.lo, testData[i][5]); + } + }); + + + /** + * Tests .toString() and .fromString(). + */ + it('testStrings', function() { + var testData = [ + [0x5e84c935, 0xcae33d0e, '14619595947299359029'], + [0x62b3b8b8, 0x93480544, '10612738313170434232'], + [0x319bfb13, 0xc01c4172, '13843011313344445203'], + [0x5b8a65fb, 0xa5885b31, '11927883880638080507'], + [0x6bdb80f1, 0xb0d1b16b, '12741159895737008369'], + [0x4b82b442, 0x2e0d8c97, '3318463081876730946'], + [0x780d5208, 0x7d76752c, '9040542135845999112'], + [0x2e46800f, 0x0993778d, '690026616168284175'], + [0xf00a7e32, 0xcd8e3931, '14811839111111540274'], + [0x1baeccd6, 0x923048c4, '10533999535534820566'], + [0x03669d29, 0xbff3ab72, '13831587386756603177'], + [0x2526073e, 0x01affc81, '121593346566522686'], + [0xc24244e0, 0xd7f40d0e, '15561076969511732448'], + [0xc56a341e, 0xa68b66a7, '12000798502816461854'], + [0x8738d64d, 0xbfe78604, '13828168534871037517'], + [0x5baff03b, 0xd7572aea, '15516918227177304123'], + [0x4a843d8a, 0x864e132b, '9677693725920476554'], + [0x25b4e94d, 0x22b54dc6, '2500990681505655117'], + [0x6bbe664b, 0x55a5cc0e, '6171563226690381387'], + [0xee916c81, 0xb00aabb3, '12685140089732426881'] + ]; + + for (var i = 0; i < testData.length; i++) { + var a = new jspb.arith.UInt64(testData[i][0], testData[i][1]); + var roundtrip = jspb.arith.UInt64.fromString(a.toString()); + assertEquals(roundtrip.lo, a.lo); + assertEquals(roundtrip.hi, a.hi); + assertEquals(a.toString(), testData[i][2]); + } + }); + + + /** + * Tests signed Int64s. These are built on UInt64s, so we only need to test + * the explicit overrides: .toString() and .fromString(). + */ + it('testSignedInt64', function() { + var testStrings = [ + '-7847499644178593666', + '3771946501229139523', + '2872856549054995060', + '-5780049594274350904', + '3383785956695105201', + '2973055184857072610', + '-3879428459215627206', + '4589812431064156631', + '8484075557333689940', + '1075325817098092407', + '-4346697501012292314', + '2488620459718316637', + '6112655187423520672', + '-3655278273928612104', + '3439154019435803196', + '1004112478843763757', + '-6587790776614368413', + '664320065099714586', + '4760412909973292912', + '-7911903989602274672' + ]; + + for (var i = 0; i < testStrings.length; i++) { + var roundtrip = + jspb.arith.Int64.fromString(testStrings[i]).toString(); + assertEquals(roundtrip, testStrings[i]); + } + }); +}); diff --git a/js/binary/constants.js b/js/binary/constants.js new file mode 100644 index 00000000..a976e0b6 --- /dev/null +++ b/js/binary/constants.js @@ -0,0 +1,320 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview This file contains constants and typedefs used by + * jspb.BinaryReader and BinaryWriter. + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.provide('jspb.AnyFieldType'); +goog.provide('jspb.BinaryConstants'); +goog.provide('jspb.BinaryMessage'); +goog.provide('jspb.BuilderFunction'); +goog.provide('jspb.ByteSource'); +goog.provide('jspb.ClonerFunction'); +goog.provide('jspb.ConstBinaryMessage'); +goog.provide('jspb.ReaderFunction'); +goog.provide('jspb.RecyclerFunction'); +goog.provide('jspb.WriterFunction'); + +goog.forwardDeclare('jspb.Message'); +goog.forwardDeclare('jsproto.BinaryExtension'); + + + +/** + * Base interface class for all const messages. Does __not__ define any + * methods, as doing so on a widely-used interface defeats dead-code + * elimination. + * @interface + */ +jspb.ConstBinaryMessage = function() {}; + + + +/** + * Base interface class for all messages. Does __not__ define any methods, as + * doing so on a widely-used interface defeats dead-code elimination. + * @interface + * @extends {jspb.ConstBinaryMessage} + */ +jspb.BinaryMessage = function() {}; + + +/** + * The types convertible to Uint8Arrays. Strings are assumed to be + * base64-encoded. + * @typedef {ArrayBuffer|Uint8Array|Array|string} + */ +jspb.ByteSource; + + +/** + * A field in jspb can be a scalar, a block of bytes, another proto, or an + * array of any of the above. + * @typedef {boolean|number|string|Uint8Array| + jspb.BinaryMessage|jsproto.BinaryExtension| + Array} + */ +jspb.AnyFieldType; + + +/** + * A builder function creates an instance of a message object. + * @typedef {function():!jspb.BinaryMessage} + */ +jspb.BuilderFunction; + + +/** + * A cloner function creates a deep copy of a message object. + * @typedef {function(jspb.ConstBinaryMessage):jspb.BinaryMessage} + */ +jspb.ClonerFunction; + + +/** + * A recycler function destroys an instance of a message object. + * @typedef {function(!jspb.BinaryMessage):void} + */ +jspb.RecyclerFunction; + + +/** + * A reader function initializes a message using data from a BinaryReader. + * @typedef {function(!jspb.BinaryMessage, !jspb.BinaryReader):void} + */ +jspb.ReaderFunction; + + +/** + * A writer function serializes a message to a BinaryWriter. + * @typedef {!function(!jspb.Message, !jspb.BinaryWriter):void | + * !function(!jspb.ConstBinaryMessage, !jspb.BinaryWriter):void} + */ +jspb.WriterFunction; + + +/** + * Field type codes, taken from proto2/public/wire_format_lite.h. + * @enum {number} + */ +jspb.BinaryConstants.FieldType = { + INVALID: -1, + DOUBLE: 1, + FLOAT: 2, + INT64: 3, + UINT64: 4, + INT32: 5, + FIXED64: 6, + FIXED32: 7, + BOOL: 8, + STRING: 9, + GROUP: 10, + MESSAGE: 11, + BYTES: 12, + UINT32: 13, + ENUM: 14, + SFIXED32: 15, + SFIXED64: 16, + SINT32: 17, + SINT64: 18, + + // Extended types for Javascript + + FHASH64: 30, // 64-bit hash string, fixed-length encoding. + VHASH64: 31 // 64-bit hash string, varint encoding. +}; + + +/** + * Wire-format type codes, taken from proto2/public/wire_format_lite.h. + * @enum {number} + */ +jspb.BinaryConstants.WireType = { + INVALID: -1, + VARINT: 0, + FIXED64: 1, + DELIMITED: 2, + START_GROUP: 3, + END_GROUP: 4, + FIXED32: 5 +}; + + +/** + * Translates field type to wire type. + * @param {jspb.BinaryConstants.FieldType} fieldType + * @return {jspb.BinaryConstants.WireType} + */ +jspb.BinaryConstants.FieldTypeToWireType = function(fieldType) { + var fieldTypes = jspb.BinaryConstants.FieldType; + var wireTypes = jspb.BinaryConstants.WireType; + switch (fieldType) { + case fieldTypes.INT32: + case fieldTypes.INT64: + case fieldTypes.UINT32: + case fieldTypes.UINT64: + case fieldTypes.SINT32: + case fieldTypes.SINT64: + case fieldTypes.BOOL: + case fieldTypes.ENUM: + case fieldTypes.VHASH64: + return wireTypes.VARINT; + + case fieldTypes.DOUBLE: + case fieldTypes.FIXED64: + case fieldTypes.SFIXED64: + case fieldTypes.FHASH64: + return wireTypes.FIXED64; + + case fieldTypes.STRING: + case fieldTypes.MESSAGE: + case fieldTypes.BYTES: + return wireTypes.DELIMITED; + + case fieldTypes.FLOAT: + case fieldTypes.FIXED32: + case fieldTypes.SFIXED32: + return wireTypes.FIXED32; + + case fieldTypes.INVALID: + case fieldTypes.GROUP: + default: + return wireTypes.INVALID; + } +}; + + +/** + * Flag to indicate a missing field. + * @const {number} + */ +jspb.BinaryConstants.INVALID_FIELD_NUMBER = -1; + + +/** + * The smallest denormal float32 value. + * @const {number} + */ +jspb.BinaryConstants.FLOAT32_EPS = 1.401298464324817e-45; + + +/** + * The smallest normal float64 value. + * @const {number} + */ +jspb.BinaryConstants.FLOAT32_MIN = 1.1754943508222875e-38; + + +/** + * The largest finite float32 value. + * @const {number} + */ +jspb.BinaryConstants.FLOAT32_MAX = 3.4028234663852886e+38; + + +/** + * The smallest denormal float64 value. + * @const {number} + */ +jspb.BinaryConstants.FLOAT64_EPS = 5e-324; + + +/** + * The smallest normal float64 value. + * @const {number} + */ +jspb.BinaryConstants.FLOAT64_MIN = 2.2250738585072014e-308; + + +/** + * The largest finite float64 value. + * @const {number} + */ +jspb.BinaryConstants.FLOAT64_MAX = 1.7976931348623157e+308; + + +/** + * Convenience constant equal to 2^20. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_20 = 1048576; + + +/** + * Convenience constant equal to 2^23. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_23 = 8388608; + + +/** + * Convenience constant equal to 2^31. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_31 = 2147483648; + + +/** + * Convenience constant equal to 2^32. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_32 = 4294967296; + + +/** + * Convenience constant equal to 2^52. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_52 = 4503599627370496; + + +/** + * Convenience constant equal to 2^63. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_63 = 9223372036854775808; + + +/** + * Convenience constant equal to 2^64. + * @const {number} + */ +jspb.BinaryConstants.TWO_TO_64 = 18446744073709551616; + + +/** + * Eight-character string of zeros, used as the default 64-bit hash value. + * @const {string} + */ +jspb.BinaryConstants.ZERO_HASH = '\0\0\0\0\0\0\0\0'; diff --git a/js/binary/decoder.js b/js/binary/decoder.js new file mode 100644 index 00000000..9004eff0 --- /dev/null +++ b/js/binary/decoder.js @@ -0,0 +1,1005 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview This file contains utilities for decoding primitive values + * (signed and unsigned integers, varints, booleans, enums, hashes, strings, + * and raw bytes) embedded in Uint8Arrays into their corresponding Javascript + * types. + * + * Major caveat - Javascript is unable to accurately represent integers larger + * than 2^53 due to its use of a double-precision floating point format or all + * numbers. If you need to guarantee that 64-bit values survive with all bits + * intact, you _must_ read them using one of the Hash64 methods, which return + * an 8-character string. + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.provide('jspb.BinaryDecoder'); +goog.provide('jspb.BinaryIterator'); + +goog.require('goog.asserts'); +goog.require('jspb.utils'); + + + +/** + * Simple helper class for traversing the contents of repeated scalar fields. + * that may or may not have been packed into a wire-format blob. + * @param {?jspb.BinaryDecoder=} opt_decoder + * @param {?function(this:jspb.BinaryDecoder):(number|boolean|string)=} + * opt_next The decoder method to use for next(). + * @param {?Array.=} opt_elements + * @constructor + * @struct + */ +jspb.BinaryIterator = function(opt_decoder, opt_next, opt_elements) { + /** @private {jspb.BinaryDecoder} */ + this.decoder_ = null; + + /** + * The BinaryDecoder member function used when iterating over packed data. + * @private {?function(this:jspb.BinaryDecoder):(number|boolean|string)} + */ + this.nextMethod_ = null; + + /** @private {Array.} */ + this.elements_ = null; + + /** @private {number} */ + this.cursor_ = 0; + + /** @private {number|boolean|string|null} */ + this.nextValue_ = null; + + /** @private {boolean} */ + this.atEnd_ = true; + + this.init_(opt_decoder, opt_next, opt_elements); +}; + + +/** + * @param {?jspb.BinaryDecoder=} opt_decoder + * @param {?function(this:jspb.BinaryDecoder):(number|boolean|string)=} + * opt_next The decoder method to use for next(). + * @param {?Array.=} opt_elements + * @private + */ +jspb.BinaryIterator.prototype.init_ = + function(opt_decoder, opt_next, opt_elements) { + if (opt_decoder && opt_next) { + this.decoder_ = opt_decoder; + this.nextMethod_ = opt_next; + } + this.elements_ = opt_elements ? opt_elements : null; + this.cursor_ = 0; + this.nextValue_ = null; + this.atEnd_ = !this.decoder_ && !this.elements_; + + this.next(); +}; + + +/** + * Global pool of BinaryIterator instances. + * @private {!Array.} + */ +jspb.BinaryIterator.instanceCache_ = []; + + +/** + * Allocates a BinaryIterator from the cache, creating a new one if the cache + * is empty. + * @param {?jspb.BinaryDecoder=} opt_decoder + * @param {?function(this:jspb.BinaryDecoder):(number|boolean|string)=} + * opt_next The decoder method to use for next(). + * @param {?Array.=} opt_elements + * @return {!jspb.BinaryIterator} + */ +jspb.BinaryIterator.alloc = function(opt_decoder, opt_next, opt_elements) { + if (jspb.BinaryIterator.instanceCache_.length) { + var iterator = jspb.BinaryIterator.instanceCache_.pop(); + iterator.init_(opt_decoder, opt_next, opt_elements); + return iterator; + } else { + return new jspb.BinaryIterator(opt_decoder, opt_next, opt_elements); + } +}; + + +/** + * Puts this instance back in the instance cache. + */ +jspb.BinaryIterator.prototype.free = function() { + this.clear(); + if (jspb.BinaryIterator.instanceCache_.length < 100) { + jspb.BinaryIterator.instanceCache_.push(this); + } +}; + + +/** + * Clears the iterator. + */ +jspb.BinaryIterator.prototype.clear = function() { + if (this.decoder_) { + this.decoder_.free(); + } + this.decoder_ = null; + this.nextMethod_ = null; + this.elements_ = null; + this.cursor_ = 0; + this.nextValue_ = null; + this.atEnd_ = true; +}; + + +/** + * Returns the element at the iterator, or null if the iterator is invalid or + * past the end of the decoder/array. + * @return {number|boolean|string|null} + */ +jspb.BinaryIterator.prototype.get = function() { + return this.nextValue_; +}; + + +/** + * Returns true if the iterator is at the end of the decoder/array. + * @return {boolean} + */ +jspb.BinaryIterator.prototype.atEnd = function() { + return this.atEnd_; +}; + + +/** + * Returns the element at the iterator and steps to the next element, + * equivalent to '*pointer++' in C. + * @return {number|boolean|string|null} + */ +jspb.BinaryIterator.prototype.next = function() { + var lastValue = this.nextValue_; + if (this.decoder_) { + if (this.decoder_.atEnd()) { + this.nextValue_ = null; + this.atEnd_ = true; + } else { + this.nextValue_ = this.nextMethod_.call(this.decoder_); + } + } else if (this.elements_) { + if (this.cursor_ == this.elements_.length) { + this.nextValue_ = null; + this.atEnd_ = true; + } else { + this.nextValue_ = this.elements_[this.cursor_++]; + } + } + return lastValue; +}; + + + +/** + * BinaryDecoder implements the decoders for all the wire types specified in + * https://developers.google.com/protocol-buffers/docs/encoding. + * + * @param {jspb.ByteSource=} opt_bytes The bytes we're reading from. + * @param {number=} opt_start The optional offset to start reading at. + * @param {number=} opt_length The optional length of the block to read - + * we'll throw an assertion if we go off the end of the block. + * @constructor + * @struct + */ +jspb.BinaryDecoder = function(opt_bytes, opt_start, opt_length) { + /** + * Typed byte-wise view of the source buffer. + * @private {Uint8Array} + */ + this.bytes_ = null; + + /** + * Start point of the block to read. + * @private {number} + */ + this.start_ = 0; + + /** + * End point of the block to read. + * @private {number} + */ + this.end_ = 0; + + /** + * Current read location in bytes_. + * @private {number} + */ + this.cursor_ = 0; + + /** + * Temporary storage for the low 32 bits of 64-bit data types that we're + * decoding. + * @private {number} + */ + this.tempLow_ = 0; + + /** + * Temporary storage for the high 32 bits of 64-bit data types that we're + * decoding. + * @private {number} + */ + this.tempHigh_ = 0; + + /** + * Set to true if this decoder encountered an error due to corrupt data. + * @private {boolean} + */ + this.error_ = false; + + if (opt_bytes) { + this.setBlock(opt_bytes, opt_start, opt_length); + } +}; + + +/** + * Global pool of BinaryDecoder instances. + * @private {!Array.} + */ +jspb.BinaryDecoder.instanceCache_ = []; + + +/** + * Pops an instance off the instance cache, or creates one if the cache is + * empty. + * @param {jspb.ByteSource=} opt_bytes The bytes we're reading from. + * @param {number=} opt_start The optional offset to start reading at. + * @param {number=} opt_length The optional length of the block to read - + * we'll throw an assertion if we go off the end of the block. + * @return {!jspb.BinaryDecoder} + */ +jspb.BinaryDecoder.alloc = function(opt_bytes, opt_start, opt_length) { + if (jspb.BinaryDecoder.instanceCache_.length) { + var newDecoder = jspb.BinaryDecoder.instanceCache_.pop(); + if (opt_bytes) { + newDecoder.setBlock(opt_bytes, opt_start, opt_length); + } + return newDecoder; + } else { + return new jspb.BinaryDecoder(opt_bytes, opt_start, opt_length); + } +}; + + +/** + * Puts this instance back in the instance cache. + */ +jspb.BinaryDecoder.prototype.free = function() { + this.clear(); + if (jspb.BinaryDecoder.instanceCache_.length < 100) { + jspb.BinaryDecoder.instanceCache_.push(this); + } +}; + + +/** + * Makes a copy of this decoder. + * @return {!jspb.BinaryDecoder} + */ +jspb.BinaryDecoder.prototype.clone = function() { + return jspb.BinaryDecoder.alloc(this.bytes_, + this.start_, this.end_ - this.start_); +}; + + +/** + * Clears the decoder. + */ +jspb.BinaryDecoder.prototype.clear = function() { + this.bytes_ = null; + this.start_ = 0; + this.end_ = 0; + this.cursor_ = 0; + this.error_ = false; +}; + + +/** + * Returns the raw buffer. + * @return {Uint8Array} The raw buffer. + */ +jspb.BinaryDecoder.prototype.getBuffer = function() { + return this.bytes_; +}; + + +/** + * Changes the block of bytes we're decoding. + * @param {!jspb.ByteSource} data The bytes we're reading from. + * @param {number=} opt_start The optional offset to start reading at. + * @param {number=} opt_length The optional length of the block to read - + * we'll throw an assertion if we go off the end of the block. + */ +jspb.BinaryDecoder.prototype.setBlock = + function(data, opt_start, opt_length) { + this.bytes_ = jspb.utils.byteSourceToUint8Array(data); + this.start_ = goog.isDef(opt_start) ? opt_start : 0; + this.end_ = + goog.isDef(opt_length) ? this.start_ + opt_length : this.bytes_.length; + this.cursor_ = this.start_; +}; + + +/** + * @return {number} + */ +jspb.BinaryDecoder.prototype.getEnd = function() { + return this.end_; +}; + + +/** + * @param {number} end + */ +jspb.BinaryDecoder.prototype.setEnd = function(end) { + this.end_ = end; +}; + + +/** + * Moves the read cursor back to the start of the block. + */ +jspb.BinaryDecoder.prototype.reset = function() { + this.cursor_ = this.start_; +}; + + +/** + * Returns the internal read cursor. + * @return {number} The internal read cursor. + */ +jspb.BinaryDecoder.prototype.getCursor = function() { + return this.cursor_; +}; + + +/** + * Returns the internal read cursor. + * @param {number} cursor The new cursor. + */ +jspb.BinaryDecoder.prototype.setCursor = function(cursor) { + this.cursor_ = cursor; +}; + + +/** + * Advances the stream cursor by the given number of bytes. + * @param {number} count The number of bytes to advance by. + */ +jspb.BinaryDecoder.prototype.advance = function(count) { + this.cursor_ += count; + goog.asserts.assert(this.cursor_ <= this.end_); +}; + + +/** + * Returns true if this decoder is at the end of the block. + * @return {boolean} + */ +jspb.BinaryDecoder.prototype.atEnd = function() { + return this.cursor_ == this.end_; +}; + + +/** + * Returns true if this decoder is at the end of the block. + * @return {boolean} + */ +jspb.BinaryDecoder.prototype.pastEnd = function() { + return this.cursor_ > this.end_; +}; + + +/** + * Returns true if this decoder encountered an error due to corrupt data. + * @return {boolean} + */ +jspb.BinaryDecoder.prototype.getError = function() { + return this.error_ || + (this.cursor_ < 0) || + (this.cursor_ > this.end_); +}; + + +/** + * Reads an unsigned varint from the binary stream and stores it as a split + * 64-bit integer. Since this does not convert the value to a number, no + * precision is lost. + * + * It's possible for an unsigned varint to be incorrectly encoded - more than + * 64 bits' worth of data could be present. If this happens, this method will + * throw an error. + * + * Decoding varints requires doing some funny base-128 math - for more + * details on the format, see + * https://developers.google.com/protocol-buffers/docs/encoding + * + * @private + */ +jspb.BinaryDecoder.prototype.readSplitVarint64_ = function() { + var temp; + var lowBits = 0; + var highBits = 0; + + // Read the first four bytes of the varint, stopping at the terminator if we + // see it. + for (var i = 0; i < 4; i++) { + temp = this.bytes_[this.cursor_++]; + lowBits |= (temp & 0x7F) << (i * 7); + if (temp < 128) { + this.tempLow_ = lowBits >>> 0; + this.tempHigh_ = 0; + return; + } + } + + // Read the fifth byte, which straddles the low and high dwords. + temp = this.bytes_[this.cursor_++]; + lowBits |= (temp & 0x7F) << 28; + highBits |= (temp & 0x7F) >> 4; + if (temp < 128) { + this.tempLow_ = lowBits >>> 0; + this.tempHigh_ = highBits >>> 0; + return; + } + + // Read the sixth through tenth byte. + for (var i = 0; i < 5; i++) { + temp = this.bytes_[this.cursor_++]; + highBits |= (temp & 0x7F) << (i * 7 + 3); + if (temp < 128) { + this.tempLow_ = lowBits >>> 0; + this.tempHigh_ = highBits >>> 0; + return; + } + } + + // If we did not see the terminator, the encoding was invalid. + goog.asserts.fail('Failed to read varint, encoding is invalid.'); + this.error_ = true; +}; + + +/** + * Skips over a varint in the block without decoding it. + */ +jspb.BinaryDecoder.prototype.skipVarint = function() { + while (this.bytes_[this.cursor_] & 0x80) { + this.cursor_++; + } + this.cursor_++; +}; + + +/** + * Skips backwards over a varint in the block - to do this correctly, we have + * to know the value we're skipping backwards over or things are ambiguous. + * @param {number} value The varint value to unskip. + */ +jspb.BinaryDecoder.prototype.unskipVarint = function(value) { + while (value > 128) { + this.cursor_--; + value = value >>> 7; + } + this.cursor_--; +}; + + +/** + * Reads a 32-bit varint from the binary stream. Due to a quirk of the encoding + * format and Javascript's handling of bitwise math, this actually works + * correctly for both signed and unsigned 32-bit varints. + * + * This function is called vastly more frequently than any other in + * BinaryDecoder, so it has been unrolled and tweaked for performance. + * + * If there are more than 32 bits of data in the varint, it _must_ be due to + * sign-extension. If we're in debug mode and the high 32 bits don't match the + * expected sign extension, this method will throw an error. + * + * Decoding varints requires doing some funny base-128 math - for more + * details on the format, see + * https://developers.google.com/protocol-buffers/docs/encoding + * + * @return {number} The decoded unsigned 32-bit varint. + */ +jspb.BinaryDecoder.prototype.readUnsignedVarint32 = function() { + var temp; + var bytes = this.bytes_; + + temp = bytes[this.cursor_ + 0]; + var x = (temp & 0x7F); + if (temp < 128) { + this.cursor_ += 1; + goog.asserts.assert(this.cursor_ <= this.end_); + return x; + } + + temp = bytes[this.cursor_ + 1]; + x |= (temp & 0x7F) << 7; + if (temp < 128) { + this.cursor_ += 2; + goog.asserts.assert(this.cursor_ <= this.end_); + return x; + } + + temp = bytes[this.cursor_ + 2]; + x |= (temp & 0x7F) << 14; + if (temp < 128) { + this.cursor_ += 3; + goog.asserts.assert(this.cursor_ <= this.end_); + return x; + } + + temp = bytes[this.cursor_ + 3]; + x |= (temp & 0x7F) << 21; + if (temp < 128) { + this.cursor_ += 4; + goog.asserts.assert(this.cursor_ <= this.end_); + return x; + } + + temp = bytes[this.cursor_ + 4]; + x |= (temp & 0x0F) << 28; + if (temp < 128) { + // We're reading the high bits of an unsigned varint. The byte we just read + // also contains bits 33 through 35, which we're going to discard. Those + // bits _must_ be zero, or the encoding is invalid. + goog.asserts.assert((temp & 0xF0) == 0); + this.cursor_ += 5; + goog.asserts.assert(this.cursor_ <= this.end_); + return x >>> 0; + } + + // If we get here, we're reading the sign extension of a negative 32-bit int. + // We can skip these bytes, as we know in advance that they have to be all + // 1's if the varint is correctly encoded. Since we also know the value is + // negative, we don't have to coerce it to unsigned before we return it. + + goog.asserts.assert((temp & 0xF0) == 0xF0); + goog.asserts.assert(bytes[this.cursor_ + 5] == 0xFF); + goog.asserts.assert(bytes[this.cursor_ + 6] == 0xFF); + goog.asserts.assert(bytes[this.cursor_ + 7] == 0xFF); + goog.asserts.assert(bytes[this.cursor_ + 8] == 0xFF); + goog.asserts.assert(bytes[this.cursor_ + 9] == 0x01); + + this.cursor_ += 10; + goog.asserts.assert(this.cursor_ <= this.end_); + return x; +}; + + +/** + * The readUnsignedVarint32 above deals with signed 32-bit varints correctly, + * so this is just an alias. + * + * @return {number} The decoded signed 32-bit varint. + */ +jspb.BinaryDecoder.prototype.readSignedVarint32 = + jspb.BinaryDecoder.prototype.readUnsignedVarint32; + + +/** + * Reads a 32-bit unsigned variant and returns its value as a string. + * + * @return {string} The decoded unsigned 32-bit varint as a string. + */ +jspb.BinaryDecoder.prototype.readUnsignedVarint32String = function() { + // 32-bit integers fit in JavaScript numbers without loss of precision, so + // string variants of 32-bit varint readers can simply delegate then convert + // to string. + var value = this.readUnsignedVarint32(); + return value.toString(); +}; + +/** + * Reads a 32-bit signed variant and returns its value as a string. + * + * @return {string} The decoded signed 32-bit varint as a string. + */ +jspb.BinaryDecoder.prototype.readSignedVarint32String = function() { + // 32-bit integers fit in JavaScript numbers without loss of precision, so + // string variants of 32-bit varint readers can simply delegate then convert + // to string. + var value = this.readSignedVarint32(); + return value.toString(); +}; + + +/** + * Reads a signed, zigzag-encoded 32-bit varint from the binary stream. + * + * Zigzag encoding is a modification of varint encoding that reduces the + * storage overhead for small negative integers - for more details on the + * format, see https://developers.google.com/protocol-buffers/docs/encoding + * + * @return {number} The decoded signed, zigzag-encoded 32-bit varint. + */ +jspb.BinaryDecoder.prototype.readZigzagVarint32 = function() { + var result = this.readUnsignedVarint32(); + return (result >>> 1) ^ - (result & 1); +}; + + +/** + * Reads an unsigned 64-bit varint from the binary stream. Note that since + * Javascript represents all numbers as double-precision floats, there will be + * precision lost if the absolute value of the varint is larger than 2^53. + * + * @return {number} The decoded unsigned varint. Precision will be lost if the + * integer exceeds 2^53. + */ +jspb.BinaryDecoder.prototype.readUnsignedVarint64 = function() { + this.readSplitVarint64_(); + return jspb.utils.joinUint64(this.tempLow_, this.tempHigh_); +}; + + +/** + * Reads an unsigned 64-bit varint from the binary stream and returns the value + * as a decimal string. + * + * @return {string} The decoded unsigned varint as a decimal string. + */ +jspb.BinaryDecoder.prototype.readUnsignedVarint64String = function() { + this.readSplitVarint64_(); + return jspb.utils.joinUnsignedDecimalString(this.tempLow_, this.tempHigh_); +}; + + +/** + * Reads a signed 64-bit varint from the binary stream. Note that since + * Javascript represents all numbers as double-precision floats, there will be + * precision lost if the absolute value of the varint is larger than 2^53. + * + * @return {number} The decoded signed varint. Precision will be lost if the + * integer exceeds 2^53. + */ +jspb.BinaryDecoder.prototype.readSignedVarint64 = function() { + this.readSplitVarint64_(); + return jspb.utils.joinInt64(this.tempLow_, this.tempHigh_); +}; + + +/** + * Reads an signed 64-bit varint from the binary stream and returns the value + * as a decimal string. + * + * @return {string} The decoded signed varint as a decimal string. + */ +jspb.BinaryDecoder.prototype.readSignedVarint64String = function() { + this.readSplitVarint64_(); + return jspb.utils.joinSignedDecimalString(this.tempLow_, this.tempHigh_); +}; + + +/** + * Reads a signed, zigzag-encoded 64-bit varint from the binary stream. Note + * that since Javascript represents all numbers as double-precision floats, + * there will be precision lost if the absolute value of the varint is larger + * than 2^53. + * + * Zigzag encoding is a modification of varint encoding that reduces the + * storage overhead for small negative integers - for more details on the + * format, see https://developers.google.com/protocol-buffers/docs/encoding + * + * @return {number} The decoded zigzag varint. Precision will be lost if the + * integer exceeds 2^53. + */ +jspb.BinaryDecoder.prototype.readZigzagVarint64 = function() { + this.readSplitVarint64_(); + return jspb.utils.joinZigzag64(this.tempLow_, this.tempHigh_); +}; + + +/** + * Reads a raw unsigned 8-bit integer from the binary stream. + * + * @return {number} The unsigned 8-bit integer read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readUint8 = function() { + var a = this.bytes_[this.cursor_ + 0]; + this.cursor_ += 1; + goog.asserts.assert(this.cursor_ <= this.end_); + return a; +}; + + +/** + * Reads a raw unsigned 16-bit integer from the binary stream. + * + * @return {number} The unsigned 16-bit integer read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readUint16 = function() { + var a = this.bytes_[this.cursor_ + 0]; + var b = this.bytes_[this.cursor_ + 1]; + this.cursor_ += 2; + goog.asserts.assert(this.cursor_ <= this.end_); + return (a << 0) | (b << 8); +}; + + +/** + * Reads a raw unsigned 32-bit integer from the binary stream. + * + * @return {number} The unsigned 32-bit integer read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readUint32 = function() { + var a = this.bytes_[this.cursor_ + 0]; + var b = this.bytes_[this.cursor_ + 1]; + var c = this.bytes_[this.cursor_ + 2]; + var d = this.bytes_[this.cursor_ + 3]; + this.cursor_ += 4; + goog.asserts.assert(this.cursor_ <= this.end_); + return ((a << 0) | (b << 8) | (c << 16) | (d << 24)) >>> 0; +}; + + +/** + * Reads a raw unsigned 64-bit integer from the binary stream. Note that since + * Javascript represents all numbers as double-precision floats, there will be + * precision lost if the absolute value of the integer is larger than 2^53. + * + * @return {number} The unsigned 64-bit integer read from the binary stream. + * Precision will be lost if the integer exceeds 2^53. + */ +jspb.BinaryDecoder.prototype.readUint64 = function() { + var bitsLow = this.readUint32(); + var bitsHigh = this.readUint32(); + return jspb.utils.joinUint64(bitsLow, bitsHigh); +}; + + +/** + * Reads a raw signed 8-bit integer from the binary stream. + * + * @return {number} The signed 8-bit integer read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readInt8 = function() { + var a = this.bytes_[this.cursor_ + 0]; + this.cursor_ += 1; + goog.asserts.assert(this.cursor_ <= this.end_); + return (a << 24) >> 24; +}; + + +/** + * Reads a raw signed 16-bit integer from the binary stream. + * + * @return {number} The signed 16-bit integer read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readInt16 = function() { + var a = this.bytes_[this.cursor_ + 0]; + var b = this.bytes_[this.cursor_ + 1]; + this.cursor_ += 2; + goog.asserts.assert(this.cursor_ <= this.end_); + return (((a << 0) | (b << 8)) << 16) >> 16; +}; + + +/** + * Reads a raw signed 32-bit integer from the binary stream. + * + * @return {number} The signed 32-bit integer read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readInt32 = function() { + var a = this.bytes_[this.cursor_ + 0]; + var b = this.bytes_[this.cursor_ + 1]; + var c = this.bytes_[this.cursor_ + 2]; + var d = this.bytes_[this.cursor_ + 3]; + this.cursor_ += 4; + goog.asserts.assert(this.cursor_ <= this.end_); + return (a << 0) | (b << 8) | (c << 16) | (d << 24); +}; + + +/** + * Reads a raw signed 64-bit integer from the binary stream. Note that since + * Javascript represents all numbers as double-precision floats, there will be + * precision lost if the absolute vlaue of the integer is larger than 2^53. + * + * @return {number} The signed 64-bit integer read from the binary stream. + * Precision will be lost if the integer exceeds 2^53. + */ +jspb.BinaryDecoder.prototype.readInt64 = function() { + var bitsLow = this.readUint32(); + var bitsHigh = this.readUint32(); + return jspb.utils.joinInt64(bitsLow, bitsHigh); +}; + + +/** + * Reads a 32-bit floating-point number from the binary stream, using the + * temporary buffer to realign the data. + * + * @return {number} The float read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readFloat = function() { + var bitsLow = this.readUint32(); + var bitsHigh = 0; + return jspb.utils.joinFloat32(bitsLow, bitsHigh); +}; + + +/** + * Reads a 64-bit floating-point number from the binary stream, using the + * temporary buffer to realign the data. + * + * @return {number} The double read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readDouble = function() { + var bitsLow = this.readUint32(); + var bitsHigh = this.readUint32(); + return jspb.utils.joinFloat64(bitsLow, bitsHigh); +}; + + +/** + * Reads a boolean value from the binary stream. + * @return {boolean} The boolean read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readBool = function() { + return !!this.bytes_[this.cursor_++]; +}; + + +/** + * Reads an enum value from the binary stream, which are always encoded as + * signed varints. + * @return {number} The enum value read from the binary stream. + */ +jspb.BinaryDecoder.prototype.readEnum = function() { + return this.readSignedVarint32(); +}; + + +/** + * Reads and parses a UTF-8 encoded unicode string from the stream. + * The code is inspired by maps.vectortown.parse.StreamedDataViewReader, with + * the exception that the implementation here does not get confused if it + * encounters characters longer than three bytes. These characters are ignored + * though, as they are extremely rare: three UTF-8 bytes cover virtually all + * characters in common use (http://en.wikipedia.org/wiki/UTF-8). + * @param {number} length The length of the string to read. + * @return {string} The decoded string. + */ +jspb.BinaryDecoder.prototype.readString = function(length) { + var bytes = this.bytes_; + var cursor = this.cursor_; + var end = cursor + length; + var chars = []; + + while (cursor < end) { + var c = bytes[cursor++]; + if (c < 128) { // Regular 7-bit ASCII. + chars.push(c); + } else if (c < 192) { + // UTF-8 continuation mark. We are out of sync. This + // might happen if we attempted to read a character + // with more than three bytes. + continue; + } else if (c < 224) { // UTF-8 with two bytes. + var c2 = bytes[cursor++]; + chars.push(((c & 31) << 6) | (c2 & 63)); + } else if (c < 240) { // UTF-8 with three bytes. + var c2 = bytes[cursor++]; + var c3 = bytes[cursor++]; + chars.push(((c & 15) << 12) | ((c2 & 63) << 6) | (c3 & 63)); + } + } + + // String.fromCharCode.apply is faster than manually appending characters on + // Chrome 25+, and generates no additional cons string garbage. + var result = String.fromCharCode.apply(null, chars); + this.cursor_ = cursor; + return result; +}; + + +/** + * Reads and parses a UTF-8 encoded unicode string (with length prefix) from + * the stream. + * @return {string} The decoded string. + */ +jspb.BinaryDecoder.prototype.readStringWithLength = function() { + var length = this.readUnsignedVarint32(); + return this.readString(length); +}; + + +/** + * Reads a block of raw bytes from the binary stream. + * + * @param {number} length The number of bytes to read. + * @return {Uint8Array} The decoded block of bytes, or null if the length was + * invalid. + */ +jspb.BinaryDecoder.prototype.readBytes = function(length) { + if (length < 0 || + this.cursor_ + length > this.bytes_.length) { + this.error_ = true; + return null; + } + + var result = this.bytes_.subarray(this.cursor_, this.cursor_ + length); + + this.cursor_ += length; + goog.asserts.assert(this.cursor_ <= this.end_); + return result; +}; + + +/** + * Reads a 64-bit varint from the stream and returns it as an 8-character + * Unicode string for use as a hash table key. + * + * @return {string} The hash value. + */ +jspb.BinaryDecoder.prototype.readVarintHash64 = function() { + this.readSplitVarint64_(); + return jspb.utils.joinHash64(this.tempLow_, this.tempHigh_); +}; + + +/** + * Reads a 64-bit fixed-width value from the stream and returns it as an + * 8-character Unicode string for use as a hash table key. + * + * @return {string} The hash value. + */ +jspb.BinaryDecoder.prototype.readFixedHash64 = function() { + var bytes = this.bytes_; + var cursor = this.cursor_; + + var a = bytes[cursor + 0]; + var b = bytes[cursor + 1]; + var c = bytes[cursor + 2]; + var d = bytes[cursor + 3]; + var e = bytes[cursor + 4]; + var f = bytes[cursor + 5]; + var g = bytes[cursor + 6]; + var h = bytes[cursor + 7]; + + this.cursor_ += 8; + + return String.fromCharCode(a, b, c, d, e, f, g, h); +}; diff --git a/js/binary/decoder_test.js b/js/binary/decoder_test.js new file mode 100644 index 00000000..27342e49 --- /dev/null +++ b/js/binary/decoder_test.js @@ -0,0 +1,327 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Test cases for jspb's binary protocol buffer decoder. + * + * There are two particular magic numbers that need to be pointed out - + * 2^64-1025 is the largest number representable as both a double and an + * unsigned 64-bit integer, and 2^63-513 is the largest number representable as + * both a double and a signed 64-bit integer. + * + * Test suite is written using Jasmine -- see http://jasmine.github.io/ + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.require('goog.testing.asserts'); +goog.require('jspb.BinaryConstants'); +goog.require('jspb.BinaryDecoder'); +goog.require('jspb.BinaryWriter'); + + +/** + * Tests raw encoding and decoding of unsigned types. + * @param {Function} readValue + * @param {Function} writeValue + * @param {number} epsilon + * @param {number} upperLimit + * @param {Function} filter + * @suppress {missingProperties|visibility} + */ +function doTestUnsignedValue(readValue, + writeValue, epsilon, upperLimit, filter) { + var writer = new jspb.BinaryWriter(); + + // Encode zero and limits. + writeValue.call(writer, filter(0)); + writeValue.call(writer, filter(epsilon)); + writeValue.call(writer, filter(upperLimit)); + + // Encode positive values. + for (var cursor = epsilon; cursor < upperLimit; cursor *= 1.1) { + writeValue.call(writer, filter(cursor)); + } + + var reader = jspb.BinaryDecoder.alloc(writer.getResultBuffer()); + + // Check zero and limits. + assertEquals(filter(0), readValue.call(reader)); + assertEquals(filter(epsilon), readValue.call(reader)); + assertEquals(filter(upperLimit), readValue.call(reader)); + + // Check positive values. + for (var cursor = epsilon; cursor < upperLimit; cursor *= 1.1) { + if (filter(cursor) != readValue.call(reader)) throw 'fail!'; + } +} + + +/** + * Tests raw encoding and decoding of signed types. + * @param {Function} readValue + * @param {Function} writeValue + * @param {number} epsilon + * @param {number} lowerLimit + * @param {number} upperLimit + * @param {Function} filter + * @suppress {missingProperties} + */ +function doTestSignedValue(readValue, + writeValue, epsilon, lowerLimit, upperLimit, filter) { + var writer = new jspb.BinaryWriter(); + + // Encode zero and limits. + writeValue.call(writer, filter(lowerLimit)); + writeValue.call(writer, filter(-epsilon)); + writeValue.call(writer, filter(0)); + writeValue.call(writer, filter(epsilon)); + writeValue.call(writer, filter(upperLimit)); + + var inputValues = []; + + // Encode negative values. + for (var cursor = lowerLimit; cursor < -epsilon; cursor /= 1.1) { + var val = filter(cursor); + writeValue.call(writer, val); + inputValues.push(val); + } + + // Encode positive values. + for (var cursor = epsilon; cursor < upperLimit; cursor *= 1.1) { + var val = filter(cursor); + writeValue.call(writer, val); + inputValues.push(val); + } + + var reader = jspb.BinaryDecoder.alloc(writer.getResultBuffer()); + + // Check zero and limits. + assertEquals(filter(lowerLimit), readValue.call(reader)); + assertEquals(filter(-epsilon), readValue.call(reader)); + assertEquals(filter(0), readValue.call(reader)); + assertEquals(filter(epsilon), readValue.call(reader)); + assertEquals(filter(upperLimit), readValue.call(reader)); + + // Verify decoded values. + for (var i = 0; i < inputValues.length; i++) { + assertEquals(inputValues[i], readValue.call(reader)); + } +} + +describe('binaryDecoderTest', function() { + /** + * Tests the decoder instance cache. + * @suppress {visibility} + */ + it('testInstanceCache', function() { + // Empty the instance caches. + jspb.BinaryDecoder.instanceCache_ = []; + + // Allocating and then freeing a decoder should put it in the instance + // cache. + jspb.BinaryDecoder.alloc().free(); + + assertEquals(1, jspb.BinaryDecoder.instanceCache_.length); + + // Allocating and then freeing three decoders should leave us with three in + // the cache. + + var decoder1 = jspb.BinaryDecoder.alloc(); + var decoder2 = jspb.BinaryDecoder.alloc(); + var decoder3 = jspb.BinaryDecoder.alloc(); + decoder1.free(); + decoder2.free(); + decoder3.free(); + + assertEquals(3, jspb.BinaryDecoder.instanceCache_.length); + }); + + + /** + * Tests reading 64-bit integers as hash strings. + */ + it('testHashStrings', function() { + var writer = new jspb.BinaryWriter(); + + var hashA = String.fromCharCode(0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00); + var hashB = String.fromCharCode(0x12, 0x34, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00); + var hashC = String.fromCharCode(0x12, 0x34, 0x56, 0x78, + 0x87, 0x65, 0x43, 0x21); + var hashD = String.fromCharCode(0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF); + + writer.rawWriteVarintHash64(hashA); + writer.rawWriteVarintHash64(hashB); + writer.rawWriteVarintHash64(hashC); + writer.rawWriteVarintHash64(hashD); + + writer.rawWriteFixedHash64(hashA); + writer.rawWriteFixedHash64(hashB); + writer.rawWriteFixedHash64(hashC); + writer.rawWriteFixedHash64(hashD); + + var decoder = jspb.BinaryDecoder.alloc(writer.getResultBuffer()); + + assertEquals(hashA, decoder.readVarintHash64()); + assertEquals(hashB, decoder.readVarintHash64()); + assertEquals(hashC, decoder.readVarintHash64()); + assertEquals(hashD, decoder.readVarintHash64()); + + assertEquals(hashA, decoder.readFixedHash64()); + assertEquals(hashB, decoder.readFixedHash64()); + assertEquals(hashC, decoder.readFixedHash64()); + assertEquals(hashD, decoder.readFixedHash64()); + }); + + + /** + * Verifies that misuse of the decoder class triggers assertions. + * @suppress {checkTypes|visibility} + */ + it('testDecodeErrors', function() { + // Reading a value past the end of the stream should trigger an assertion. + var decoder = jspb.BinaryDecoder.alloc([0, 1, 2]); + assertThrows(function() {decoder.readUint64()}); + + // Overlong varints should trigger assertions. + decoder.setBlock( + [255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0]); + assertThrows(function() {decoder.readUnsignedVarint64()}); + decoder.reset(); + assertThrows(function() {decoder.readSignedVarint64()}); + decoder.reset(); + assertThrows(function() {decoder.readZigzagVarint64()}); + + // Positive 32-bit varints encoded with 1 bits in positions 33 through 35 + // should trigger assertions. + decoder.setBlock([255, 255, 255, 255, 0x1F]); + assertThrows(function() {decoder.readUnsignedVarint32()}); + + decoder.setBlock([255, 255, 255, 255, 0x2F]); + assertThrows(function() {decoder.readUnsignedVarint32()}); + + decoder.setBlock([255, 255, 255, 255, 0x4F]); + assertThrows(function() {decoder.readUnsignedVarint32()}); + + // Negative 32-bit varints encoded with non-1 bits in the high dword should + // trigger assertions. + decoder.setBlock([255, 255, 255, 255, 255, 255, 0, 255, 255, 1]); + assertThrows(function() {decoder.readUnsignedVarint32()}); + + decoder.setBlock([255, 255, 255, 255, 255, 255, 255, 255, 255, 0]); + assertThrows(function() {decoder.readUnsignedVarint32()}); + }); + + + /** + * Tests raw encoding and decoding of unsigned integers. + */ + it('testRawUnsigned', function() { + doTestUnsignedValue( + jspb.BinaryDecoder.prototype.readUint8, + jspb.BinaryWriter.prototype.rawWriteUint8, + 1, 0xFF, Math.round); + + doTestUnsignedValue( + jspb.BinaryDecoder.prototype.readUint16, + jspb.BinaryWriter.prototype.rawWriteUint16, + 1, 0xFFFF, Math.round); + + doTestUnsignedValue( + jspb.BinaryDecoder.prototype.readUint32, + jspb.BinaryWriter.prototype.rawWriteUint32, + 1, 0xFFFFFFFF, Math.round); + + doTestUnsignedValue( + jspb.BinaryDecoder.prototype.readUint64, + jspb.BinaryWriter.prototype.rawWriteUint64, + 1, Math.pow(2, 64) - 1025, Math.round); + }); + + + /** + * Tests raw encoding and decoding of signed integers. + */ + it('testRawSigned', function() { + doTestSignedValue( + jspb.BinaryDecoder.prototype.readInt8, + jspb.BinaryWriter.prototype.rawWriteInt8, + 1, -0x80, 0x7F, Math.round); + + doTestSignedValue( + jspb.BinaryDecoder.prototype.readInt16, + jspb.BinaryWriter.prototype.rawWriteInt16, + 1, -0x8000, 0x7FFF, Math.round); + + doTestSignedValue( + jspb.BinaryDecoder.prototype.readInt32, + jspb.BinaryWriter.prototype.rawWriteInt32, + 1, -0x80000000, 0x7FFFFFFF, Math.round); + + doTestSignedValue( + jspb.BinaryDecoder.prototype.readInt64, + jspb.BinaryWriter.prototype.rawWriteInt64, + 1, -Math.pow(2, 63), Math.pow(2, 63) - 513, Math.round); + }); + + + /** + * Tests raw encoding and decoding of floats. + */ + it('testRawFloats', function() { + /** + * @param {number} x + * @return {number} + */ + function truncate(x) { + var temp = new Float32Array(1); + temp[0] = x; + return temp[0]; + } + doTestSignedValue( + jspb.BinaryDecoder.prototype.readFloat, + jspb.BinaryWriter.prototype.rawWriteFloat, + jspb.BinaryConstants.FLOAT32_EPS, + -jspb.BinaryConstants.FLOAT32_MAX, + jspb.BinaryConstants.FLOAT32_MAX, + truncate); + + doTestSignedValue( + jspb.BinaryDecoder.prototype.readDouble, + jspb.BinaryWriter.prototype.rawWriteDouble, + jspb.BinaryConstants.FLOAT64_EPS * 10, + -jspb.BinaryConstants.FLOAT64_MAX, + jspb.BinaryConstants.FLOAT64_MAX, + function(x) { return x; }); + }); +}); diff --git a/js/binary/proto_test.js b/js/binary/proto_test.js new file mode 100644 index 00000000..32d8412f --- /dev/null +++ b/js/binary/proto_test.js @@ -0,0 +1,588 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Test suite is written using Jasmine -- see http://jasmine.github.io/ + +goog.require('goog.testing.asserts'); +goog.require('proto.jspb.test.ExtendsWithMessage'); +goog.require('proto.jspb.test.ForeignEnum'); +goog.require('proto.jspb.test.ForeignMessage'); +goog.require('proto.jspb.test.TestAllTypes'); +goog.require('proto.jspb.test.TestExtendable'); + +var suite = {}; + +/** + * Helper: fill all fields on a TestAllTypes message. + * @param {proto.jspb.test.TestAllTypes} msg + */ +function fillAllFields(msg) { + msg.setOptionalInt32(-42); + // can be exactly represented by JS number (64-bit double, i.e., 52-bit + // mantissa). + msg.setOptionalInt64(-0x7fffffff00000000); + msg.setOptionalUint32(0x80000000); + msg.setOptionalUint64(0xf000000000000000); + msg.setOptionalSint32(-100); + msg.setOptionalSint64(-0x8000000000000000); + msg.setOptionalFixed32(1234); + msg.setOptionalFixed64(0x1234567800000000); + msg.setOptionalSfixed32(-1234); + msg.setOptionalSfixed64(-0x1234567800000000); + msg.setOptionalFloat(1.5); + msg.setOptionalDouble(-1.5); + msg.setOptionalBool(true); + msg.setOptionalString('hello world'); + msg.setOptionalBytes('bytes'); + msg.setOptionalGroup(new proto.jspb.test.TestAllTypes.OptionalGroup()); + msg.getOptionalGroup().setA(100); + var submsg = new proto.jspb.test.ForeignMessage(); + submsg.setC(16); + msg.setOptionalForeignMessage(submsg); + msg.setOptionalForeignEnum(proto.jspb.test.ForeignEnum.FOREIGN_FOO); + msg.setOptionalInt32String('-12345'); + msg.setOptionalUint32String('12345'); + msg.setOptionalInt64String('-123456789012345'); + msg.setOptionalUint64String('987654321098765'); + msg.setOneofString('oneof'); + + msg.setRepeatedInt32List([-42]); + msg.setRepeatedInt64List([-0x7fffffff00000000]); + msg.setRepeatedUint32List([0x80000000]); + msg.setRepeatedUint64List([0xf000000000000000]); + msg.setRepeatedSint32List([-100]); + msg.setRepeatedSint64List([-0x8000000000000000]); + msg.setRepeatedFixed32List([1234]); + msg.setRepeatedFixed64List([0x1234567800000000]); + msg.setRepeatedSfixed32List([-1234]); + msg.setRepeatedSfixed64List([-0x1234567800000000]); + msg.setRepeatedFloatList([1.5]); + msg.setRepeatedDoubleList([-1.5]); + msg.setRepeatedBoolList([true]); + msg.setRepeatedStringList(['hello world']); + msg.setRepeatedBytesList(['bytes']); + msg.setRepeatedGroupList([new proto.jspb.test.TestAllTypes.RepeatedGroup()]); + msg.getRepeatedGroupList()[0].setA(100); + submsg = new proto.jspb.test.ForeignMessage(); + submsg.setC(1000); + msg.setRepeatedForeignMessageList([submsg]); + msg.setRepeatedForeignEnumList([proto.jspb.test.ForeignEnum.FOREIGN_FOO]); + + msg.setPackedRepeatedInt32List([-42]); + msg.setPackedRepeatedInt64List([-0x7fffffff00000000]); + msg.setPackedRepeatedUint32List([0x80000000]); + msg.setPackedRepeatedUint64List([0xf000000000000000]); + msg.setPackedRepeatedSint32List([-100]); + msg.setPackedRepeatedSint64List([-0x8000000000000000]); + msg.setPackedRepeatedFixed32List([1234]); + msg.setPackedRepeatedFixed64List([0x1234567800000000]); + msg.setPackedRepeatedSfixed32List([-1234]); + msg.setPackedRepeatedSfixed64List([-0x1234567800000000]); + msg.setPackedRepeatedFloatList([1.5]); + msg.setPackedRepeatedDoubleList([-1.5]); + msg.setPackedRepeatedBoolList([true]); + + msg.setRepeatedInt32StringList(['-12345']); + msg.setRepeatedUint32StringList(['12345']); + msg.setRepeatedInt64StringList(['-123456789012345']); + msg.setRepeatedUint64StringList(['987654321098765']); + msg.setPackedRepeatedInt32StringList(['-12345']); + msg.setPackedRepeatedUint32StringList(['12345']); + msg.setPackedRepeatedInt64StringList(['-123456789012345']); + msg.setPackedRepeatedUint64StringList(['987654321098765']); +} + + +/** + * Helper: compare a bytes field to a string with codepoints 0--255. + * @param {Uint8Array|string} arr + * @param {string} str + * @return {boolean} + */ +function bytesCompare(arr, str) { + if (arr.length != str.length) { + return false; + } + if (typeof arr == 'string') { + for (var i = 0; i < arr.length; i++) { + if (arr.charCodeAt(i) != str.charCodeAt(i)) { + return false; + } + } + return true; + } else { + for (var i = 0; i < arr.length; i++) { + if (arr[i] != str.charCodeAt(i)) { + return false; + } + } + return true; + } +} + + +/** + * Helper: verify contents of given TestAllTypes message as set by + * fillAllFields(). + * @param {proto.jspb.test.TestAllTypes} msg + */ +function checkAllFields(msg) { + assertEquals(msg.getOptionalInt32(), -42); + assertEquals(msg.getOptionalInt64(), -0x7fffffff00000000); + assertEquals(msg.getOptionalUint32(), 0x80000000); + assertEquals(msg.getOptionalUint64(), 0xf000000000000000); + assertEquals(msg.getOptionalSint32(), -100); + assertEquals(msg.getOptionalSint64(), -0x8000000000000000); + assertEquals(msg.getOptionalFixed32(), 1234); + assertEquals(msg.getOptionalFixed64(), 0x1234567800000000); + assertEquals(msg.getOptionalSfixed32(), -1234); + assertEquals(msg.getOptionalSfixed64(), -0x1234567800000000); + assertEquals(msg.getOptionalFloat(), 1.5); + assertEquals(msg.getOptionalDouble(), -1.5); + assertEquals(msg.getOptionalBool(), true); + assertEquals(msg.getOptionalString(), 'hello world'); + assertEquals(true, bytesCompare(msg.getOptionalBytes(), 'bytes')); + assertEquals(msg.getOptionalGroup().getA(), 100); + assertEquals(msg.getOptionalForeignMessage().getC(), 16); + assertEquals(msg.getOptionalForeignEnum(), + proto.jspb.test.ForeignEnum.FOREIGN_FOO); + assertEquals(msg.getOptionalInt32String(), '-12345'); + assertEquals(msg.getOptionalUint32String(), '12345'); + assertEquals(msg.getOptionalInt64String(), '-123456789012345'); + assertEquals(msg.getOptionalUint64String(), '987654321098765'); + assertEquals(msg.getOneofString(), 'oneof'); + assertEquals(msg.getOneofFieldCase(), + proto.jspb.test.TestAllTypes.OneofFieldCase.ONEOF_STRING); + + assertElementsEquals(msg.getRepeatedInt32List(), [-42]); + assertElementsEquals(msg.getRepeatedInt64List(), [-0x7fffffff00000000]); + assertElementsEquals(msg.getRepeatedUint32List(), [0x80000000]); + assertElementsEquals(msg.getRepeatedUint64List(), [0xf000000000000000]); + assertElementsEquals(msg.getRepeatedSint32List(), [-100]); + assertElementsEquals(msg.getRepeatedSint64List(), [-0x8000000000000000]); + assertElementsEquals(msg.getRepeatedFixed32List(), [1234]); + assertElementsEquals(msg.getRepeatedFixed64List(), [0x1234567800000000]); + assertElementsEquals(msg.getRepeatedSfixed32List(), [-1234]); + assertElementsEquals(msg.getRepeatedSfixed64List(), [-0x1234567800000000]); + assertElementsEquals(msg.getRepeatedFloatList(), [1.5]); + assertElementsEquals(msg.getRepeatedDoubleList(), [-1.5]); + assertElementsEquals(msg.getRepeatedBoolList(), [true]); + assertElementsEquals(msg.getRepeatedStringList(), ['hello world']); + assertEquals(msg.getRepeatedBytesList().length, 1); + assertEquals(true, bytesCompare(msg.getRepeatedBytesList()[0], 'bytes')); + assertEquals(msg.getRepeatedGroupList().length, 1); + assertEquals(msg.getRepeatedGroupList()[0].getA(), 100); + assertEquals(msg.getRepeatedForeignMessageList().length, 1); + assertEquals(msg.getRepeatedForeignMessageList()[0].getC(), 1000); + assertElementsEquals(msg.getRepeatedForeignEnumList(), + [proto.jspb.test.ForeignEnum.FOREIGN_FOO]); + + assertElementsEquals(msg.getPackedRepeatedInt32List(), [-42]); + assertElementsEquals(msg.getPackedRepeatedInt64List(), + [-0x7fffffff00000000]); + assertElementsEquals(msg.getPackedRepeatedUint32List(), [0x80000000]); + assertElementsEquals(msg.getPackedRepeatedUint64List(), [0xf000000000000000]); + assertElementsEquals(msg.getPackedRepeatedSint32List(), [-100]); + assertElementsEquals(msg.getPackedRepeatedSint64List(), + [-0x8000000000000000]); + assertElementsEquals(msg.getPackedRepeatedFixed32List(), [1234]); + assertElementsEquals(msg.getPackedRepeatedFixed64List(), + [0x1234567800000000]); + assertElementsEquals(msg.getPackedRepeatedSfixed32List(), [-1234]); + assertElementsEquals(msg.getPackedRepeatedSfixed64List(), + [-0x1234567800000000]); + assertElementsEquals(msg.getPackedRepeatedFloatList(), [1.5]); + assertElementsEquals(msg.getPackedRepeatedDoubleList(), [-1.5]); + assertElementsEquals(msg.getPackedRepeatedBoolList(), [true]); + + assertEquals(msg.getRepeatedInt32StringList().length, 1); + assertElementsEquals(msg.getRepeatedInt32StringList(), ['-12345']); + assertEquals(msg.getRepeatedUint32StringList().length, 1); + assertElementsEquals(msg.getRepeatedUint32StringList(), ['12345']); + assertEquals(msg.getRepeatedInt64StringList().length, 1); + assertElementsEquals(msg.getRepeatedInt64StringList(), ['-123456789012345']); + assertEquals(msg.getRepeatedUint64StringList().length, 1); + assertElementsEquals(msg.getRepeatedUint64StringList(), ['987654321098765']); + + assertEquals(msg.getPackedRepeatedInt32StringList().length, 1); + assertElementsEquals(msg.getPackedRepeatedInt32StringList(), ['-12345']); + assertEquals(msg.getPackedRepeatedUint32StringList().length, 1); + assertElementsEquals(msg.getPackedRepeatedUint32StringList(), ['12345']); + assertEquals(msg.getPackedRepeatedInt64StringList().length, 1); + assertElementsEquals(msg.getPackedRepeatedInt64StringList(), + ['-123456789012345']); + assertEquals(msg.getPackedRepeatedUint64StringList().length, 1); + assertElementsEquals(msg.getPackedRepeatedUint64StringList(), + ['987654321098765']); +} + + +/** + * Helper: verify that all expected extensions are present. + * @param {!proto.jspb.test.TestExtendable} msg + */ +function checkExtensions(msg) { + assertEquals(-42, + msg.getExtension(proto.jspb.test.extendOptionalInt32)); + assertEquals(-0x7fffffff00000000, + msg.getExtension(proto.jspb.test.extendOptionalInt64)); + assertEquals(0x80000000, + msg.getExtension(proto.jspb.test.extendOptionalUint32)); + assertEquals(0xf000000000000000, + msg.getExtension(proto.jspb.test.extendOptionalUint64)); + assertEquals(-100, + msg.getExtension(proto.jspb.test.extendOptionalSint32)); + assertEquals(-0x8000000000000000, + msg.getExtension(proto.jspb.test.extendOptionalSint64)); + assertEquals(1234, + msg.getExtension(proto.jspb.test.extendOptionalFixed32)); + assertEquals(0x1234567800000000, + msg.getExtension(proto.jspb.test.extendOptionalFixed64)); + assertEquals(-1234, + msg.getExtension(proto.jspb.test.extendOptionalSfixed32)); + assertEquals(-0x1234567800000000, + msg.getExtension(proto.jspb.test.extendOptionalSfixed64)); + assertEquals(1.5, + msg.getExtension(proto.jspb.test.extendOptionalFloat)); + assertEquals(-1.5, + msg.getExtension(proto.jspb.test.extendOptionalDouble)); + assertEquals(true, + msg.getExtension(proto.jspb.test.extendOptionalBool)); + assertEquals('hello world', + msg.getExtension(proto.jspb.test.extendOptionalString)); + assertEquals(true, + bytesCompare(msg.getExtension(proto.jspb.test.extendOptionalBytes), + 'bytes')); + assertEquals(16, + msg.getExtension( + proto.jspb.test.ExtendsWithMessage.optionalExtension).getFoo()); + assertEquals(proto.jspb.test.ForeignEnum.FOREIGN_FOO, + msg.getExtension(proto.jspb.test.extendOptionalForeignEnum)); + assertEquals('-12345', + msg.getExtension(proto.jspb.test.extendOptionalInt32String)); + assertEquals('12345', + msg.getExtension(proto.jspb.test.extendOptionalUint32String)); + assertEquals('-123456789012345', + msg.getExtension(proto.jspb.test.extendOptionalInt64String)); + assertEquals('987654321098765', + msg.getExtension(proto.jspb.test.extendOptionalUint64String)); + + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedInt32List), + [-42]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedInt64List), + [-0x7fffffff00000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedUint32List), + [0x80000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedUint64List), + [0xf000000000000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedSint32List), + [-100]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedSint64List), + [-0x8000000000000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedFixed32List), + [1234]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedFixed64List), + [0x1234567800000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedSfixed32List), + [-1234]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedSfixed64List), + [-0x1234567800000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedFloatList), + [1.5]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedDoubleList), + [-1.5]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedBoolList), + [true]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedStringList), + ['hello world']); + assertEquals(true, + bytesCompare( + msg.getExtension(proto.jspb.test.extendRepeatedBytesList)[0], + 'bytes')); + assertEquals(1000, + msg.getExtension( + proto.jspb.test.ExtendsWithMessage.repeatedExtensionList)[0] + .getFoo()); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedForeignEnumList), + [proto.jspb.test.ForeignEnum.FOREIGN_FOO]); + + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedInt32StringList), + ['-12345']); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedUint32StringList), + ['12345']); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedInt64StringList), + ['-123456789012345']); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendRepeatedUint64StringList), + ['987654321098765']); + + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedInt32List), + [-42]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedInt64List), + [-0x7fffffff00000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedUint32List), + [0x80000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedUint64List), + [0xf000000000000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedSint32List), + [-100]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedSint64List), + [-0x8000000000000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedFixed32List), + [1234]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedFixed64List), + [0x1234567800000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedSfixed32List), + [-1234]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedSfixed64List), + [-0x1234567800000000]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedFloatList), + [1.5]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedDoubleList), + [-1.5]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedBoolList), + [true]); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedForeignEnumList), + [proto.jspb.test.ForeignEnum.FOREIGN_FOO]); + + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedInt32StringList), + ['-12345']); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedUint32StringList), + ['12345']); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedInt64StringList), + ['-123456789012345']); + assertElementsEquals( + msg.getExtension(proto.jspb.test.extendPackedRepeatedUint64StringList), + ['987654321098765']); +} + + +describe('protoBinaryTest', function() { + /** + * Tests a basic serialization-deserializaton round-trip with all supported + * field types (on the TestAllTypes message type). + */ + it('testRoundTrip', function() { + var msg = new proto.jspb.test.TestAllTypes(); + fillAllFields(msg); + var encoded = msg.serializeBinary(); + var decoded = proto.jspb.test.TestAllTypes.deserializeBinary(encoded); + checkAllFields(decoded); + }); + + + /** + * Helper: fill all extension values. + * @param {proto.jspb.test.TestExtendable} msg + */ + function fillExtensions(msg) { + msg.setExtension( + proto.jspb.test.extendOptionalInt32, -42); + msg.setExtension( + proto.jspb.test.extendOptionalInt64, -0x7fffffff00000000); + msg.setExtension( + proto.jspb.test.extendOptionalUint32, 0x80000000); + msg.setExtension( + proto.jspb.test.extendOptionalUint64, 0xf000000000000000); + msg.setExtension( + proto.jspb.test.extendOptionalSint32, -100); + msg.setExtension( + proto.jspb.test.extendOptionalSint64, -0x8000000000000000); + msg.setExtension( + proto.jspb.test.extendOptionalFixed32, 1234); + msg.setExtension( + proto.jspb.test.extendOptionalFixed64, 0x1234567800000000); + msg.setExtension( + proto.jspb.test.extendOptionalSfixed32, -1234); + msg.setExtension( + proto.jspb.test.extendOptionalSfixed64, -0x1234567800000000); + msg.setExtension( + proto.jspb.test.extendOptionalFloat, 1.5); + msg.setExtension( + proto.jspb.test.extendOptionalDouble, -1.5); + msg.setExtension( + proto.jspb.test.extendOptionalBool, true); + msg.setExtension( + proto.jspb.test.extendOptionalString, 'hello world'); + msg.setExtension( + proto.jspb.test.extendOptionalBytes, 'bytes'); + var submsg = new proto.jspb.test.ExtendsWithMessage(); + submsg.setFoo(16); + msg.setExtension( + proto.jspb.test.ExtendsWithMessage.optionalExtension, submsg); + msg.setExtension( + proto.jspb.test.extendOptionalForeignEnum, + proto.jspb.test.ForeignEnum.FOREIGN_FOO); + msg.setExtension( + proto.jspb.test.extendOptionalInt32String, '-12345'); + msg.setExtension( + proto.jspb.test.extendOptionalUint32String, '12345'); + msg.setExtension( + proto.jspb.test.extendOptionalInt64String, '-123456789012345'); + msg.setExtension( + proto.jspb.test.extendOptionalUint64String, '987654321098765'); + + msg.setExtension( + proto.jspb.test.extendRepeatedInt32List, [-42]); + msg.setExtension( + proto.jspb.test.extendRepeatedInt64List, [-0x7fffffff00000000]); + msg.setExtension( + proto.jspb.test.extendRepeatedUint32List, [0x80000000]); + msg.setExtension( + proto.jspb.test.extendRepeatedUint64List, [0xf000000000000000]); + msg.setExtension( + proto.jspb.test.extendRepeatedSint32List, [-100]); + msg.setExtension( + proto.jspb.test.extendRepeatedSint64List, [-0x8000000000000000]); + msg.setExtension( + proto.jspb.test.extendRepeatedFixed32List, [1234]); + msg.setExtension( + proto.jspb.test.extendRepeatedFixed64List, [0x1234567800000000]); + msg.setExtension( + proto.jspb.test.extendRepeatedSfixed32List, [-1234]); + msg.setExtension( + proto.jspb.test.extendRepeatedSfixed64List, [-0x1234567800000000]); + msg.setExtension( + proto.jspb.test.extendRepeatedFloatList, [1.5]); + msg.setExtension( + proto.jspb.test.extendRepeatedDoubleList, [-1.5]); + msg.setExtension( + proto.jspb.test.extendRepeatedBoolList, [true]); + msg.setExtension( + proto.jspb.test.extendRepeatedStringList, ['hello world']); + msg.setExtension( + proto.jspb.test.extendRepeatedBytesList, ['bytes']); + submsg = new proto.jspb.test.ExtendsWithMessage(); + submsg.setFoo(1000); + msg.setExtension( + proto.jspb.test.ExtendsWithMessage.repeatedExtensionList, [submsg]); + msg.setExtension(proto.jspb.test.extendRepeatedForeignEnumList, + [proto.jspb.test.ForeignEnum.FOREIGN_FOO]); + + msg.setExtension( + proto.jspb.test.extendRepeatedInt32StringList, ['-12345']); + msg.setExtension( + proto.jspb.test.extendRepeatedUint32StringList, ['12345']); + msg.setExtension( + proto.jspb.test.extendRepeatedInt64StringList, ['-123456789012345']); + msg.setExtension( + proto.jspb.test.extendRepeatedUint64StringList, ['987654321098765']); + + msg.setExtension( + proto.jspb.test.extendPackedRepeatedInt32List, [-42]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedInt64List, [-0x7fffffff00000000]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedUint32List, [0x80000000]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedUint64List, [0xf000000000000000]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedSint32List, [-100]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedSint64List, [-0x8000000000000000]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedFixed32List, [1234]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedFixed64List, [0x1234567800000000]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedSfixed32List, [-1234]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedSfixed64List, + [-0x1234567800000000]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedFloatList, [1.5]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedDoubleList, [-1.5]); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedBoolList, [true]); + msg.setExtension(proto.jspb.test.extendPackedRepeatedForeignEnumList, + [proto.jspb.test.ForeignEnum.FOREIGN_FOO]); + + msg.setExtension( + proto.jspb.test.extendPackedRepeatedInt32StringList, + ['-12345']); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedUint32StringList, + ['12345']); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedInt64StringList, + ['-123456789012345']); + msg.setExtension( + proto.jspb.test.extendPackedRepeatedUint64StringList, + ['987654321098765']); + } + + + /** + * Tests extension serialization and deserialization. + */ + it('testExtensions', function() { + var msg = new proto.jspb.test.TestExtendable(); + fillExtensions(msg); + var encoded = msg.serializeBinary(); + var decoded = proto.jspb.test.TestExtendable.deserializeBinary(encoded); + checkExtensions(decoded); + }); +}); diff --git a/js/binary/reader.js b/js/binary/reader.js new file mode 100644 index 00000000..abcd1660 --- /dev/null +++ b/js/binary/reader.js @@ -0,0 +1,1127 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview This file contains utilities for converting binary, + * wire-format protocol buffers into Javascript data structures. + * + * jspb's BinaryReader class wraps the BinaryDecoder class to add methods + * that understand the protocol buffer syntax and can do the type checking and + * bookkeeping necessary to parse trees of nested messages. + * + * Major caveat - Users of this library _must_ keep their Javascript proto + * parsing code in sync with the original .proto file - presumably you'll be + * using the typed jspb code generator, but if you bypass that you'll need + * to keep things in sync by hand. + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.provide('jspb.BinaryReader'); + +goog.require('goog.asserts'); +goog.require('jspb.BinaryConstants'); +goog.require('jspb.BinaryDecoder'); + + + +/** + * BinaryReader implements the decoders for all the wire types specified in + * https://developers.google.com/protocol-buffers/docs/encoding. + * + * @param {jspb.ByteSource=} opt_bytes The bytes we're reading from. + * @param {number=} opt_start The optional offset to start reading at. + * @param {number=} opt_length The optional length of the block to read - + * we'll throw an assertion if we go off the end of the block. + * @constructor + * @struct + */ +jspb.BinaryReader = function(opt_bytes, opt_start, opt_length) { + /** + * Wire-format decoder. + * @private {!jspb.BinaryDecoder} + */ + this.decoder_ = jspb.BinaryDecoder.alloc(opt_bytes, opt_start, opt_length); + + /** + * Cursor immediately before the field tag. + * @private {number} + */ + this.fieldCursor_ = this.decoder_.getCursor(); + + /** + * Field number of the next field in the buffer, filled in by nextField(). + * @private {number} + */ + this.nextField_ = jspb.BinaryConstants.INVALID_FIELD_NUMBER; + + /** + * Wire type of the next proto field in the buffer, filled in by + * nextField(). + * @private {jspb.BinaryConstants.WireType} + */ + this.nextWireType_ = jspb.BinaryConstants.WireType.INVALID; + + /** + * Set to true if this reader encountered an error due to corrupt data. + * @private {boolean} + */ + this.error_ = false; + + /** + * User-defined reader callbacks. + * @private {Object.} + */ + this.readCallbacks_ = null; +}; + + +/** + * Global pool of BinaryReader instances. + * @private {!Array.} + */ +jspb.BinaryReader.instanceCache_ = []; + + +/** + * Pops an instance off the instance cache, or creates one if the cache is + * empty. + * @param {jspb.ByteSource=} opt_bytes The bytes we're reading from. + * @param {number=} opt_start The optional offset to start reading at. + * @param {number=} opt_length The optional length of the block to read - + * we'll throw an assertion if we go off the end of the block. + * @return {!jspb.BinaryReader} + */ +jspb.BinaryReader.alloc = + function(opt_bytes, opt_start, opt_length) { + if (jspb.BinaryReader.instanceCache_.length) { + var newReader = jspb.BinaryReader.instanceCache_.pop(); + if (opt_bytes) { + newReader.decoder_.setBlock(opt_bytes, opt_start, opt_length); + } + return newReader; + } else { + return new jspb.BinaryReader(opt_bytes, opt_start, opt_length); + } +}; + + +/** + * Alias for the above method. + * @param {jspb.ByteSource=} opt_bytes The bytes we're reading from. + * @param {number=} opt_start The optional offset to start reading at. + * @param {number=} opt_length The optional length of the block to read - + * we'll throw an assertion if we go off the end of the block. + * @return {!jspb.BinaryReader} + */ +jspb.BinaryReader.prototype.alloc = jspb.BinaryReader.alloc; + + +/** + * Puts this instance back in the instance cache. + */ +jspb.BinaryReader.prototype.free = function() { + this.decoder_.clear(); + this.nextField_ = jspb.BinaryConstants.INVALID_FIELD_NUMBER; + this.nextWireType_ = jspb.BinaryConstants.WireType.INVALID; + this.error_ = false; + this.readCallbacks_ = null; + + if (jspb.BinaryReader.instanceCache_.length < 100) { + jspb.BinaryReader.instanceCache_.push(this); + } +}; + + +/** + * Returns the cursor immediately before the current field's tag. + * @return {number} The internal read cursor. + */ +jspb.BinaryReader.prototype.getFieldCursor = function() { + return this.fieldCursor_; +}; + + +/** + * Returns the internal read cursor. + * @return {number} The internal read cursor. + */ +jspb.BinaryReader.prototype.getCursor = function() { + return this.decoder_.getCursor(); +}; + + +/** + * Returns the raw buffer. + * @return {Uint8Array} The raw buffer. + */ +jspb.BinaryReader.prototype.getBuffer = function() { + return this.decoder_.getBuffer(); +}; + + +/** + * @return {number} The field number of the next field in the buffer, or + * INVALID_FIELD_NUMBER if there is no next field. + */ +jspb.BinaryReader.prototype.getFieldNumber = function() { + return this.nextField_; +}; + + +/** + * @return {jspb.BinaryConstants.WireType} The wire type of the next field + * in the stream, or WireType.INVALID if there is no next field. + */ +jspb.BinaryReader.prototype.getWireType = function() { + return this.nextWireType_; +}; + + +/** + * @return {boolean} Whether the current wire type is an end-group tag. Used as + * an exit condition in decoder loops in generated code. + */ +jspb.BinaryReader.prototype.isEndGroup = function() { + return this.nextWireType_ == jspb.BinaryConstants.WireType.END_GROUP; +}; + + +/** + * Returns true if this reader hit an error due to corrupt data. + * @return {boolean} + */ +jspb.BinaryReader.prototype.getError = function() { + return this.error_ || this.decoder_.getError(); +}; + + +/** + * Points this reader at a new block of bytes. + * @param {!Uint8Array} bytes The block of bytes we're reading from. + * @param {number} start The offset to start reading at. + * @param {number} length The length of the block to read. + */ +jspb.BinaryReader.prototype.setBlock = function(bytes, start, length) { + this.decoder_.setBlock(bytes, start, length); + this.nextField_ = jspb.BinaryConstants.INVALID_FIELD_NUMBER; + this.nextWireType_ = jspb.BinaryConstants.WireType.INVALID; +}; + + +/** + * Rewinds the stream cursor to the beginning of the buffer and resets all + * internal state. + */ +jspb.BinaryReader.prototype.reset = function() { + this.decoder_.reset(); + this.nextField_ = jspb.BinaryConstants.INVALID_FIELD_NUMBER; + this.nextWireType_ = jspb.BinaryConstants.WireType.INVALID; +}; + + +/** + * Advances the stream cursor by the given number of bytes. + * @param {number} count The number of bytes to advance by. + */ +jspb.BinaryReader.prototype.advance = function(count) { + this.decoder_.advance(count); +}; + + +/** + * Reads the next field header in the stream if there is one, returns true if + * we saw a valid field header or false if we've read the whole stream. + * Throws an error if we encountered a deprecated START_GROUP/END_GROUP field. + * @return {boolean} True if the stream contains more fields. + */ +jspb.BinaryReader.prototype.nextField = function() { + // If we're at the end of the block, there are no more fields. + if (this.decoder_.atEnd()) { + return false; + } + + // If we hit an error decoding the previous field, stop now before we + // try to decode anything else + if (this.getError()) { + goog.asserts.fail('Decoder hit an error'); + return false; + } + + // Otherwise just read the header of the next field. + this.fieldCursor_ = this.decoder_.getCursor(); + var header = this.decoder_.readUnsignedVarint32(); + + var nextField = header >>> 3; + var nextWireType = /** @type {jspb.BinaryConstants.WireType} */ + (header & 0x7); + + // If the wire type isn't one of the valid ones, something's broken. + if (nextWireType != jspb.BinaryConstants.WireType.VARINT && + nextWireType != jspb.BinaryConstants.WireType.FIXED32 && + nextWireType != jspb.BinaryConstants.WireType.FIXED64 && + nextWireType != jspb.BinaryConstants.WireType.DELIMITED && + nextWireType != jspb.BinaryConstants.WireType.START_GROUP && + nextWireType != jspb.BinaryConstants.WireType.END_GROUP) { + goog.asserts.fail('Invalid wire type'); + this.error_ = true; + return false; + } + + this.nextField_ = nextField; + this.nextWireType_ = nextWireType; + + return true; +}; + + +/** + * Winds the reader back to just before this field's header. + */ +jspb.BinaryReader.prototype.unskipHeader = function() { + this.decoder_.unskipVarint((this.nextField_ << 3) | this.nextWireType_); +}; + + +/** + * Skips all contiguous fields whose header matches the one we just read. + */ +jspb.BinaryReader.prototype.skipMatchingFields = function() { + var field = this.nextField_; + this.unskipHeader(); + + while (this.nextField() && (this.getFieldNumber() == field)) { + this.skipField(); + } + + if (!this.decoder_.atEnd()) { + this.unskipHeader(); + } +}; + + +/** + * Skips over the next varint field in the binary stream. + */ +jspb.BinaryReader.prototype.skipVarintField = function() { + if (this.nextWireType_ != jspb.BinaryConstants.WireType.VARINT) { + goog.asserts.fail('Invalid wire type for skipVarintField'); + this.skipField(); + return; + } + + this.decoder_.skipVarint(); +}; + + +/** + * Skips over the next delimited field in the binary stream. + */ +jspb.BinaryReader.prototype.skipDelimitedField = function() { + if (this.nextWireType_ != jspb.BinaryConstants.WireType.DELIMITED) { + goog.asserts.fail('Invalid wire type for skipDelimitedField'); + this.skipField(); + return; + } + + var length = this.decoder_.readUnsignedVarint32(); + this.decoder_.advance(length); +}; + + +/** + * Skips over the next fixed32 field in the binary stream. + */ +jspb.BinaryReader.prototype.skipFixed32Field = function() { + if (this.nextWireType_ != jspb.BinaryConstants.WireType.FIXED32) { + goog.asserts.fail('Invalid wire type for skipFixed32Field'); + this.skipField(); + return; + } + + this.decoder_.advance(4); +}; + + +/** + * Skips over the next fixed64 field in the binary stream. + */ +jspb.BinaryReader.prototype.skipFixed64Field = function() { + if (this.nextWireType_ != jspb.BinaryConstants.WireType.FIXED64) { + goog.asserts.fail('Invalid wire type for skipFixed64Field'); + this.skipField(); + return; + } + + this.decoder_.advance(8); +}; + + +/** + * Skips over the next group field in the binary stream. + */ +jspb.BinaryReader.prototype.skipGroup = function() { + // Keep a stack of start-group tags that must be matched by end-group tags. + var nestedGroups = [this.nextField_]; + do { + if (!this.nextField()) { + goog.asserts.fail('Unmatched start-group tag: stream EOF'); + this.error_ = true; + return; + } + if (this.nextWireType_ == + jspb.BinaryConstants.WireType.START_GROUP) { + // Nested group start. + nestedGroups.push(this.nextField_); + } else if (this.nextWireType_ == + jspb.BinaryConstants.WireType.END_GROUP) { + // Group end: check that it matches top-of-stack. + if (this.nextField_ != nestedGroups.pop()) { + goog.asserts.fail('Unmatched end-group tag'); + this.error_ = true; + return; + } + } + } while (nestedGroups.length > 0); +}; + + +/** + * Skips over the next field in the binary stream - this is useful if we're + * decoding a message that contain unknown fields. + */ +jspb.BinaryReader.prototype.skipField = function() { + switch (this.nextWireType_) { + case jspb.BinaryConstants.WireType.VARINT: + this.skipVarintField(); + break; + case jspb.BinaryConstants.WireType.FIXED64: + this.skipFixed64Field(); + break; + case jspb.BinaryConstants.WireType.DELIMITED: + this.skipDelimitedField(); + break; + case jspb.BinaryConstants.WireType.FIXED32: + this.skipFixed32Field(); + break; + case jspb.BinaryConstants.WireType.START_GROUP: + this.skipGroup(); + break; + default: + goog.asserts.fail('Invalid wire encoding for field.'); + } +}; + + +/** + * Registers a user-defined read callback. + * @param {string} callbackName + * @param {function(!jspb.BinaryReader):*} callback + */ +jspb.BinaryReader.prototype.registerReadCallback = + function(callbackName, callback) { + if (goog.isNull(this.readCallbacks_)) { + this.readCallbacks_ = {}; + } + goog.asserts.assert(!this.readCallbacks_[callbackName]); + this.readCallbacks_[callbackName] = callback; +}; + + +/** + * Runs a registered read callback. + * @param {string} callbackName The name the callback is registered under. + * @return {*} The value returned by the callback. + */ +jspb.BinaryReader.prototype.runReadCallback = function(callbackName) { + goog.asserts.assert(!goog.isNull(this.readCallbacks_)); + var callback = this.readCallbacks_[callbackName]; + goog.asserts.assert(callback); + return callback(this); +}; + + +/** + * Reads a field of any valid non-message type from the binary stream. + * @param {jspb.BinaryConstants.FieldType} fieldType + * @return {jspb.AnyFieldType} + */ +jspb.BinaryReader.prototype.readAny = function(fieldType) { + this.nextWireType_ = jspb.BinaryConstants.FieldTypeToWireType(fieldType); + var fieldTypes = jspb.BinaryConstants.FieldType; + switch (fieldType) { + case fieldTypes.DOUBLE: + return this.readDouble(); + case fieldTypes.FLOAT: + return this.readFloat(); + case fieldTypes.INT64: + return this.readInt64(); + case fieldTypes.UINT64: + return this.readUint64(); + case fieldTypes.INT32: + return this.readInt32(); + case fieldTypes.FIXED64: + return this.readFixed64(); + case fieldTypes.FIXED32: + return this.readFixed32(); + case fieldTypes.BOOL: + return this.readBool(); + case fieldTypes.STRING: + return this.readString(); + case fieldTypes.GROUP: + goog.asserts.fail('Group field type not supported in readAny()'); + case fieldTypes.MESSAGE: + goog.asserts.fail('Message field type not supported in readAny()'); + case fieldTypes.BYTES: + return this.readBytes(); + case fieldTypes.UINT32: + return this.readUint32(); + case fieldTypes.ENUM: + return this.readEnum(); + case fieldTypes.SFIXED32: + return this.readSfixed32(); + case fieldTypes.SFIXED64: + return this.readSfixed64(); + case fieldTypes.SINT32: + return this.readSint32(); + case fieldTypes.SINT64: + return this.readSint64(); + case fieldTypes.FHASH64: + return this.readFixedHash64(); + case fieldTypes.VHASH64: + return this.readVarintHash64(); + default: + goog.asserts.fail('Invalid field type in readAny()'); + } + return 0; +}; + + +/** + * Deserialize a proto into the provided message object using the provided + * reader function. This function is templated as we currently have one client + * who is using manual deserialization instead of the code-generated versions. + * @template T + * @param {T} message + * @param {function(T, !jspb.BinaryReader)} reader + */ +jspb.BinaryReader.prototype.readMessage = function(message, reader) { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED); + + // Save the current endpoint of the decoder and move it to the end of the + // embedded message. + var oldEnd = this.decoder_.getEnd(); + var length = this.decoder_.readUnsignedVarint32(); + var newEnd = this.decoder_.getCursor() + length; + this.decoder_.setEnd(newEnd); + + // Deserialize the embedded message. + reader(message, this); + + // Advance the decoder past the embedded message and restore the endpoint. + this.decoder_.setCursor(newEnd); + this.decoder_.setEnd(oldEnd); +}; + + +/** + * Deserialize a proto into the provided message object using the provided + * reader function, assuming that the message is serialized as a group + * with the given tag. + * @template T + * @param {number} field + * @param {T} message + * @param {function(T, !jspb.BinaryReader)} reader + */ +jspb.BinaryReader.prototype.readGroup = + function(field, message, reader) { + // Ensure that the wire type is correct. + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.START_GROUP); + // Ensure that the field number is correct. + goog.asserts.assert(this.nextField_ == field); + + // Deserialize the message. The deserialization will stop at an END_GROUP tag. + reader(message, this); + + if (!this.error_ && + this.nextWireType_ != jspb.BinaryConstants.WireType.END_GROUP) { + goog.asserts.fail('Group submessage did not end with an END_GROUP tag'); + this.error_ = true; + } +}; + + +/** + * Return a decoder that wraps the current delimited field. + * @return {!jspb.BinaryDecoder} + */ +jspb.BinaryReader.prototype.getFieldDecoder = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED); + + var length = this.decoder_.readUnsignedVarint32(); + var start = this.decoder_.getCursor(); + var end = start + length; + + var innerDecoder = jspb.BinaryDecoder.alloc(this.decoder_.getBuffer(), + start, length); + this.decoder_.setCursor(end); + return innerDecoder; +}; + + +/** + * Reads a signed 32-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * @return {number} The value of the signed 32-bit integer field. + */ +jspb.BinaryReader.prototype.readInt32 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readSignedVarint32(); +}; + + +/** + * Reads a signed 32-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * Returns the value as a string. + * + * @return {string} The value of the signed 32-bit integer field as a decimal + * string. + */ +jspb.BinaryReader.prototype.readInt32String = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readSignedVarint32String(); +}; + + +/** + * Reads a signed 64-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * @return {number} The value of the signed 64-bit integer field. + */ +jspb.BinaryReader.prototype.readInt64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readSignedVarint64(); +}; + + +/** + * Reads a signed 64-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * Returns the value as a string. + * + * @return {string} The value of the signed 64-bit integer field as a decimal + * string. + */ +jspb.BinaryReader.prototype.readInt64String = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readSignedVarint64String(); +}; + + +/** + * Reads an unsigned 32-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * @return {number} The value of the unsigned 32-bit integer field. + */ +jspb.BinaryReader.prototype.readUint32 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readUnsignedVarint32(); +}; + + +/** + * Reads an unsigned 32-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * Returns the value as a string. + * + * @return {string} The value of the unsigned 32-bit integer field as a decimal + * string. + */ +jspb.BinaryReader.prototype.readUint32String = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readUnsignedVarint32String(); +}; + + +/** + * Reads an unsigned 64-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * @return {number} The value of the unsigned 64-bit integer field. + */ +jspb.BinaryReader.prototype.readUint64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readUnsignedVarint64(); +}; + + +/** + * Reads an unsigned 64-bit integer field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * Returns the value as a string. + * + * @return {string} The value of the unsigned 64-bit integer field as a decimal + * string. + */ +jspb.BinaryReader.prototype.readUint64String = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readUnsignedVarint64String(); +}; + + +/** + * Reads a signed zigzag-encoded 32-bit integer field from the binary stream, + * or throws an error if the next field in the stream is not of the correct + * wire type. + * + * @return {number} The value of the signed 32-bit integer field. + */ +jspb.BinaryReader.prototype.readSint32 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readZigzagVarint32(); +}; + + +/** + * Reads a signed zigzag-encoded 64-bit integer field from the binary stream, + * or throws an error if the next field in the stream is not of the correct + * wire type. + * + * @return {number} The value of the signed 64-bit integer field. + */ +jspb.BinaryReader.prototype.readSint64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readZigzagVarint64(); +}; + + +/** + * Reads an unsigned 32-bit fixed-length integer fiield from the binary stream, + * or throws an error if the next field in the stream is not of the correct + * wire type. + * + * @return {number} The value of the double field. + */ +jspb.BinaryReader.prototype.readFixed32 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED32); + return this.decoder_.readUint32(); +}; + + +/** + * Reads an unsigned 64-bit fixed-length integer fiield from the binary stream, + * or throws an error if the next field in the stream is not of the correct + * wire type. + * + * @return {number} The value of the float field. + */ +jspb.BinaryReader.prototype.readFixed64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED64); + return this.decoder_.readUint64(); +}; + + +/** + * Reads a signed 32-bit fixed-length integer fiield from the binary stream, or + * throws an error if the next field in the stream is not of the correct wire + * type. + * + * @return {number} The value of the double field. + */ +jspb.BinaryReader.prototype.readSfixed32 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED32); + return this.decoder_.readInt32(); +}; + + +/** + * Reads a signed 64-bit fixed-length integer fiield from the binary stream, or + * throws an error if the next field in the stream is not of the correct wire + * type. + * + * @return {number} The value of the float field. + */ +jspb.BinaryReader.prototype.readSfixed64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED64); + return this.decoder_.readInt64(); +}; + + +/** + * Reads a 32-bit floating-point field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * @return {number} The value of the float field. + */ +jspb.BinaryReader.prototype.readFloat = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED32); + return this.decoder_.readFloat(); +}; + + +/** + * Reads a 64-bit floating-point field from the binary stream, or throws an + * error if the next field in the stream is not of the correct wire type. + * + * @return {number} The value of the double field. + */ +jspb.BinaryReader.prototype.readDouble = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED64); + return this.decoder_.readDouble(); +}; + + +/** + * Reads a boolean field from the binary stream, or throws an error if the next + * field in the stream is not of the correct wire type. + * + * @return {boolean} The value of the boolean field. + */ +jspb.BinaryReader.prototype.readBool = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return !!this.decoder_.readUnsignedVarint32(); +}; + + +/** + * Reads an enum field from the binary stream, or throws an error if the next + * field in the stream is not of the correct wire type. + * + * @return {number} The value of the enum field. + */ +jspb.BinaryReader.prototype.readEnum = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readSignedVarint64(); +}; + + +/** + * Reads a string field from the binary stream, or throws an error if the next + * field in the stream is not of the correct wire type. + * + * @return {string} The value of the string field. + */ +jspb.BinaryReader.prototype.readString = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED); + var length = this.decoder_.readUnsignedVarint32(); + return this.decoder_.readString(length); +}; + + +/** + * Reads a length-prefixed block of bytes from the binary stream, or returns + * null if the next field in the stream has an invalid length value. + * + * @return {Uint8Array} The block of bytes. + */ +jspb.BinaryReader.prototype.readBytes = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED); + var length = this.decoder_.readUnsignedVarint32(); + return this.decoder_.readBytes(length); +}; + + +/** + * Reads a 64-bit varint or fixed64 field from the stream and returns it as a + * 8-character Unicode string for use as a hash table key, or throws an error + * if the next field in the stream is not of the correct wire type. + * + * @return {string} The hash value. + */ +jspb.BinaryReader.prototype.readVarintHash64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.VARINT); + return this.decoder_.readVarintHash64(); +}; + + +/** + * Reads a 64-bit varint or fixed64 field from the stream and returns it as a + * 8-character Unicode string for use as a hash table key, or throws an error + * if the next field in the stream is not of the correct wire type. + * + * @return {string} The hash value. + */ +jspb.BinaryReader.prototype.readFixedHash64 = function() { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.FIXED64); + return this.decoder_.readFixedHash64(); +}; + + +/** + * Reads a packed scalar field using the supplied raw reader function. + * @param {function()} decodeMethod + * @return {!Array} + * @private + */ +jspb.BinaryReader.prototype.readPackedField_ = function(decodeMethod) { + goog.asserts.assert( + this.nextWireType_ == jspb.BinaryConstants.WireType.DELIMITED); + var length = this.decoder_.readUnsignedVarint32(); + var end = this.decoder_.getCursor() + length; + var result = []; + while (this.decoder_.getCursor() < end) { + // TODO(aappleby): .call is slow + result.push(decodeMethod.call(this.decoder_)); + } + return result; +}; + + +/** + * Reads a packed int32 field, which consists of a length header and a list of + * signed varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedInt32 = function() { + return this.readPackedField_(this.decoder_.readSignedVarint32); +}; + + +/** + * Reads a packed int32 field, which consists of a length header and a list of + * signed varints. Returns a list of strings. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedInt32String = function() { + return this.readPackedField_(this.decoder_.readSignedVarint32String); +}; + + +/** + * Reads a packed int64 field, which consists of a length header and a list of + * signed varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedInt64 = function() { + return this.readPackedField_(this.decoder_.readSignedVarint64); +}; + + +/** + * Reads a packed int64 field, which consists of a length header and a list of + * signed varints. Returns a list of strings. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedInt64String = function() { + return this.readPackedField_(this.decoder_.readSignedVarint64String); +}; + + +/** + * Reads a packed uint32 field, which consists of a length header and a list of + * unsigned varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedUint32 = function() { + return this.readPackedField_(this.decoder_.readUnsignedVarint32); +}; + + +/** + * Reads a packed uint32 field, which consists of a length header and a list of + * unsigned varints. Returns a list of strings. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedUint32String = function() { + return this.readPackedField_(this.decoder_.readUnsignedVarint32String); +}; + + +/** + * Reads a packed uint64 field, which consists of a length header and a list of + * unsigned varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedUint64 = function() { + return this.readPackedField_(this.decoder_.readUnsignedVarint64); +}; + + +/** + * Reads a packed uint64 field, which consists of a length header and a list of + * unsigned varints. Returns a list of strings. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedUint64String = function() { + return this.readPackedField_(this.decoder_.readUnsignedVarint64String); +}; + + +/** + * Reads a packed sint32 field, which consists of a length header and a list of + * zigzag varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedSint32 = function() { + return this.readPackedField_(this.decoder_.readZigzagVarint32); +}; + + +/** + * Reads a packed sint64 field, which consists of a length header and a list of + * zigzag varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedSint64 = function() { + return this.readPackedField_(this.decoder_.readZigzagVarint64); +}; + + +/** + * Reads a packed fixed32 field, which consists of a length header and a list + * of unsigned 32-bit ints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedFixed32 = function() { + return this.readPackedField_(this.decoder_.readUint32); +}; + + +/** + * Reads a packed fixed64 field, which consists of a length header and a list + * of unsigned 64-bit ints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedFixed64 = function() { + return this.readPackedField_(this.decoder_.readUint64); +}; + + +/** + * Reads a packed sfixed32 field, which consists of a length header and a list + * of 32-bit ints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedSfixed32 = function() { + return this.readPackedField_(this.decoder_.readInt32); +}; + + +/** + * Reads a packed sfixed64 field, which consists of a length header and a list + * of 64-bit ints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedSfixed64 = function() { + return this.readPackedField_(this.decoder_.readInt64); +}; + + +/** + * Reads a packed float field, which consists of a length header and a list of + * floats. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedFloat = function() { + return this.readPackedField_(this.decoder_.readFloat); +}; + + +/** + * Reads a packed double field, which consists of a length header and a list of + * doubles. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedDouble = function() { + return this.readPackedField_(this.decoder_.readDouble); +}; + + +/** + * Reads a packed bool field, which consists of a length header and a list of + * unsigned varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedBool = function() { + return this.readPackedField_(this.decoder_.readBool); +}; + + +/** + * Reads a packed enum field, which consists of a length header and a list of + * unsigned varints. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedEnum = function() { + return this.readPackedField_(this.decoder_.readEnum); +}; + + +/** + * Reads a packed varint hash64 field, which consists of a length header and a + * list of varint hash64s. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedVarintHash64 = function() { + return this.readPackedField_(this.decoder_.readVarintHash64); +}; + + +/** + * Reads a packed fixed hash64 field, which consists of a length header and a + * list of fixed hash64s. + * @return {!Array.} + */ +jspb.BinaryReader.prototype.readPackedFixedHash64 = function() { + return this.readPackedField_(this.decoder_.readFixedHash64); +}; diff --git a/js/binary/reader_test.js b/js/binary/reader_test.js new file mode 100644 index 00000000..a6482610 --- /dev/null +++ b/js/binary/reader_test.js @@ -0,0 +1,889 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Test cases for jspb's binary protocol buffer reader. + * + * There are two particular magic numbers that need to be pointed out - + * 2^64-1025 is the largest number representable as both a double and an + * unsigned 64-bit integer, and 2^63-513 is the largest number representable as + * both a double and a signed 64-bit integer. + * + * Test suite is written using Jasmine -- see http://jasmine.github.io/ + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.require('goog.testing.asserts'); +goog.require('jspb.BinaryConstants'); +goog.require('jspb.BinaryDecoder'); +goog.require('jspb.BinaryReader'); +goog.require('jspb.BinaryWriter'); + + + +describe('binaryReaderTest', function() { + /** + * Tests the reader instance cache. + * @suppress {visibility} + */ + it('testInstanceCaches', function() { + var writer = new jspb.BinaryWriter(); + var dummyMessage = /** @type {!jspb.BinaryMessage} */({}); + writer.writeMessage(1, dummyMessage, goog.nullFunction); + writer.writeMessage(2, dummyMessage, goog.nullFunction); + + var buffer = writer.getResultBuffer(); + + // Empty the instance caches. + jspb.BinaryReader.instanceCache_ = []; + + // Allocating and then freeing three decoders should leave us with three in + // the cache. + + var decoder1 = jspb.BinaryDecoder.alloc(); + var decoder2 = jspb.BinaryDecoder.alloc(); + var decoder3 = jspb.BinaryDecoder.alloc(); + decoder1.free(); + decoder2.free(); + decoder3.free(); + + assertEquals(3, jspb.BinaryDecoder.instanceCache_.length); + assertEquals(0, jspb.BinaryReader.instanceCache_.length); + + // Allocating and then freeing a reader should remove one decoder from its + // cache, but it should stay stuck to the reader afterwards since we can't + // have a reader without a decoder. + jspb.BinaryReader.alloc().free(); + + assertEquals(2, jspb.BinaryDecoder.instanceCache_.length); + assertEquals(1, jspb.BinaryReader.instanceCache_.length); + + // Allocating a reader should remove a reader from the cache. + var reader = jspb.BinaryReader.alloc(buffer); + + assertEquals(2, jspb.BinaryDecoder.instanceCache_.length); + assertEquals(0, jspb.BinaryReader.instanceCache_.length); + + // Processing the message reuses the current reader. + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + reader.readMessage(dummyMessage, function() { + assertEquals(0, jspb.BinaryReader.instanceCache_.length); + }); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + reader.readMessage(dummyMessage, function() { + assertEquals(0, jspb.BinaryReader.instanceCache_.length); + }); + + assertEquals(false, reader.nextField()); + + assertEquals(2, jspb.BinaryDecoder.instanceCache_.length); + assertEquals(0, jspb.BinaryReader.instanceCache_.length); + + // Freeing the reader should put it back into the cache. + reader.free(); + + assertEquals(2, jspb.BinaryDecoder.instanceCache_.length); + assertEquals(1, jspb.BinaryReader.instanceCache_.length); + }); + + + /** + * @param {number} x + * @return {number} + */ + function truncate(x) { + var temp = new Float32Array(1); + temp[0] = x; + return temp[0]; + } + + + /** + * Verifies that misuse of the reader class triggers assertions. + * @suppress {checkTypes|visibility} + */ + it('testReadErrors', function() { + // Calling readMessage on a non-delimited field should trigger an + // assertion. + var reader = jspb.BinaryReader.alloc([8, 1]); + var dummyMessage = /** @type {!jspb.BinaryMessage} */({}); + reader.nextField(); + assertThrows(function() { + reader.readMessage(dummyMessage, goog.nullFunction); + }); + + // Reading past the end of the stream should trigger an assertion. + reader = jspb.BinaryReader.alloc([9, 1]); + reader.nextField(); + assertThrows(function() {reader.readFixed64()}); + + // Reading past the end of a submessage should trigger an assertion. + reader = jspb.BinaryReader.alloc([10, 4, 13, 1, 1, 1]); + reader.nextField(); + reader.readMessage(dummyMessage, function() { + reader.nextField(); + assertThrows(function() {reader.readFixed32()}); + }); + + // Skipping an invalid field should trigger an assertion. + reader = jspb.BinaryReader.alloc([12, 1]); + reader.nextWireType_ = 1000; + assertThrows(function() {reader.skipField()}); + + // Reading fields with the wrong wire type should assert. + reader = jspb.BinaryReader.alloc([9, 0, 0, 0, 0, 0, 0, 0, 0]); + reader.nextField(); + assertThrows(function() {reader.readInt32()}); + assertThrows(function() {reader.readInt32String()}); + assertThrows(function() {reader.readInt64()}); + assertThrows(function() {reader.readInt64String()}); + assertThrows(function() {reader.readUint32()}); + assertThrows(function() {reader.readUint32String()}); + assertThrows(function() {reader.readUint64()}); + assertThrows(function() {reader.readUint64String()}); + assertThrows(function() {reader.readSint32()}); + assertThrows(function() {reader.readBool()}); + assertThrows(function() {reader.readEnum()}); + + reader = jspb.BinaryReader.alloc([8, 1]); + reader.nextField(); + assertThrows(function() {reader.readFixed32()}); + assertThrows(function() {reader.readFixed64()}); + assertThrows(function() {reader.readSfixed32()}); + assertThrows(function() {reader.readSfixed64()}); + assertThrows(function() {reader.readFloat()}); + assertThrows(function() {reader.readDouble()}); + + assertThrows(function() {reader.readString()}); + assertThrows(function() {reader.readBytes()}); + }); + + + /** + * Tests encoding and decoding of unsigned field types. + * @param {Function} readField + * @param {Function} writeField + * @param {number} epsilon + * @param {number} upperLimit + * @param {Function} filter + * @private + * @suppress {missingProperties} + */ + function doTestUnsignedField_(readField, + writeField, epsilon, upperLimit, filter) { + assertNotNull(readField); + assertNotNull(writeField); + + var writer = new jspb.BinaryWriter(); + + // Encode zero and limits. + writeField.call(writer, 1, filter(0)); + writeField.call(writer, 2, filter(epsilon)); + writeField.call(writer, 3, filter(upperLimit)); + + // Encode positive values. + for (var cursor = epsilon; cursor < upperLimit; cursor *= 1.1) { + writeField.call(writer, 4, filter(cursor)); + } + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + // Check zero and limits. + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + assertEquals(filter(0), readField.call(reader)); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + assertEquals(filter(epsilon), readField.call(reader)); + + reader.nextField(); + assertEquals(3, reader.getFieldNumber()); + assertEquals(filter(upperLimit), readField.call(reader)); + + // Check positive values. + for (var cursor = epsilon; cursor < upperLimit; cursor *= 1.1) { + reader.nextField(); + if (4 != reader.getFieldNumber()) throw 'fail!'; + if (filter(cursor) != readField.call(reader)) throw 'fail!'; + } + }; + + + /** + * Tests encoding and decoding of signed field types. + * @param {Function} readField + * @param {Function} writeField + * @param {number} epsilon + * @param {number} lowerLimit + * @param {number} upperLimit + * @param {Function} filter + * @private + * @suppress {missingProperties} + */ + function doTestSignedField_(readField, + writeField, epsilon, lowerLimit, upperLimit, filter) { + var writer = new jspb.BinaryWriter(); + + // Encode zero and limits. + writeField.call(writer, 1, filter(lowerLimit)); + writeField.call(writer, 2, filter(-epsilon)); + writeField.call(writer, 3, filter(0)); + writeField.call(writer, 4, filter(epsilon)); + writeField.call(writer, 5, filter(upperLimit)); + + var inputValues = []; + + // Encode negative values. + for (var cursor = lowerLimit; cursor < -epsilon; cursor /= 1.1) { + var val = filter(cursor); + writeField.call(writer, 6, val); + inputValues.push({ + fieldNumber: 6, + value: val + }); + } + + // Encode positive values. + for (var cursor = epsilon; cursor < upperLimit; cursor *= 1.1) { + var val = filter(cursor); + writeField.call(writer, 7, val); + inputValues.push({ + fieldNumber: 7, + value: val + }); + } + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + // Check zero and limits. + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + assertEquals(filter(lowerLimit), readField.call(reader)); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + assertEquals(filter(-epsilon), readField.call(reader)); + + reader.nextField(); + assertEquals(3, reader.getFieldNumber()); + assertEquals(filter(0), readField.call(reader)); + + reader.nextField(); + assertEquals(4, reader.getFieldNumber()); + assertEquals(filter(epsilon), readField.call(reader)); + + reader.nextField(); + assertEquals(5, reader.getFieldNumber()); + assertEquals(filter(upperLimit), readField.call(reader)); + + for (var i = 0; i < inputValues.length; i++) { + var expected = inputValues[i]; + reader.nextField(); + assertEquals(expected.fieldNumber, reader.getFieldNumber()); + assertEquals(expected.value, readField.call(reader)); + } + }; + + + /** + * Tests fields that use varint encoding. + */ + it('testVarintFields', function() { + assertNotNull(jspb.BinaryReader.prototype.readUint32); + assertNotNull(jspb.BinaryReader.prototype.writeUint32); + assertNotNull(jspb.BinaryReader.prototype.readUint64); + assertNotNull(jspb.BinaryReader.prototype.writeUint64); + assertNotNull(jspb.BinaryReader.prototype.readBool); + assertNotNull(jspb.BinaryReader.prototype.writeBool); + doTestUnsignedField_( + jspb.BinaryReader.prototype.readUint32, + jspb.BinaryWriter.prototype.writeUint32, + 1, Math.pow(2, 32) - 1, Math.round); + + doTestUnsignedField_( + jspb.BinaryReader.prototype.readUint64, + jspb.BinaryWriter.prototype.writeUint64, + 1, Math.pow(2, 64) - 1025, Math.round); + + doTestSignedField_( + jspb.BinaryReader.prototype.readInt32, + jspb.BinaryWriter.prototype.writeInt32, + 1, -Math.pow(2, 31), Math.pow(2, 31) - 1, Math.round); + + doTestSignedField_( + jspb.BinaryReader.prototype.readInt64, + jspb.BinaryWriter.prototype.writeInt64, + 1, -Math.pow(2, 63), Math.pow(2, 63) - 513, Math.round); + + doTestSignedField_( + jspb.BinaryReader.prototype.readEnum, + jspb.BinaryWriter.prototype.writeEnum, + 1, -Math.pow(2, 31), Math.pow(2, 31) - 1, Math.round); + + doTestUnsignedField_( + jspb.BinaryReader.prototype.readBool, + jspb.BinaryWriter.prototype.writeBool, + 1, 1, function(x) { return !!x; }); + }); + + + /** + * Tests 64-bit fields that are handled as strings. + */ + it('testStringInt64Fields', function() { + var writer = new jspb.BinaryWriter(); + + var testSignedData = [ + '2730538252207801776', + '-2688470994844604560', + '3398529779486536359', + '3568577411627971000', + '272477188847484900', + '-6649058714086158188', + '-7695254765712060806', + '-4525541438037104029', + '-4993706538836508568', + '4990160321893729138' + ]; + var testUnsignedData = [ + '7822732630241694882', + '6753602971916687352', + '2399935075244442116', + '8724292567325338867', + '16948784802625696584', + '4136275908516066934', + '3575388346793700364', + '5167142028379259461', + '1557573948689737699', + '17100725280812548567' + ]; + + for (var i = 0; i < testSignedData.length; i++) { + writer.writeInt64String(2 * i + 1, testSignedData[i]); + writer.writeUint64String(2 * i + 2, testUnsignedData[i]); + } + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + for (var i = 0; i < testSignedData.length; i++) { + reader.nextField(); + assertEquals(2 * i + 1, reader.getFieldNumber()); + assertEquals(testSignedData[i], reader.readInt64String()); + reader.nextField(); + assertEquals(2 * i + 2, reader.getFieldNumber()); + assertEquals(testUnsignedData[i], reader.readUint64String()); + } + }); + + + /** + * Tests fields that use zigzag encoding. + */ + it('testZigzagFields', function() { + doTestSignedField_( + jspb.BinaryReader.prototype.readSint32, + jspb.BinaryWriter.prototype.writeSint32, + 1, -Math.pow(2, 31), Math.pow(2, 31) - 1, Math.round); + + doTestSignedField_( + jspb.BinaryReader.prototype.readSint64, + jspb.BinaryWriter.prototype.writeSint64, + 1, -Math.pow(2, 63), Math.pow(2, 63) - 513, Math.round); + }); + + + /** + * Tests fields that use fixed-length encoding. + */ + it('testFixedFields', function() { + doTestUnsignedField_( + jspb.BinaryReader.prototype.readFixed32, + jspb.BinaryWriter.prototype.writeFixed32, + 1, Math.pow(2, 32) - 1, Math.round); + + doTestUnsignedField_( + jspb.BinaryReader.prototype.readFixed64, + jspb.BinaryWriter.prototype.writeFixed64, + 1, Math.pow(2, 64) - 1025, Math.round); + + doTestSignedField_( + jspb.BinaryReader.prototype.readSfixed32, + jspb.BinaryWriter.prototype.writeSfixed32, + 1, -Math.pow(2, 31), Math.pow(2, 31) - 1, Math.round); + + doTestSignedField_( + jspb.BinaryReader.prototype.readSfixed64, + jspb.BinaryWriter.prototype.writeSfixed64, + 1, -Math.pow(2, 63), Math.pow(2, 63) - 513, Math.round); + }); + + + /** + * Tests floating point fields. + */ + it('testFloatFields', function() { + doTestSignedField_( + jspb.BinaryReader.prototype.readFloat, + jspb.BinaryWriter.prototype.writeFloat, + jspb.BinaryConstants.FLOAT32_MIN, + -jspb.BinaryConstants.FLOAT32_MAX, + jspb.BinaryConstants.FLOAT32_MAX, + truncate); + + doTestSignedField_( + jspb.BinaryReader.prototype.readDouble, + jspb.BinaryWriter.prototype.writeDouble, + jspb.BinaryConstants.FLOAT64_EPS * 10, + -jspb.BinaryConstants.FLOAT64_MIN, + jspb.BinaryConstants.FLOAT64_MIN, + function(x) { return x; }); + }); + + + /** + * Tests length-delimited string fields. + */ + it('testStringFields', function() { + var s1 = 'The quick brown fox jumps over the lazy dog.'; + var s2 = '人人生而自由，在尊嚴和權利上一律平等。'; + + var writer = new jspb.BinaryWriter(); + + writer.writeString(1, s1); + writer.writeString(2, s2); + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + assertEquals(s1, reader.readString()); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + assertEquals(s2, reader.readString()); + }); + + + /** + * Tests length-delimited byte fields. + */ + it('testByteFields', function() { + var message = []; + var lowerLimit = 1; + var upperLimit = 256; + var scale = 1.1; + + var writer = new jspb.BinaryWriter(); + + for (var cursor = lowerLimit; cursor < upperLimit; cursor *= 1.1) { + var len = Math.round(cursor); + var bytes = []; + for (var i = 0; i < len; i++) bytes.push(i % 256); + + writer.writeBytes(len, bytes); + } + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + for (var cursor = lowerLimit; reader.nextField(); cursor *= 1.1) { + var len = Math.round(cursor); + if (len != reader.getFieldNumber()) throw 'fail!'; + + var bytes = reader.readBytes(); + if (len != bytes.length) throw 'fail!'; + for (var i = 0; i < bytes.length; i++) { + if (i % 256 != bytes[i]) throw 'fail!'; + } + } + }); + + + /** + * Tests nested messages. + */ + it('testNesting', function() { + var writer = new jspb.BinaryWriter(); + var dummyMessage = /** @type {!jspb.BinaryMessage} */({}); + + writer.writeInt32(1, 100); + + // Add one message with 3 int fields. + writer.writeMessage(2, dummyMessage, function() { + writer.writeInt32(3, 300); + writer.writeInt32(4, 400); + writer.writeInt32(5, 500); + }); + + // Add one empty message. + writer.writeMessage(6, dummyMessage, goog.nullFunction); + + writer.writeInt32(7, 700); + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + // Validate outermost message. + + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + assertEquals(100, reader.readInt32()); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + reader.readMessage(dummyMessage, function() { + // Validate embedded message 1. + reader.nextField(); + assertEquals(3, reader.getFieldNumber()); + assertEquals(300, reader.readInt32()); + + reader.nextField(); + assertEquals(4, reader.getFieldNumber()); + assertEquals(400, reader.readInt32()); + + reader.nextField(); + assertEquals(5, reader.getFieldNumber()); + assertEquals(500, reader.readInt32()); + + assertEquals(false, reader.nextField()); + }); + + reader.nextField(); + assertEquals(6, reader.getFieldNumber()); + reader.readMessage(dummyMessage, function() { + // Validate embedded message 2. + + assertEquals(false, reader.nextField()); + }); + + reader.nextField(); + assertEquals(7, reader.getFieldNumber()); + assertEquals(700, reader.readInt32()); + + assertEquals(false, reader.nextField()); + }); + + /** + * Tests skipping fields of each type by interleaving them with sentinel + * values and skipping everything that's not a sentinel. + */ + it('testSkipField', function() { + var writer = new jspb.BinaryWriter(); + + var sentinel = 123456789; + + // Write varint fields of different sizes. + writer.writeInt32(1, sentinel); + writer.writeInt32(1, 1); + writer.writeInt32(1, 1000); + writer.writeInt32(1, 1000000); + writer.writeInt32(1, 1000000000); + + // Write fixed 64-bit encoded fields. + writer.writeInt32(2, sentinel); + writer.writeDouble(2, 1); + writer.writeFixed64(2, 1); + writer.writeSfixed64(2, 1); + + // Write fixed 32-bit encoded fields. + writer.writeInt32(3, sentinel); + writer.writeFloat(3, 1); + writer.writeFixed32(3, 1); + writer.writeSfixed32(3, 1); + + // Write delimited fields. + writer.writeInt32(4, sentinel); + writer.writeBytes(4, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + writer.writeString(4, 'The quick brown fox jumps over the lazy dog'); + + // Write a group with a nested group inside. We use the internal + // .rawWriteVarint() to ensure the tested wire data is what we want, + // independently of any serialization logic. + writer.writeInt32(5, sentinel); + // Start group, field 5. + writer.rawWriteVarint( + (5 << 3) + jspb.BinaryConstants.WireType.START_GROUP); + // Varint, field 42. + writer.rawWriteVarint( + (42 << 3) + jspb.BinaryConstants.WireType.VARINT); + // Varint data. + writer.rawWriteVarint(42); + // Start group, field 6. + writer.rawWriteVarint( + (6 << 3) + jspb.BinaryConstants.WireType.START_GROUP); + // Varint, field 84. + writer.rawWriteVarint( + (84 << 3) + jspb.BinaryConstants.WireType.VARINT); + writer.rawWriteVarint(42); + // End group, field 6. + writer.rawWriteVarint( + (6 << 3) + jspb.BinaryConstants.WireType.END_GROUP); + // End group, field 5. + writer.rawWriteVarint( + (5 << 3) + jspb.BinaryConstants.WireType.END_GROUP); + + // Write final sentinel. + writer.writeInt32(6, sentinel); + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + function skip(field, count) { + for (var i = 0; i < count; i++) { + reader.nextField(); + if (field != reader.getFieldNumber()) throw 'fail!'; + reader.skipField(); + } + } + + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + assertEquals(sentinel, reader.readInt32()); + skip(1, 4); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + assertEquals(sentinel, reader.readInt32()); + skip(2, 3); + + reader.nextField(); + assertEquals(3, reader.getFieldNumber()); + assertEquals(sentinel, reader.readInt32()); + skip(3, 3); + + reader.nextField(); + assertEquals(4, reader.getFieldNumber()); + assertEquals(sentinel, reader.readInt32()); + skip(4, 2); + + reader.nextField(); + assertEquals(5, reader.getFieldNumber()); + assertEquals(sentinel, reader.readInt32()); + skip(5, 1); + + reader.nextField(); + assertEquals(6, reader.getFieldNumber()); + assertEquals(sentinel, reader.readInt32()); + }); + + + /** + * Tests packed fields. + */ + it('testPackedFields', function() { + var writer = new jspb.BinaryWriter(); + + var sentinel = 123456789; + + var unsignedData = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + var signedData = [-1, 2, -3, 4, -5, 6, -7, 8, -9, 10]; + var floatData = [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.10]; + var doubleData = [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.10]; + var boolData = [true, false, true, true, false, false, true, false]; + + for (var i = 0; i < floatData.length; i++) { + floatData[i] = truncate(floatData[i]); + } + + writer.writeInt32(1, sentinel); + + writer.writePackedInt32(2, signedData); + writer.writePackedInt64(2, signedData); + writer.writePackedUint32(2, unsignedData); + writer.writePackedUint64(2, unsignedData); + writer.writePackedSint32(2, signedData); + writer.writePackedSint64(2, signedData); + writer.writePackedFixed32(2, unsignedData); + writer.writePackedFixed64(2, unsignedData); + writer.writePackedSfixed32(2, signedData); + writer.writePackedSfixed64(2, signedData); + writer.writePackedFloat(2, floatData); + writer.writePackedDouble(2, doubleData); + writer.writePackedBool(2, boolData); + writer.writePackedEnum(2, unsignedData); + + writer.writeInt32(3, sentinel); + + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + reader.nextField(); + assertEquals(sentinel, reader.readInt32()); + + reader.nextField(); + assertElementsEquals(reader.readPackedInt32(), signedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedInt64(), signedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedUint32(), unsignedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedUint64(), unsignedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedSint32(), signedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedSint64(), signedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedFixed32(), unsignedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedFixed64(), unsignedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedSfixed32(), signedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedSfixed64(), signedData); + + reader.nextField(); + assertElementsEquals(reader.readPackedFloat(), floatData); + + reader.nextField(); + assertElementsEquals(reader.readPackedDouble(), doubleData); + + reader.nextField(); + assertElementsEquals(reader.readPackedBool(), boolData); + + reader.nextField(); + assertElementsEquals(reader.readPackedEnum(), unsignedData); + + reader.nextField(); + assertEquals(sentinel, reader.readInt32()); + }); + + + /** + * Byte blobs inside nested messages should always have their byte offset set + * relative to the start of the outermost blob, not the start of their parent + * blob. + */ + it('testNestedBlobs', function() { + // Create a proto consisting of two nested messages, with the inner one + // containing a blob of bytes. + + var fieldTag = (1 << 3) | jspb.BinaryConstants.WireType.DELIMITED; + var blob = [1, 2, 3, 4, 5]; + var writer = new jspb.BinaryWriter(); + var dummyMessage = /** @type {!jspb.BinaryMessage} */({}); + + writer.writeMessage(1, dummyMessage, function() { + writer.writeMessage(1, dummyMessage, function() { + writer.writeBytes(1, blob); + }); + }); + + // Peel off the outer two message layers. Each layer should have two bytes + // of overhead, one for the field tag and one for the length of the inner + // blob. + + var decoder1 = new jspb.BinaryDecoder(writer.getResultBuffer()); + assertEquals(fieldTag, decoder1.readUnsignedVarint32()); + assertEquals(blob.length + 4, decoder1.readUnsignedVarint32()); + + var decoder2 = new jspb.BinaryDecoder(decoder1.readBytes(blob.length + 4)); + assertEquals(fieldTag, decoder2.readUnsignedVarint32()); + assertEquals(blob.length + 2, decoder2.readUnsignedVarint32()); + + assertEquals(fieldTag, decoder2.readUnsignedVarint32()); + assertEquals(blob.length, decoder2.readUnsignedVarint32()); + var bytes = decoder2.readBytes(blob.length); + + assertElementsEquals(bytes, blob); + }); + + + /** + * Tests read callbacks. + */ + it('testReadCallbacks', function() { + var writer = new jspb.BinaryWriter(); + var dummyMessage = /** @type {!jspb.BinaryMessage} */({}); + + // Add an int, a submessage, and another int. + writer.writeInt32(1, 100); + + writer.writeMessage(2, dummyMessage, function() { + writer.writeInt32(3, 300); + writer.writeInt32(4, 400); + writer.writeInt32(5, 500); + }); + + writer.writeInt32(7, 700); + + // Create the reader and register a custom read callback. + var reader = jspb.BinaryReader.alloc(writer.getResultBuffer()); + + /** + * @param {!jspb.BinaryReader} reader + * @return {*} + */ + function readCallback(reader) { + reader.nextField(); + assertEquals(3, reader.getFieldNumber()); + assertEquals(300, reader.readInt32()); + + reader.nextField(); + assertEquals(4, reader.getFieldNumber()); + assertEquals(400, reader.readInt32()); + + reader.nextField(); + assertEquals(5, reader.getFieldNumber()); + assertEquals(500, reader.readInt32()); + + assertEquals(false, reader.nextField()); + }; + + reader.registerReadCallback('readCallback', readCallback); + + // Read the container message. + reader.nextField(); + assertEquals(1, reader.getFieldNumber()); + assertEquals(100, reader.readInt32()); + + reader.nextField(); + assertEquals(2, reader.getFieldNumber()); + reader.readMessage(dummyMessage, function() { + // Decode the embedded message using the registered callback. + reader.runReadCallback('readCallback'); + }); + + reader.nextField(); + assertEquals(7, reader.getFieldNumber()); + assertEquals(700, reader.readInt32()); + + assertEquals(false, reader.nextField()); + }); +}); diff --git a/js/binary/utils.js b/js/binary/utils.js new file mode 100644 index 00000000..92600389 --- /dev/null +++ b/js/binary/utils.js @@ -0,0 +1,979 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview This file contains helper code used by jspb.BinaryReader + * and BinaryWriter. + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.provide('jspb.utils'); + +goog.require('goog.asserts'); +goog.require('goog.crypt.base64'); +goog.require('goog.string'); +goog.require('jspb.BinaryConstants'); + + +/** + * Javascript can't natively handle 64-bit data types, so to manipulate them we + * have to split them into two 32-bit halves and do the math manually. + * + * Instead of instantiating and passing small structures around to do this, we + * instead just use two global temporary values. This one stores the low 32 + * bits of a split value - for example, if the original value was a 64-bit + * integer, this temporary value will contain the low 32 bits of that integer. + * If the original value was a double, this temporary value will contain the + * low 32 bits of the binary representation of that double, etcetera. + * @type {number} + */ +jspb.utils.split64Low = 0; + + +/** + * And correspondingly, this temporary variable will contain the high 32 bits + * of whatever value was split. + * @type {number} + */ +jspb.utils.split64High = 0; + + +/** + * Splits an unsigned Javascript integer into two 32-bit halves and stores it + * in the temp values above. + * @param {number} value The number to split. + */ +jspb.utils.splitUint64 = function(value) { + // Extract low 32 bits and high 32 bits as unsigned integers. + var lowBits = value >>> 0; + var highBits = Math.floor((value - lowBits) / + jspb.BinaryConstants.TWO_TO_32) >>> 0; + + jspb.utils.split64Low = lowBits; + jspb.utils.split64High = highBits; +}; + + +/** + * Splits a signed Javascript integer into two 32-bit halves and stores it in + * the temp values above. + * @param {number} value The number to split. + */ +jspb.utils.splitInt64 = function(value) { + // Convert to sign-magnitude representation. + var sign = (value < 0); + value = Math.abs(value); + + // Extract low 32 bits and high 32 bits as unsigned integers. + var lowBits = value >>> 0; + var highBits = Math.floor((value - lowBits) / + jspb.BinaryConstants.TWO_TO_32); + highBits = highBits >>> 0; + + // Perform two's complement conversion if the sign bit was set. + if (sign) { + highBits = ~highBits >>> 0; + lowBits = ~lowBits >>> 0; + lowBits += 1; + if (lowBits > 0xFFFFFFFF) { + lowBits = 0; + highBits++; + if (highBits > 0xFFFFFFFF) highBits = 0; + } + } + + jspb.utils.split64Low = lowBits; + jspb.utils.split64High = highBits; +}; + + +/** + * Convers a signed Javascript integer into zigzag format, splits it into two + * 32-bit halves, and stores it in the temp values above. + * @param {number} value The number to split. + */ +jspb.utils.splitZigzag64 = function(value) { + // Convert to sign-magnitude and scale by 2 before we split the value. + var sign = (value < 0); + value = Math.abs(value) * 2; + + jspb.utils.splitUint64(value); + var lowBits = jspb.utils.split64Low; + var highBits = jspb.utils.split64High; + + // If the value is negative, subtract 1 from the split representation so we + // don't lose the sign bit due to precision issues. + if (sign) { + if (lowBits == 0) { + if (highBits == 0) { + lowBits = 0xFFFFFFFF; + highBits = 0xFFFFFFFF; + } else { + highBits--; + lowBits = 0xFFFFFFFF; + } + } else { + lowBits--; + } + } + + jspb.utils.split64Low = lowBits; + jspb.utils.split64High = highBits; +}; + + +/** + * Converts a floating-point number into 32-bit IEEE representation and stores + * it in the temp values above. + * @param {number} value + */ +jspb.utils.splitFloat32 = function(value) { + var sign = (value < 0) ? 1 : 0; + value = sign ? -value : value; + var exp; + var mant; + + // Handle zeros. + if (value === 0) { + if ((1 / value) > 0) { + // Positive zero. + jspb.utils.split64High = 0; + jspb.utils.split64Low = 0x00000000; + } else { + // Negative zero. + jspb.utils.split64High = 0; + jspb.utils.split64Low = 0x80000000; + } + return; + } + + // Handle nans. + if (isNaN(value)) { + jspb.utils.split64High = 0; + jspb.utils.split64Low = 0x7FFFFFFF; + return; + } + + // Handle infinities. + if (value > jspb.BinaryConstants.FLOAT32_MAX) { + jspb.utils.split64High = 0; + jspb.utils.split64Low = ((sign << 31) | (0x7F800000)) >>> 0; + return; + } + + // Handle denormals. + if (value < jspb.BinaryConstants.FLOAT32_MIN) { + // Number is a denormal. + mant = Math.round(value / Math.pow(2, -149)); + jspb.utils.split64High = 0; + jspb.utils.split64Low = ((sign << 31) | mant) >>> 0; + return; + } + + exp = Math.floor(Math.log(value) / Math.LN2); + mant = value * Math.pow(2, -exp); + mant = Math.round(mant * jspb.BinaryConstants.TWO_TO_23) & 0x7FFFFF; + + jspb.utils.split64High = 0; + jspb.utils.split64Low = ((sign << 31) | ((exp + 127) << 23) | mant) >>> 0; +}; + + +/** + * Converts a floating-point number into 64-bit IEEE representation and stores + * it in the temp values above. + * @param {number} value + */ +jspb.utils.splitFloat64 = function(value) { + var sign = (value < 0) ? 1 : 0; + value = sign ? -value : value; + + // Handle zeros. + if (value === 0) { + if ((1 / value) > 0) { + // Positive zero. + jspb.utils.split64High = 0x00000000; + jspb.utils.split64Low = 0x00000000; + } else { + // Negative zero. + jspb.utils.split64High = 0x80000000; + jspb.utils.split64Low = 0x00000000; + } + return; + } + + // Handle nans. + if (isNaN(value)) { + jspb.utils.split64High = 0x7FFFFFFF; + jspb.utils.split64Low = 0xFFFFFFFF; + return; + } + + // Handle infinities. + if (value > jspb.BinaryConstants.FLOAT64_MAX) { + jspb.utils.split64High = ((sign << 31) | (0x7FF00000)) >>> 0; + jspb.utils.split64Low = 0; + return; + } + + // Handle denormals. + if (value < jspb.BinaryConstants.FLOAT64_MIN) { + // Number is a denormal. + var mant = value / Math.pow(2, -1074); + var mantHigh = (mant / jspb.BinaryConstants.TWO_TO_32); + jspb.utils.split64High = ((sign << 31) | mantHigh) >>> 0; + jspb.utils.split64Low = (mant >>> 0); + return; + } + + var exp = Math.floor(Math.log(value) / Math.LN2); + if (exp == 1024) exp = 1023; + var mant = value * Math.pow(2, -exp); + + var mantHigh = (mant * jspb.BinaryConstants.TWO_TO_20) & 0xFFFFF; + var mantLow = (mant * jspb.BinaryConstants.TWO_TO_52) >>> 0; + + jspb.utils.split64High = + ((sign << 31) | ((exp + 1023) << 20) | mantHigh) >>> 0; + jspb.utils.split64Low = mantLow; +}; + + +/** + * Converts an 8-character hash string into two 32-bit numbers and stores them + * in the temp values above. + * @param {string} hash + */ +jspb.utils.splitHash64 = function(hash) { + var a = hash.charCodeAt(0); + var b = hash.charCodeAt(1); + var c = hash.charCodeAt(2); + var d = hash.charCodeAt(3); + var e = hash.charCodeAt(4); + var f = hash.charCodeAt(5); + var g = hash.charCodeAt(6); + var h = hash.charCodeAt(7); + + jspb.utils.split64Low = (a + (b << 8) + (c << 16) + (d << 24)) >>> 0; + jspb.utils.split64High = (e + (f << 8) + (g << 16) + (h << 24)) >>> 0; +}; + + +/** + * Joins two 32-bit values into a 64-bit unsigned integer. Precision will be + * lost if the result is greater than 2^52. + * @param {number} bitsLow + * @param {number} bitsHigh + * @return {number} + */ +jspb.utils.joinUint64 = function(bitsLow, bitsHigh) { + return bitsHigh * jspb.BinaryConstants.TWO_TO_32 + bitsLow; +}; + + +/** + * Joins two 32-bit values into a 64-bit signed integer. Precision will be lost + * if the result is greater than 2^52. + * @param {number} bitsLow + * @param {number} bitsHigh + * @return {number} + */ +jspb.utils.joinInt64 = function(bitsLow, bitsHigh) { + // If the high bit is set, do a manual two's complement conversion. + var sign = (bitsHigh & 0x80000000); + if (sign) { + bitsLow = (~bitsLow + 1) >>> 0; + bitsHigh = ~bitsHigh >>> 0; + if (bitsLow == 0) { + bitsHigh = (bitsHigh + 1) >>> 0; + } + } + + var result = jspb.utils.joinUint64(bitsLow, bitsHigh); + return sign ? -result : result; +}; + + +/** + * Joins two 32-bit values into a 64-bit unsigned integer and applies zigzag + * decoding. Precision will be lost if the result is greater than 2^52. + * @param {number} bitsLow + * @param {number} bitsHigh + * @return {number} + */ +jspb.utils.joinZigzag64 = function(bitsLow, bitsHigh) { + // Extract the sign bit and shift right by one. + var sign = bitsLow & 1; + bitsLow = ((bitsLow >>> 1) | (bitsHigh << 31)) >>> 0; + bitsHigh = bitsHigh >>> 1; + + // Increment the split value if the sign bit was set. + if (sign) { + bitsLow = (bitsLow + 1) >>> 0; + if (bitsLow == 0) { + bitsHigh = (bitsHigh + 1) >>> 0; + } + } + + var result = jspb.utils.joinUint64(bitsLow, bitsHigh); + return sign ? -result : result; +}; + + +/** + * Joins two 32-bit values into a 32-bit IEEE floating point number and + * converts it back into a Javascript number. + * @param {number} bitsLow The low 32 bits of the binary number; + * @param {number} bitsHigh The high 32 bits of the binary number. + * @return {number} + */ +jspb.utils.joinFloat32 = function(bitsLow, bitsHigh) { + var sign = ((bitsLow >> 31) * 2 + 1); + var exp = (bitsLow >>> 23) & 0xFF; + var mant = bitsLow & 0x7FFFFF; + + if (exp == 0xFF) { + if (mant) { + return NaN; + } else { + return sign * Infinity; + } + } + + if (exp == 0) { + // Denormal. + return sign * Math.pow(2, -149) * mant; + } else { + return sign * Math.pow(2, exp - 150) * + (mant + Math.pow(2, 23)); + } +}; + + +/** + * Joins two 32-bit values into a 64-bit IEEE floating point number and + * converts it back into a Javascript number. + * @param {number} bitsLow The low 32 bits of the binary number; + * @param {number} bitsHigh The high 32 bits of the binary number. + * @return {number} + */ +jspb.utils.joinFloat64 = function(bitsLow, bitsHigh) { + var sign = ((bitsHigh >> 31) * 2 + 1); + var exp = (bitsHigh >>> 20) & 0x7FF; + var mant = jspb.BinaryConstants.TWO_TO_32 * (bitsHigh & 0xFFFFF) + bitsLow; + + if (exp == 0x7FF) { + if (mant) { + return NaN; + } else { + return sign * Infinity; + } + } + + if (exp == 0) { + // Denormal. + return sign * Math.pow(2, -1074) * mant; + } else { + return sign * Math.pow(2, exp - 1075) * + (mant + jspb.BinaryConstants.TWO_TO_52); + } +}; + + +/** + * Joins two 32-bit values into an 8-character hash string. + * @param {number} bitsLow + * @param {number} bitsHigh + * @return {string} + */ +jspb.utils.joinHash64 = function(bitsLow, bitsHigh) { + var a = (bitsLow >>> 0) & 0xFF; + var b = (bitsLow >>> 8) & 0xFF; + var c = (bitsLow >>> 16) & 0xFF; + var d = (bitsLow >>> 24) & 0xFF; + var e = (bitsHigh >>> 0) & 0xFF; + var f = (bitsHigh >>> 8) & 0xFF; + var g = (bitsHigh >>> 16) & 0xFF; + var h = (bitsHigh >>> 24) & 0xFF; + + return String.fromCharCode(a, b, c, d, e, f, g, h); +}; + + +/** + * Individual digits for number->string conversion. + * @const {!Array.} + */ +jspb.utils.DIGITS = [ + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' +]; + + +/** + * Losslessly converts a 64-bit unsigned integer in 32:32 split representation + * into a decimal string. + * @param {number} bitsLow The low 32 bits of the binary number; + * @param {number} bitsHigh The high 32 bits of the binary number. + * @return {string} The binary number represented as a string. + */ +jspb.utils.joinUnsignedDecimalString = function(bitsLow, bitsHigh) { + // Skip the expensive conversion if the number is small enough to use the + // built-in conversions. + if (bitsHigh <= 0x1FFFFF) { + return '' + (jspb.BinaryConstants.TWO_TO_32 * bitsHigh + bitsLow); + } + + // What this code is doing is essentially converting the input number from + // base-2 to base-1e7, which allows us to represent the 64-bit range with + // only 3 (very large) digits. Those digits are then trivial to convert to + // a base-10 string. + + // The magic numbers used here are - + // 2^24 = 16777216 = (1,6777216) in base-1e7. + // 2^48 = 281474976710656 = (2,8147497,6710656) in base-1e7. + + // Split 32:32 representation into 16:24:24 representation so our + // intermediate digits don't overflow. + var low = bitsLow & 0xFFFFFF; + var mid = (((bitsLow >>> 24) | (bitsHigh << 8)) >>> 0) & 0xFFFFFF; + var high = (bitsHigh >> 16) & 0xFFFF; + + // Assemble our three base-1e7 digits, ignoring carries. The maximum + // value in a digit at this step is representable as a 48-bit integer, which + // can be stored in a 64-bit floating point number. + var digitA = low + (mid * 6777216) + (high * 6710656); + var digitB = mid + (high * 8147497); + var digitC = (high * 2); + + // Apply carries from A to B and from B to C. + var base = 10000000; + if (digitA >= base) { + digitB += Math.floor(digitA / base); + digitA %= base; + } + + if (digitB >= base) { + digitC += Math.floor(digitB / base); + digitB %= base; + } + + // Convert base-1e7 digits to base-10, omitting leading zeroes. + var table = jspb.utils.DIGITS; + var start = false; + var result = ''; + + function emit(digit) { + var temp = base; + for (var i = 0; i < 7; i++) { + temp /= 10; + var decimalDigit = ((digit / temp) % 10) >>> 0; + if ((decimalDigit == 0) && !start) continue; + start = true; + result += table[decimalDigit]; + } + } + + if (digitC || start) emit(digitC); + if (digitB || start) emit(digitB); + if (digitA || start) emit(digitA); + + return result; +}; + + +/** + * Losslessly converts a 64-bit signed integer in 32:32 split representation + * into a decimal string. + * @param {number} bitsLow The low 32 bits of the binary number; + * @param {number} bitsHigh The high 32 bits of the binary number. + * @return {string} The binary number represented as a string. + */ +jspb.utils.joinSignedDecimalString = function(bitsLow, bitsHigh) { + // If we're treating the input as a signed value and the high bit is set, do + // a manual two's complement conversion before the decimal conversion. + var negative = (bitsHigh & 0x80000000); + if (negative) { + bitsLow = (~bitsLow + 1) >>> 0; + var carry = (bitsLow == 0) ? 1 : 0; + bitsHigh = (~bitsHigh + carry) >>> 0; + } + + var result = jspb.utils.joinUnsignedDecimalString(bitsLow, bitsHigh); + return negative ? '-' + result : result; +}; + + +/** + * Convert an 8-character hash string representing either a signed or unsigned + * 64-bit integer into its decimal representation without losing accuracy. + * @param {string} hash The hash string to convert. + * @param {boolean} signed True if we should treat the hash string as encoding + * a signed integer. + * @return {string} + */ +jspb.utils.hash64ToDecimalString = function(hash, signed) { + jspb.utils.splitHash64(hash); + var bitsLow = jspb.utils.split64Low; + var bitsHigh = jspb.utils.split64High; + return signed ? + jspb.utils.joinSignedDecimalString(bitsLow, bitsHigh) : + jspb.utils.joinUnsignedDecimalString(bitsLow, bitsHigh); +}; + + +/** + * Converts an array of 8-character hash strings into their decimal + * representations. + * @param {!Array.} hashes The array of hash strings to convert. + * @param {boolean} signed True if we should treat the hash string as encoding + * a signed integer. + * @return {!Array.} + */ +jspb.utils.hash64ArrayToDecimalStrings = function(hashes, signed) { + var result = new Array(hashes.length); + for (var i = 0; i < hashes.length; i++) { + result[i] = jspb.utils.hash64ToDecimalString(hashes[i], signed); + } + return result; +}; + + +/** + * Converts an 8-character hash string into its hexadecimal representation. + * @param {string} hash + * @return {string} + */ +jspb.utils.hash64ToHexString = function(hash) { + var temp = new Array(18); + temp[0] = '0'; + temp[1] = 'x'; + + for (var i = 0; i < 8; i++) { + var c = hash.charCodeAt(7 - i); + temp[i * 2 + 2] = jspb.utils.DIGITS[c >> 4]; + temp[i * 2 + 3] = jspb.utils.DIGITS[c & 0xF]; + } + + var result = temp.join(''); + return result; +}; + + +/** + * Converts a '0x<16 digits>' hex string into its hash string representation. + * @param {string} hex + * @return {string} + */ +jspb.utils.hexStringToHash64 = function(hex) { + hex = hex.toLowerCase(); + goog.asserts.assert(hex.length == 18); + goog.asserts.assert(hex[0] == '0'); + goog.asserts.assert(hex[1] == 'x'); + + var result = ''; + for (var i = 0; i < 8; i++) { + var hi = jspb.utils.DIGITS.indexOf(hex[i * 2 + 2]); + var lo = jspb.utils.DIGITS.indexOf(hex[i * 2 + 3]); + result = String.fromCharCode(hi * 16 + lo) + result; + } + + return result; +}; + + +/** + * Convert an 8-character hash string representing either a signed or unsigned + * 64-bit integer into a Javascript number. Will lose accuracy if the result is + * larger than 2^52. + * @param {string} hash The hash string to convert. + * @param {boolean} signed True if the has should be interpreted as a signed + * number. + * @return {number} + */ +jspb.utils.hash64ToNumber = function(hash, signed) { + jspb.utils.splitHash64(hash); + var bitsLow = jspb.utils.split64Low; + var bitsHigh = jspb.utils.split64High; + return signed ? jspb.utils.joinInt64(bitsLow, bitsHigh) : + jspb.utils.joinUint64(bitsLow, bitsHigh); +}; + + +/** + * Convert a Javascript number into an 8-character hash string. Will lose + * precision if the value is non-integral or greater than 2^64. + * @param {number} value The integer to convert. + * @return {string} + */ +jspb.utils.numberToHash64 = function(value) { + jspb.utils.splitInt64(value); + return jspb.utils.joinHash64(jspb.utils.split64Low, + jspb.utils.split64High); +}; + + +/** + * Counts the number of contiguous varints in a buffer. + * @param {!Uint8Array} buffer The buffer to scan. + * @param {number} start The starting point in the buffer to scan. + * @param {number} end The end point in the buffer to scan. + * @return {number} The number of varints in the buffer. + */ +jspb.utils.countVarints = function(buffer, start, end) { + // Count how many high bits of each byte were set in the buffer. + var count = 0; + for (var i = start; i < end; i++) { + count += buffer[i] >> 7; + } + + // The number of varints in the buffer equals the size of the buffer minus + // the number of non-terminal bytes in the buffer (those with the high bit + // set). + return (end - start) - count; +}; + + +/** + * Counts the number of contiguous varint fields with the given field number in + * the buffer. + * @param {!Uint8Array} buffer The buffer to scan. + * @param {number} start The starting point in the buffer to scan. + * @param {number} end The end point in the buffer to scan. + * @param {number} field The field number to count. + * @return {number} The number of matching fields in the buffer. + */ +jspb.utils.countVarintFields = function(buffer, start, end, field) { + var count = 0; + var cursor = start; + var tag = field * 8 + jspb.BinaryConstants.WireType.VARINT; + + if (tag < 128) { + // Single-byte field tag, we can use a slightly quicker count. + while (cursor < end) { + // Skip the field tag, or exit if we find a non-matching tag. + if (buffer[cursor++] != tag) return count; + + // Field tag matches, we've found a valid field. + count++; + + // Skip the varint. + while (1) { + var x = buffer[cursor++]; + if ((x & 0x80) == 0) break; + } + } + } else { + while (cursor < end) { + // Skip the field tag, or exit if we find a non-matching tag. + var temp = tag; + while (temp > 128) { + if (buffer[cursor] != ((temp & 0x7F) | 0x80)) return count; + cursor++; + temp >>= 7; + } + if (buffer[cursor++] != temp) return count; + + // Field tag matches, we've found a valid field. + count++; + + // Skip the varint. + while (1) { + var x = buffer[cursor++]; + if ((x & 0x80) == 0) break; + } + } + } + return count; +}; + + +/** + * Counts the number of contiguous fixed32 fields with the given tag in the + * buffer. + * @param {!Uint8Array} buffer The buffer to scan. + * @param {number} start The starting point in the buffer to scan. + * @param {number} end The end point in the buffer to scan. + * @param {number} tag The tag value to count. + * @param {number} stride The number of bytes to skip per field. + * @return {number} The number of fields with a matching tag in the buffer. + * @private + */ +jspb.utils.countFixedFields_ = + function(buffer, start, end, tag, stride) { + var count = 0; + var cursor = start; + + if (tag < 128) { + // Single-byte field tag, we can use a slightly quicker count. + while (cursor < end) { + // Skip the field tag, or exit if we find a non-matching tag. + if (buffer[cursor++] != tag) return count; + + // Field tag matches, we've found a valid field. + count++; + + // Skip the value. + cursor += stride; + } + } else { + while (cursor < end) { + // Skip the field tag, or exit if we find a non-matching tag. + var temp = tag; + while (temp > 128) { + if (buffer[cursor++] != ((temp & 0x7F) | 0x80)) return count; + temp >>= 7; + } + if (buffer[cursor++] != temp) return count; + + // Field tag matches, we've found a valid field. + count++; + + // Skip the value. + cursor += stride; + } + } + return count; +}; + + +/** + * Counts the number of contiguous fixed32 fields with the given field number + * in the buffer. + * @param {!Uint8Array} buffer The buffer to scan. + * @param {number} start The starting point in the buffer to scan. + * @param {number} end The end point in the buffer to scan. + * @param {number} field The field number to count. + * @return {number} The number of matching fields in the buffer. + */ +jspb.utils.countFixed32Fields = function(buffer, start, end, field) { + var tag = field * 8 + jspb.BinaryConstants.WireType.FIXED32; + return jspb.utils.countFixedFields_(buffer, start, end, tag, 4); +}; + + +/** + * Counts the number of contiguous fixed64 fields with the given field number + * in the buffer. + * @param {!Uint8Array} buffer The buffer to scan. + * @param {number} start The starting point in the buffer to scan. + * @param {number} end The end point in the buffer to scan. + * @param {number} field The field number to count + * @return {number} The number of matching fields in the buffer. + */ +jspb.utils.countFixed64Fields = function(buffer, start, end, field) { + var tag = field * 8 + jspb.BinaryConstants.WireType.FIXED64; + return jspb.utils.countFixedFields_(buffer, start, end, tag, 8); +}; + + +/** + * Counts the number of contiguous delimited fields with the given field number + * in the buffer. + * @param {!Uint8Array} buffer The buffer to scan. + * @param {number} start The starting point in the buffer to scan. + * @param {number} end The end point in the buffer to scan. + * @param {number} field The field number to count. + * @return {number} The number of matching fields in the buffer. + */ +jspb.utils.countDelimitedFields = function(buffer, start, end, field) { + var count = 0; + var cursor = start; + var tag = field * 8 + jspb.BinaryConstants.WireType.DELIMITED; + + while (cursor < end) { + // Skip the field tag, or exit if we find a non-matching tag. + var temp = tag; + while (temp > 128) { + if (buffer[cursor++] != ((temp & 0x7F) | 0x80)) return count; + temp >>= 7; + } + if (buffer[cursor++] != temp) return count; + + // Field tag matches, we've found a valid field. + count++; + + // Decode the length prefix. + var length = 0; + var shift = 1; + while (1) { + temp = buffer[cursor++]; + length += (temp & 0x7f) * shift; + shift *= 128; + if ((temp & 0x80) == 0) break; + } + + // Advance the cursor past the blob. + cursor += length; + } + return count; +}; + + +/** + * Clones a scalar field. Pulling this out to a helper method saves us a few + * bytes of generated code. + * @param {Array} array + * @return {Array} + */ +jspb.utils.cloneRepeatedScalarField = function(array) { + return array ? array.slice() : null; +}; + + +/** + * Clones an array of messages using the provided cloner function. + * @param {Array.} messages + * @param {jspb.ClonerFunction} cloner + * @return {Array.} + */ +jspb.utils.cloneRepeatedMessageField = function(messages, cloner) { + if (messages === null) return null; + var result = []; + for (var i = 0; i < messages.length; i++) { + result.push(cloner(messages[i])); + } + return result; +}; + + +/** + * Clones an array of byte blobs. + * @param {Array.} blobs + * @return {Array.} + */ +jspb.utils.cloneRepeatedBlobField = function(blobs) { + if (blobs === null) return null; + var result = []; + for (var i = 0; i < blobs.length; i++) { + result.push(new Uint8Array(blobs[i])); + } + return result; +}; + + +/** + * String-ify bytes for text format. Should be optimized away in non-debug. + * The returned string uses \xXX escapes for all values and is itself quoted. + * [1, 31] serializes to '"\x01\x1f"'. + * @param {jspb.ByteSource} byteSource The bytes to serialize. + * @param {boolean=} opt_stringIsRawBytes The string is interpreted as a series + * of raw bytes rather than base64 data. + * @return {string} Stringified bytes for text format. + */ +jspb.utils.debugBytesToTextFormat = function(byteSource, + opt_stringIsRawBytes) { + var s = '"'; + if (byteSource) { + var bytes = + jspb.utils.byteSourceToUint8Array(byteSource, opt_stringIsRawBytes); + for (var i = 0; i < bytes.length; i++) { + s += '\\x'; + if (bytes[i] < 16) s += '0'; + s += bytes[i].toString(16); + } + } + return s + '"'; +}; + + +/** + * String-ify a scalar for text format. Should be optimized away in non-debug. + * @param {string|number|boolean} scalar The scalar to stringify. + * @return {string} Stringified scalar for text format. + */ +jspb.utils.debugScalarToTextFormat = function(scalar) { + if (goog.isString(scalar)) { + return goog.string.quote(scalar); + } else { + return scalar.toString(); + } +}; + + +/** + * Utility function: convert a string with codepoints 0--255 inclusive to a + * Uint8Array. If any codepoints greater than 255 exist in the string, throws an + * exception. + * @param {string} str + * @return {!Uint8Array} + * @private + */ +jspb.utils.stringToByteArray_ = function(str) { + var arr = new Uint8Array(str.length); + for (var i = 0; i < str.length; i++) { + var codepoint = str.charCodeAt(i); + if (codepoint > 255) { + throw new Error('Conversion error: string contains codepoint ' + + 'outside of byte range'); + } + arr[i] = codepoint; + } + return arr; +}; + + +/** + * Converts any type defined in jspb.ByteSource into a Uint8Array. + * @param {!jspb.ByteSource} data + * @param {boolean=} opt_stringIsRawBytes Interpret a string as a series of raw + * bytes (encoded as codepoints 0--255 inclusive) rather than base64 data + * (default behavior). + * @return {!Uint8Array} + * @suppress {invalidCasts} + */ +jspb.utils.byteSourceToUint8Array = function(data, opt_stringIsRawBytes) { + if (data.constructor === Uint8Array) { + return /** @type {!Uint8Array} */(data); + } + + if (data.constructor === ArrayBuffer) { + data = /** @type {!ArrayBuffer} */(data); + return /** @type {!Uint8Array} */(new Uint8Array(data)); + } + + if (data.constructor === Array) { + data = /** @type {!Array.} */(data); + return /** @type {!Uint8Array} */(new Uint8Array(data)); + } + + if (data.constructor === String) { + data = /** @type {string} */(data); + if (opt_stringIsRawBytes) { + return jspb.utils.stringToByteArray_(data); + } else { + return goog.crypt.base64.decodeStringToUint8Array(data); + } + } + + goog.asserts.fail('Type not convertible to Uint8Array.'); + return /** @type {!Uint8Array} */(new Uint8Array(0)); +}; diff --git a/js/binary/utils_test.js b/js/binary/utils_test.js new file mode 100644 index 00000000..5c330791 --- /dev/null +++ b/js/binary/utils_test.js @@ -0,0 +1,632 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Test cases for jspb's helper functions. + * + * Test suite is written using Jasmine -- see http://jasmine.github.io/ + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.require('goog.crypt.base64'); +goog.require('goog.testing.asserts'); +goog.require('jspb.BinaryConstants'); +goog.require('jspb.BinaryWriter'); +goog.require('jspb.utils'); + + +/** + * @param {number} x + * @return {number} + */ +function truncate(x) { + var temp = new Float32Array(1); + temp[0] = x; + return temp[0]; +} + + +/** + * Converts an 64-bit integer in split representation to a 64-bit hash string + * (8 bits encoded per character). + * @param {number} bitsLow The low 32 bits of the split 64-bit integer. + * @param {number} bitsHigh The high 32 bits of the split 64-bit integer. + * @return {string} The encoded hash string, 8 bits per character. + */ +function toHashString(bitsLow, bitsHigh) { + return String.fromCharCode((bitsLow >>> 0) & 0xFF, + (bitsLow >>> 8) & 0xFF, + (bitsLow >>> 16) & 0xFF, + (bitsLow >>> 24) & 0xFF, + (bitsHigh >>> 0) & 0xFF, + (bitsHigh >>> 8) & 0xFF, + (bitsHigh >>> 16) & 0xFF, + (bitsHigh >>> 24) & 0xFF); +} + + +describe('binaryUtilsTest', function() { + /** + * Tests lossless binary-to-decimal conversion. + */ + it('testDecimalConversion', function() { + // Check some magic numbers. + var result = + jspb.utils.joinUnsignedDecimalString(0x89e80001, 0x8ac72304); + assertEquals('10000000000000000001', result); + + result = jspb.utils.joinUnsignedDecimalString(0xacd05f15, 0x1b69b4b); + assertEquals('123456789123456789', result); + + result = jspb.utils.joinUnsignedDecimalString(0xeb1f0ad2, 0xab54a98c); + assertEquals('12345678901234567890', result); + + result = jspb.utils.joinUnsignedDecimalString(0xe3b70cb1, 0x891087b8); + assertEquals('9876543210987654321', result); + + // Check limits. + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x00000000); + assertEquals('0', result); + + result = jspb.utils.joinUnsignedDecimalString(0xFFFFFFFF, 0xFFFFFFFF); + assertEquals('18446744073709551615', result); + + // Check each bit of the low dword. + for (var i = 0; i < 32; i++) { + var low = (1 << i) >>> 0; + result = jspb.utils.joinUnsignedDecimalString(low, 0); + assertEquals('' + Math.pow(2, i), result); + } + + // Check the first 20 bits of the high dword. + for (var i = 0; i < 20; i++) { + var high = (1 << i) >>> 0; + result = jspb.utils.joinUnsignedDecimalString(0, high); + assertEquals('' + Math.pow(2, 32 + i), result); + } + + // V8's internal double-to-string conversion is inaccurate for values above + // 2^52, even if they're representable integers - check the rest of the bits + // manually against the correct string representations of 2^N. + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x00100000); + assertEquals('4503599627370496', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x00200000); + assertEquals('9007199254740992', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x00400000); + assertEquals('18014398509481984', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x00800000); + assertEquals('36028797018963968', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x01000000); + assertEquals('72057594037927936', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x02000000); + assertEquals('144115188075855872', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x04000000); + assertEquals('288230376151711744', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x08000000); + assertEquals('576460752303423488', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x10000000); + assertEquals('1152921504606846976', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x20000000); + assertEquals('2305843009213693952', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x40000000); + assertEquals('4611686018427387904', result); + + result = jspb.utils.joinUnsignedDecimalString(0x00000000, 0x80000000); + assertEquals('9223372036854775808', result); + }); + + + /** + * Going from hash strings to decimal strings should also be lossless. + */ + it('testHashToDecimalConversion', function() { + var result; + var convert = jspb.utils.hash64ToDecimalString; + + result = convert(toHashString(0x00000000, 0x00000000), false); + assertEquals('0', result); + + result = convert(toHashString(0x00000000, 0x00000000), true); + assertEquals('0', result); + + result = convert(toHashString(0xFFFFFFFF, 0xFFFFFFFF), false); + assertEquals('18446744073709551615', result); + + result = convert(toHashString(0xFFFFFFFF, 0xFFFFFFFF), true); + assertEquals('-1', result); + + result = convert(toHashString(0x00000000, 0x80000000), false); + assertEquals('9223372036854775808', result); + + result = convert(toHashString(0x00000000, 0x80000000), true); + assertEquals('-9223372036854775808', result); + + result = convert(toHashString(0xacd05f15, 0x01b69b4b), false); + assertEquals('123456789123456789', result); + + result = convert(toHashString(~0xacd05f15 + 1, ~0x01b69b4b), true); + assertEquals('-123456789123456789', result); + + // And converting arrays of hashes should work the same way. + result = jspb.utils.hash64ArrayToDecimalStrings([ + toHashString(0xFFFFFFFF, 0xFFFFFFFF), + toHashString(0x00000000, 0x80000000), + toHashString(0xacd05f15, 0x01b69b4b)], false); + assertEquals(3, result.length); + assertEquals('18446744073709551615', result[0]); + assertEquals('9223372036854775808', result[1]); + assertEquals('123456789123456789', result[2]); + }); + + + /** + * Going from hash strings to hex strings should be lossless. + */ + it('testHashToHexConversion', function() { + var result; + var convert = jspb.utils.hash64ToHexString; + + result = convert(toHashString(0x00000000, 0x00000000)); + assertEquals('0x0000000000000000', result); + + result = convert(toHashString(0xFFFFFFFF, 0xFFFFFFFF)); + assertEquals('0xffffffffffffffff', result); + + result = convert(toHashString(0x12345678, 0x9ABCDEF0)); + assertEquals('0x9abcdef012345678', result); + }); + + + /** + * Going from hex strings to hash strings should be lossless. + */ + it('testHexToHashConversion', function() { + var result; + var convert = jspb.utils.hexStringToHash64; + + result = convert('0x0000000000000000'); + assertEquals(String.fromCharCode.apply(null, + [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]), result); + + result = convert('0xffffffffffffffff'); + assertEquals(String.fromCharCode.apply(null, + [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]), result); + + // Hex string is big-endian, hash string is little-endian. + result = convert('0x123456789ABCDEF0'); + assertEquals(String.fromCharCode.apply(null, + [0xF0, 0xDE, 0xBC, 0x9A, 0x78, 0x56, 0x34, 0x12]), result); + + // Capitalization should not matter. + result = convert('0x0000abcdefABCDEF'); + assertEquals(String.fromCharCode.apply(null, + [0xEF, 0xCD, 0xAB, 0xEF, 0xCD, 0xAB, 0x00, 0x00]), result); + }); + + + /** + * Going from numbers to hash strings should be lossless for up to 53 bits of + * precision. + */ + it('testNumberToHashConversion', function() { + var result; + var convert = jspb.utils.numberToHash64; + + result = convert(0x0000000000000); + assertEquals('0x0000000000000000', jspb.utils.hash64ToHexString(result)); + + result = convert(0xFFFFFFFFFFFFF); + assertEquals('0x000fffffffffffff', jspb.utils.hash64ToHexString(result)); + + result = convert(0x123456789ABCD); + assertEquals('0x000123456789abcd', jspb.utils.hash64ToHexString(result)); + + result = convert(0xDCBA987654321); + assertEquals('0x000dcba987654321', jspb.utils.hash64ToHexString(result)); + + // 53 bits of precision should not be truncated. + result = convert(0x10000000000001); + assertEquals('0x0010000000000001', jspb.utils.hash64ToHexString(result)); + + // 54 bits of precision should be truncated. + result = convert(0x20000000000001); + assertNotEquals( + '0x0020000000000001', jspb.utils.hash64ToHexString(result)); + }); + + + /** + * Sanity check the behavior of Javascript's strings when doing funny things + * with unicode characters. + */ + it('sanityCheckUnicodeStrings', function() { + var strings = new Array(65536); + + // All possible unsigned 16-bit values should be storable in a string, they + // shouldn't do weird things with the length of the string, and they should + // come back out of the string unchanged. + for (var i = 0; i < 65536; i++) { + strings[i] = 'a' + String.fromCharCode(i) + 'a'; + if (3 != strings[i].length) throw 'fail!'; + if (i != strings[i].charCodeAt(1)) throw 'fail!'; + } + + // Each unicode character should compare equal to itself and not equal to a + // different unicode character. + for (var i = 0; i < 65536; i++) { + if (strings[i] != strings[i]) throw 'fail!'; + if (strings[i] == strings[(i + 1) % 65536]) throw 'fail!'; + } + }); + + + /** + * Tests conversion from 32-bit floating point numbers to split64 numbers. + */ + it('testFloat32ToSplit64', function() { + var f32_eps = jspb.BinaryConstants.FLOAT32_EPS; + var f32_min = jspb.BinaryConstants.FLOAT32_MIN; + var f32_max = jspb.BinaryConstants.FLOAT32_MAX; + + // NaN. + jspb.utils.splitFloat32(NaN); + if (!isNaN(jspb.utils.joinFloat32(jspb.utils.split64Low, + jspb.utils.split64High))) { + throw 'fail!'; + } + + /** + * @param {number} x + * @param {number=} opt_bits + */ + function test(x, opt_bits) { + jspb.utils.splitFloat32(x); + if (goog.isDef(opt_bits)) { + if (opt_bits != jspb.utils.split64Low) throw 'fail!'; + } + if (truncate(x) != jspb.utils.joinFloat32(jspb.utils.split64Low, + jspb.utils.split64High)) { + throw 'fail!'; + } + } + + // Positive and negative infinity. + test(Infinity, 0x7f800000); + test(-Infinity, 0xff800000); + + // Positive and negative zero. + test(0, 0x00000000); + test(-0, 0x80000000); + + // Positive and negative epsilon. + test(f32_eps, 0x00000001); + test(-f32_eps, 0x80000001); + + // Positive and negative min. + test(f32_min, 0x00800000); + test(-f32_min, 0x80800000); + + // Positive and negative max. + test(f32_max, 0x7F7FFFFF); + test(-f32_max, 0xFF7FFFFF); + + // Various positive values. + var cursor = f32_eps * 10; + while (cursor != Infinity) { + test(cursor); + cursor *= 1.1; + } + + // Various negative values. + cursor = -f32_eps * 10; + while (cursor != -Infinity) { + test(cursor); + cursor *= 1.1; + } + }); + + + /** + * Tests conversion from 64-bit floating point numbers to split64 numbers. + */ + it('testFloat64ToSplit64', function() { + var f64_eps = jspb.BinaryConstants.FLOAT64_EPS; + var f64_min = jspb.BinaryConstants.FLOAT64_MIN; + var f64_max = jspb.BinaryConstants.FLOAT64_MAX; + + // NaN. + jspb.utils.splitFloat64(NaN); + if (!isNaN(jspb.utils.joinFloat64(jspb.utils.split64Low, + jspb.utils.split64High))) { + throw 'fail!'; + } + + /** + * @param {number} x + * @param {number=} opt_highBits + * @param {number=} opt_lowBits + */ + function test(x, opt_highBits, opt_lowBits) { + jspb.utils.splitFloat64(x); + if (goog.isDef(opt_highBits)) { + if (opt_highBits != jspb.utils.split64High) throw 'fail!'; + } + if (goog.isDef(opt_lowBits)) { + if (opt_lowBits != jspb.utils.split64Low) throw 'fail!'; + } + if (x != jspb.utils.joinFloat64(jspb.utils.split64Low, + jspb.utils.split64High)) { + throw 'fail!'; + } + } + + // Positive and negative infinity. + test(Infinity, 0x7ff00000, 0x00000000); + test(-Infinity, 0xfff00000, 0x00000000); + + // Positive and negative zero. + test(0, 0x00000000, 0x00000000); + test(-0, 0x80000000, 0x00000000); + + // Positive and negative epsilon. + test(f64_eps, 0x00000000, 0x00000001); + test(-f64_eps, 0x80000000, 0x00000001); + + // Positive and negative min. + test(f64_min, 0x00100000, 0x00000000); + test(-f64_min, 0x80100000, 0x00000000); + + // Positive and negative max. + test(f64_max, 0x7FEFFFFF, 0xFFFFFFFF); + test(-f64_max, 0xFFEFFFFF, 0xFFFFFFFF); + + // Various positive values. + var cursor = f64_eps * 10; + while (cursor != Infinity) { + test(cursor); + cursor *= 1.1; + } + + // Various negative values. + cursor = -f64_eps * 10; + while (cursor != -Infinity) { + test(cursor); + cursor *= 1.1; + } + }); + + + /** + * Tests counting packed varints. + */ + it('testCountVarints', function() { + var writer = new jspb.BinaryWriter(); + + var count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.rawWriteVarint(Math.floor(i)); + count++; + } + + var buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, jspb.utils.countVarints(buffer, 0, buffer.length)); + }); + + + /** + * Tests counting matching varint fields. + */ + it('testCountVarintFields', function() { + var writer = new jspb.BinaryWriter(); + + var count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.writeUint64(1, Math.floor(i)); + count++; + } + writer.writeString(2, 'terminator'); + + var buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countVarintFields(buffer, 0, buffer.length, 1)); + + writer = new jspb.BinaryWriter(); + + count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.writeUint64(123456789, Math.floor(i)); + count++; + } + writer.writeString(2, 'terminator'); + + buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countVarintFields(buffer, 0, buffer.length, 123456789)); + }); + + + /** + * Tests counting matching fixed32 fields. + */ + it('testCountFixed32Fields', function() { + var writer = new jspb.BinaryWriter(); + + var count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.writeFixed32(1, Math.floor(i)); + count++; + } + writer.writeString(2, 'terminator'); + + var buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countFixed32Fields(buffer, 0, buffer.length, 1)); + + writer = new jspb.BinaryWriter(); + + count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.writeFixed32(123456789, Math.floor(i)); + count++; + } + writer.writeString(2, 'terminator'); + + buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countFixed32Fields(buffer, 0, buffer.length, 123456789)); + }); + + + /** + * Tests counting matching fixed64 fields. + */ + it('testCountFixed64Fields', function() { + var writer = new jspb.BinaryWriter(); + + var count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.writeDouble(1, i); + count++; + } + writer.writeString(2, 'terminator'); + + var buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countFixed64Fields(buffer, 0, buffer.length, 1)); + + writer = new jspb.BinaryWriter(); + + count = 0; + for (var i = 1; i < 1000000000; i *= 1.1) { + writer.writeDouble(123456789, i); + count++; + } + writer.writeString(2, 'terminator'); + + buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countFixed64Fields(buffer, 0, buffer.length, 123456789)); + }); + + + /** + * Tests counting matching delimited fields. + */ + it('testCountDelimitedFields', function() { + var writer = new jspb.BinaryWriter(); + + var count = 0; + for (var i = 1; i < 1000; i *= 1.1) { + writer.writeBytes(1, [Math.floor(i)]); + count++; + } + writer.writeString(2, 'terminator'); + + var buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countDelimitedFields(buffer, 0, buffer.length, 1)); + + writer = new jspb.BinaryWriter(); + + count = 0; + for (var i = 1; i < 1000; i *= 1.1) { + writer.writeBytes(123456789, [Math.floor(i)]); + count++; + } + writer.writeString(2, 'terminator'); + + buffer = new Uint8Array(writer.getResultBuffer()); + assertEquals(count, + jspb.utils.countDelimitedFields(buffer, 0, buffer.length, 123456789)); + }); + + + /** + * Tests byte format for debug strings. + */ + it('testDebugBytesToTextFormat', function() { + assertEquals('""', jspb.utils.debugBytesToTextFormat(null)); + assertEquals('"\\x00\\x10\\xff"', + jspb.utils.debugBytesToTextFormat([0, 16, 255])); + }); + + + /** + * Tests converting byte blob sources into byte blobs. + */ + it('testByteSourceToUint8Array', function() { + var convert = jspb.utils.byteSourceToUint8Array; + + var sourceData = []; + for (var i = 0; i < 256; i++) { + sourceData.push(i); + } + + var sourceBytes = new Uint8Array(sourceData); + var sourceBuffer = sourceBytes.buffer; + var sourceBase64 = goog.crypt.base64.encodeByteArray(sourceData); + var sourceString = String.fromCharCode.apply(null, sourceData); + + function check(result) { + assertEquals(Uint8Array, result.constructor); + assertEquals(sourceData.length, result.length); + for (var i = 0; i < result.length; i++) { + assertEquals(sourceData[i], result[i]); + } + } + + // Converting Uint8Arrays into Uint8Arrays should be a no-op. + assertEquals(sourceBytes, convert(sourceBytes)); + + // Converting Array. into Uint8Arrays should work. + check(convert(sourceData)); + + // Converting ArrayBuffers into Uint8Arrays should work. + check(convert(sourceBuffer)); + + // Converting base64-encoded strings into Uint8Arrays should work. + check(convert(sourceBase64)); + + // Converting binary-data strings into Uint8Arrays should work. + check(convert(sourceString, /* opt_stringIsRawBytes = */ true)); + }); +}); diff --git a/js/binary/writer.js b/js/binary/writer.js new file mode 100644 index 00000000..a1849457 --- /dev/null +++ b/js/binary/writer.js @@ -0,0 +1,2124 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview This file contains utilities for encoding Javascript objects + * into binary, wire-format protocol buffers (in the form of Uint8Arrays) that + * a server can consume directly. + * + * jspb's BinaryWriter class defines methods for efficiently encoding + * Javascript objects into binary, wire-format protocol buffers and supports + * all the fundamental field types used in protocol buffers. + * + * Major caveat 1 - Users of this library _must_ keep their Javascript proto + * parsing code in sync with the original .proto file - presumably you'll be + * using the typed jspb code generator, but if you bypass that you'll need + * to keep things in sync by hand. + * + * Major caveat 2 - Javascript is unable to accurately represent integers + * larger than 2^53 due to its use of a double-precision floating point format + * for all numbers. BinaryWriter does not make any special effort to preserve + * precision for values above this limit - if you need to pass 64-bit integers + * (hash codes, for example) between the client and server without precision + * loss, do _not_ use this library. + * + * Major caveat 3 - This class uses typed arrays and must not be used on older + * browsers that do not support them. + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.provide('jspb.BinaryWriter'); + +goog.require('goog.asserts'); +goog.require('goog.crypt.base64'); +goog.require('jspb.BinaryConstants'); +goog.require('jspb.arith.Int64'); +goog.require('jspb.arith.UInt64'); +goog.require('jspb.utils'); + +goog.forwardDeclare('jspb.Message'); + + + +/** + * BinaryWriter implements encoders for all the wire types specified in + * https://developers.google.com/protocol-buffers/docs/encoding. + * + * @constructor + * @struct + */ +jspb.BinaryWriter = function() { + /** + * Blocks of serialized data that will be concatenated once all messages have + * been written. + * @private {!Array>} + */ + this.blocks_ = []; + + /** + * Total number of bytes in the blocks_ array. Does _not_ include the temp + * buffer. + * @private {number} + */ + this.totalLength_ = 0; + + /** + * Temporary buffer holding a message that we're still serializing. When we + * get to a stopping point (either the start of a new submessage, or when we + * need to append a raw Uint8Array), the temp buffer will be added to the + * block array above and a new temp buffer will be created. + * @private {!Array.} + */ + this.temp_ = []; + + /** + * A stack of bookmarks containing the parent blocks for each message started + * via beginSubMessage(), needed as bookkeeping for endSubMessage(). + * TODO(aappleby): Deprecated, users should be calling writeMessage(). + * @private {!Array.} + */ + this.bookmarks_ = []; +}; + + +/** + * @typedef {{block: !Array., length: number}} + * @private + */ +jspb.BinaryWriter.Bookmark_; + + +/** + * Saves the current temp buffer in the blocks_ array and starts a new one. + * @return {!Array.} Returns a reference to the old temp buffer. + * @private + */ +jspb.BinaryWriter.prototype.saveTempBuffer_ = function() { + var oldTemp = this.temp_; + this.blocks_.push(this.temp_); + this.totalLength_ += this.temp_.length; + this.temp_ = []; + return oldTemp; +}; + + +/** + * Append a typed array of bytes onto the buffer. + * + * @param {!Uint8Array} arr The byte array to append. + * @private + */ +jspb.BinaryWriter.prototype.appendUint8Array_ = function(arr) { + if (this.temp_.length) { + this.saveTempBuffer_(); + } + this.blocks_.push(arr); + this.totalLength_ += arr.length; +}; + + +/** + * Append an untyped array of bytes onto the buffer. + * + * @param {!Array.} arr The byte array to append. + * @private + */ +jspb.BinaryWriter.prototype.appendArray_ = function(arr) { + if (this.temp_.length) { + this.saveTempBuffer_(); + } + this.temp_ = arr; +}; + + +/** + * Begins a length-delimited section by writing the field header to the current + * temp buffer and then saving it in the block array. Returns the saved block, + * which we will append the length to in endDelimited_ below. + * @param {number} field + * @return {!jspb.BinaryWriter.Bookmark_} + * @private + */ +jspb.BinaryWriter.prototype.beginDelimited_ = function(field) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + return {block: this.saveTempBuffer_(), length: this.totalLength_}; +}; + + +/** + * Ends a length-delimited block by encoding the _change_ in length of the + * buffer to the parent block and adds the number of bytes needed to encode + * that length to the total byte length. Note that 'parentLength' _must_ be the + * total length _after_ the field header was written in beginDelimited_ above. + * @param {!jspb.BinaryWriter.Bookmark_} bookmark + * @private + */ +jspb.BinaryWriter.prototype.endDelimited_ = function(bookmark) { + var messageLength = this.totalLength_ + this.temp_.length - bookmark.length; + goog.asserts.assert(messageLength >= 0); + + var bytes = 1; + while (messageLength > 127) { + bookmark.block.push((messageLength & 0x7f) | 0x80); + messageLength = messageLength >>> 7; + bytes++; + } + + bookmark.block.push(messageLength); + this.totalLength_ += bytes; +}; + + +/** + * Resets the writer, throwing away any accumulated buffers. + */ +jspb.BinaryWriter.prototype.reset = function() { + this.blocks_ = []; + this.temp_ = []; + this.totalLength_ = 0; + this.bookmarks_ = []; +}; + + +/** + * Converts the encoded data into a Uint8Array. + * @return {!Uint8Array} + */ +jspb.BinaryWriter.prototype.getResultBuffer = function() { + goog.asserts.assert(this.bookmarks_.length == 0); + + var flat = new Uint8Array(this.totalLength_ + this.temp_.length); + + var blocks = this.blocks_; + var blockCount = blocks.length; + var offset = 0; + + for (var i = 0; i < blockCount; i++) { + var block = blocks[i]; + flat.set(block, offset); + offset += block.length; + } + + flat.set(this.temp_, offset); + offset += this.temp_.length; + + // Post condition: `flattened` must have had every byte written. + goog.asserts.assert(offset == flat.length); + + // Replace our block list with the flattened block, which lets GC reclaim + // the temp blocks sooner. + this.blocks_ = [flat]; + this.temp_ = []; + + return flat; +}; + + +/** + * Converts the encoded data into a bas64-encoded string. + * @return {string} + */ +jspb.BinaryWriter.prototype.getResultBase64String = function() { + return goog.crypt.base64.encodeByteArray(this.getResultBuffer()); +}; + + +/** + * Begins a new sub-message. The client must call endSubMessage() when they're + * done. + * TODO(aappleby): Deprecated. Move callers to writeMessage(). + * @param {number} field The field number of the sub-message. + */ +jspb.BinaryWriter.prototype.beginSubMessage = function(field) { + this.bookmarks_.push(this.beginDelimited_(field)); +}; + + +/** + * Finishes a sub-message and packs it into the parent messages' buffer. + * TODO(aappleby): Deprecated. Move callers to writeMessage(). + */ +jspb.BinaryWriter.prototype.endSubMessage = function() { + goog.asserts.assert(this.bookmarks_.length >= 0); + this.endDelimited_(this.bookmarks_.pop()); +}; + + +/** + * Encodes a 32-bit unsigned integer into its wire-format varint representation + * and stores it in the buffer. + * @param {number} value The integer to convert. + */ +jspb.BinaryWriter.prototype.rawWriteUnsignedVarint32 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + + while (value > 127) { + this.temp_.push((value & 0x7f) | 0x80); + value = value >>> 7; + } + + this.temp_.push(value); +}; + + +/** + * Encodes a 32-bit signed integer into its wire-format varint representation + * and stores it in the buffer. + * @param {number} value The integer to convert. + */ +jspb.BinaryWriter.prototype.rawWriteSignedVarint32 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + if (value >= 0) { + this.rawWriteUnsignedVarint32(value); + return; + } + + // Write nine bytes with a _signed_ right shift so we preserve the sign bit. + for (var i = 0; i < 9; i++) { + this.temp_.push((value & 0x7f) | 0x80); + value = value >> 7; + } + + // The above loop writes out 63 bits, so the last byte is always the sign bit + // which is always set for negative numbers. + this.temp_.push(1); +}; + + +/** + * Encodes an unsigned 64-bit integer in 32:32 split representation into its + * wire-format varint representation and stores it in the buffer. + * @param {number} lowBits The low 32 bits of the int. + * @param {number} highBits The high 32 bits of the int. + */ +jspb.BinaryWriter.prototype.rawWriteSplitVarint = + function(lowBits, highBits) { + // Break the binary representation into chunks of 7 bits, set the 8th bit + // in each chunk if it's not the final chunk, and append to the result. + while (highBits > 0 || lowBits > 127) { + this.temp_.push((lowBits & 0x7f) | 0x80); + lowBits = ((lowBits >>> 7) | (highBits << 25)) >>> 0; + highBits = highBits >>> 7; + } + this.temp_.push(lowBits); +}; + + +/** + * Encodes a JavaScript integer into its wire-format varint representation and + * stores it in the buffer. Due to the way the varint encoding works this + * behaves correctly for both signed and unsigned integers, though integers + * that are not representable in 64 bits will still be truncated. + * @param {number} value The integer to convert. + */ +jspb.BinaryWriter.prototype.rawWriteVarint = function(value) { + goog.asserts.assert(value == Math.floor(value)); + jspb.utils.splitInt64(value); + this.rawWriteSplitVarint(jspb.utils.split64Low, + jspb.utils.split64High); +}; + + +/** + * Encodes a jspb.arith.{Int64,UInt64} instance into its wire-format + * varint representation and stores it in the buffer. Due to the way the varint + * encoding works this behaves correctly for both signed and unsigned integers, + * though integers that are not representable in 64 bits will still be + * truncated. + * @param {jspb.arith.Int64|jspb.arith.UInt64} value + */ +jspb.BinaryWriter.prototype.rawWriteVarintFromNum = function(value) { + this.rawWriteSplitVarint(value.lo, value.hi); +}; + + +/** + * Encodes a JavaScript integer into its wire-format, zigzag-encoded varint + * representation and stores it in the buffer. + * @param {number} value The integer to convert. + */ +jspb.BinaryWriter.prototype.rawWriteZigzagVarint32 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + this.rawWriteUnsignedVarint32(((value << 1) ^ (value >> 31)) >>> 0); +}; + + +/** + * Encodes a JavaScript integer into its wire-format, zigzag-encoded varint + * representation and stores it in the buffer. Integers not representable in 64 + * bits will be truncated. + * @param {number} value The integer to convert. + */ +jspb.BinaryWriter.prototype.rawWriteZigzagVarint = function(value) { + goog.asserts.assert(value == Math.floor(value)); + jspb.utils.splitZigzag64(value); + this.rawWriteSplitVarint(jspb.utils.split64Low, + jspb.utils.split64High); +}; + + +/** + * Writes a raw 8-bit unsigned integer to the buffer. Numbers outside the range + * [0,2^8) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteUint8 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= 0) && (value < 256)); + this.temp_.push((value >>> 0) & 0xFF); +}; + + +/** + * Writes a raw 16-bit unsigned integer to the buffer. Numbers outside the + * range [0,2^16) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteUint16 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= 0) && (value < 65536)); + this.temp_.push((value >>> 0) & 0xFF); + this.temp_.push((value >>> 8) & 0xFF); +}; + + +/** + * Writes a raw 32-bit unsigned integer to the buffer. Numbers outside the + * range [0,2^32) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteUint32 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= 0) && + (value < jspb.BinaryConstants.TWO_TO_32)); + this.temp_.push((value >>> 0) & 0xFF); + this.temp_.push((value >>> 8) & 0xFF); + this.temp_.push((value >>> 16) & 0xFF); + this.temp_.push((value >>> 24) & 0xFF); +}; + + +/** + * Writes a raw 64-bit unsigned integer to the buffer. Numbers outside the + * range [0,2^64) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteUint64 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= 0) && + (value < jspb.BinaryConstants.TWO_TO_64)); + jspb.utils.splitUint64(value); + this.rawWriteUint32(jspb.utils.split64Low); + this.rawWriteUint32(jspb.utils.split64High); +}; + + +/** + * Writes a raw 8-bit integer to the buffer. Numbers outside the range + * [-2^7,2^7) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteInt8 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= -128) && (value < 128)); + this.temp_.push((value >>> 0) & 0xFF); +}; + + +/** + * Writes a raw 16-bit integer to the buffer. Numbers outside the range + * [-2^15,2^15) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteInt16 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= -32768) && (value < 32768)); + this.temp_.push((value >>> 0) & 0xFF); + this.temp_.push((value >>> 8) & 0xFF); +}; + + +/** + * Writes a raw 32-bit integer to the buffer. Numbers outside the range + * [-2^31,2^31) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteInt32 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_31) && + (value < jspb.BinaryConstants.TWO_TO_31)); + this.temp_.push((value >>> 0) & 0xFF); + this.temp_.push((value >>> 8) & 0xFF); + this.temp_.push((value >>> 16) & 0xFF); + this.temp_.push((value >>> 24) & 0xFF); +}; + + +/** + * Writes a raw 64-bit integer to the buffer. Numbers outside the range + * [-2^63,2^63) will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteInt64 = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_63) && + (value < jspb.BinaryConstants.TWO_TO_63)); + jspb.utils.splitInt64(value); + this.rawWriteUint32(jspb.utils.split64Low); + this.rawWriteUint32(jspb.utils.split64High); +}; + + +/** + * Writes a raw single-precision floating point value to the buffer. Numbers + * requiring more than 32 bits of precision will be truncated. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteFloat = function(value) { + jspb.utils.splitFloat32(value); + this.rawWriteUint32(jspb.utils.split64Low); +}; + + +/** + * Writes a raw double-precision floating point value to the buffer. As this is + * the native format used by JavaScript, no precision will be lost. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteDouble = function(value) { + jspb.utils.splitFloat64(value); + this.rawWriteUint32(jspb.utils.split64Low); + this.rawWriteUint32(jspb.utils.split64High); +}; + + +/** + * Writes a raw boolean value to the buffer as a varint. + * @param {boolean} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteBool = function(value) { + goog.asserts.assert(goog.isBoolean(value)); + this.temp_.push(~~value); +}; + + +/** + * Writes an raw enum value to the buffer as a varint. + * @param {number} value The value to write. + */ +jspb.BinaryWriter.prototype.rawWriteEnum = function(value) { + goog.asserts.assert(value == Math.floor(value)); + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_31) && + (value < jspb.BinaryConstants.TWO_TO_31)); + this.rawWriteSignedVarint32(value); +}; + + +/** + * Writes a raw string value to the buffer. + * @param {string} string The string to write. + */ +jspb.BinaryWriter.prototype.rawWriteUtf8String = function(string) { + for (var i = 0; i < string.length; i++) { + this.temp_.push(string.charCodeAt(i)); + } +}; + + +/** + * Writes an arbitrary raw byte array to the buffer. + * @param {!Uint8Array} bytes The array of bytes to write. + */ +jspb.BinaryWriter.prototype.rawWriteBytes = function(bytes) { + this.appendUint8Array_(bytes); +}; + + +/** + * Writes an arbitrary raw byte array to the buffer. + * @param {!Uint8Array} bytes The array of bytes to write. + * @param {number} start The start of the range to write. + * @param {number} end The end of the range to write. + */ +jspb.BinaryWriter.prototype.rawWriteByteRange = function(bytes, start, end) { + this.appendUint8Array_(bytes.subarray(start, end)); +}; + + +/** + * Writes a 64-bit hash string (8 characters @ 8 bits of data each) to the + * buffer as a varint. + * @param {string} hash The hash to write. + */ +jspb.BinaryWriter.prototype.rawWriteVarintHash64 = function(hash) { + jspb.utils.splitHash64(hash); + this.rawWriteSplitVarint(jspb.utils.split64Low, + jspb.utils.split64High); +}; + + +/** + * Writes a 64-bit hash string (8 characters @ 8 bits of data each) to the + * buffer as a fixed64. + * @param {string} hash The hash to write. + */ +jspb.BinaryWriter.prototype.rawWriteFixedHash64 = function(hash) { + jspb.utils.splitHash64(hash); + this.rawWriteUint32(jspb.utils.split64Low); + this.rawWriteUint32(jspb.utils.split64High); +}; + + +/** + * Encodes a (field number, wire type) tuple into a wire-format field header + * and stores it in the buffer as a varint. + * @param {number} field The field number. + * @param {number} wireType The wire-type of the field, as specified in the + * protocol buffer documentation. + * @private + */ +jspb.BinaryWriter.prototype.rawWriteFieldHeader_ = + function(field, wireType) { + goog.asserts.assert(field >= 1 && field == Math.floor(field)); + var x = field * 8 + wireType; + this.rawWriteUnsignedVarint32(x); +}; + + +/** + * Writes a field of any valid scalar type to the binary stream. + * @param {jspb.BinaryConstants.FieldType} fieldType + * @param {number} field + * @param {jspb.AnyFieldType} value + */ +jspb.BinaryWriter.prototype.writeAny = function(fieldType, field, value) { + var fieldTypes = jspb.BinaryConstants.FieldType; + switch (fieldType) { + case fieldTypes.DOUBLE: + this.writeDouble(field, /** @type {number} */(value)); + return; + case fieldTypes.FLOAT: + this.writeFloat(field, /** @type {number} */(value)); + return; + case fieldTypes.INT64: + this.writeInt64(field, /** @type {number} */(value)); + return; + case fieldTypes.UINT64: + this.writeUint64(field, /** @type {number} */(value)); + return; + case fieldTypes.INT32: + this.writeInt32(field, /** @type {number} */(value)); + return; + case fieldTypes.FIXED64: + this.writeFixed64(field, /** @type {number} */(value)); + return; + case fieldTypes.FIXED32: + this.writeFixed32(field, /** @type {number} */(value)); + return; + case fieldTypes.BOOL: + this.writeBool(field, /** @type {boolean} */(value)); + return; + case fieldTypes.STRING: + this.writeString(field, /** @type {string} */(value)); + return; + case fieldTypes.GROUP: + goog.asserts.fail('Group field type not supported in writeAny()'); + return; + case fieldTypes.MESSAGE: + goog.asserts.fail('Message field type not supported in writeAny()'); + return; + case fieldTypes.BYTES: + this.writeBytes(field, /** @type {?Uint8Array} */(value)); + return; + case fieldTypes.UINT32: + this.writeUint32(field, /** @type {number} */(value)); + return; + case fieldTypes.ENUM: + this.writeEnum(field, /** @type {number} */(value)); + return; + case fieldTypes.SFIXED32: + this.writeSfixed32(field, /** @type {number} */(value)); + return; + case fieldTypes.SFIXED64: + this.writeSfixed64(field, /** @type {number} */(value)); + return; + case fieldTypes.SINT32: + this.writeSint32(field, /** @type {number} */(value)); + return; + case fieldTypes.SINT64: + this.writeSint64(field, /** @type {number} */(value)); + return; + case fieldTypes.FHASH64: + this.writeFixedHash64(field, /** @type {string} */(value)); + return; + case fieldTypes.VHASH64: + this.writeVarintHash64(field, /** @type {string} */(value)); + return; + default: + goog.asserts.fail('Invalid field type in writeAny()'); + return; + } +}; + + +/** + * Writes a varint field to the buffer without range checking. + * @param {number} field The field number. + * @param {number?} value The value to write. + * @private + */ +jspb.BinaryWriter.prototype.writeUnsignedVarint32_ = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteSignedVarint32(value); +}; + + +/** + * Writes a varint field to the buffer without range checking. + * @param {number} field The field number. + * @param {number?} value The value to write. + * @private + */ +jspb.BinaryWriter.prototype.writeSignedVarint32_ = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteSignedVarint32(value); +}; + + +/** + * Writes a varint field to the buffer without range checking. + * @param {number} field The field number. + * @param {number?} value The value to write. + * @private + */ +jspb.BinaryWriter.prototype.writeVarint_ = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteVarint(value); +}; + + +/** + * Writes a zigzag varint field to the buffer without range checking. + * @param {number} field The field number. + * @param {number?} value The value to write. + * @private + */ +jspb.BinaryWriter.prototype.writeZigzagVarint32_ = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteZigzagVarint32(value); +}; + + +/** + * Writes a zigzag varint field to the buffer without range checking. + * @param {number} field The field number. + * @param {number?} value The value to write. + * @private + */ +jspb.BinaryWriter.prototype.writeZigzagVarint_ = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteZigzagVarint(value); +}; + + +/** + * Writes an int32 field to the buffer. Numbers outside the range [-2^31,2^31) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeInt32 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_31) && + (value < jspb.BinaryConstants.TWO_TO_31)); + this.writeSignedVarint32_(field, value); +}; + + +/** + * Writes an int32 field represented as a string to the buffer. Numbers outside + * the range [-2^31,2^31) will be truncated. + * @param {number} field The field number. + * @param {string?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeInt32String = function(field, value) { + if (value == null) return; + var intValue = /** {number} */ parseInt(value, 10); + goog.asserts.assert((intValue >= -jspb.BinaryConstants.TWO_TO_31) && + (intValue < jspb.BinaryConstants.TWO_TO_31)); + this.writeSignedVarint32_(field, intValue); +}; + + +/** + * Writes an int64 field to the buffer. Numbers outside the range [-2^63,2^63) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeInt64 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_63) && + (value < jspb.BinaryConstants.TWO_TO_63)); + this.writeVarint_(field, value); +}; + + +/** + * Writes a int64 field (with value as a string) to the buffer. + * @param {number} field The field number. + * @param {string?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeInt64String = function(field, value) { + if (value == null) return; + var num = jspb.arith.Int64.fromString(value); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteVarintFromNum(num); +}; + + +/** + * Writes a uint32 field to the buffer. Numbers outside the range [0,2^32) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeUint32 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= 0) && + (value < jspb.BinaryConstants.TWO_TO_32)); + this.writeUnsignedVarint32_(field, value); +}; + + +/** + * Writes a uint32 field represented as a string to the buffer. Numbers outside + * the range [0,2^32) will be truncated. + * @param {number} field The field number. + * @param {string?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeUint32String = function(field, value) { + if (value == null) return; + var intValue = /** {number} */ parseInt(value, 10); + goog.asserts.assert((intValue >= 0) && + (intValue < jspb.BinaryConstants.TWO_TO_32)); + this.writeUnsignedVarint32_(field, intValue); +}; + + +/** + * Writes a uint64 field to the buffer. Numbers outside the range [0,2^64) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeUint64 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= 0) && + (value < jspb.BinaryConstants.TWO_TO_64)); + this.writeVarint_(field, value); +}; + + +/** + * Writes a uint64 field (with value as a string) to the buffer. + * @param {number} field The field number. + * @param {string?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeUint64String = function(field, value) { + if (value == null) return; + var num = jspb.arith.UInt64.fromString(value); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteVarintFromNum(num); +}; + + +/** + * Writes a sint32 field to the buffer. Numbers outside the range [-2^31,2^31) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeSint32 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_31) && + (value < jspb.BinaryConstants.TWO_TO_31)); + this.writeZigzagVarint32_(field, value); +}; + + +/** + * Writes a sint64 field to the buffer. Numbers outside the range [-2^63,2^63) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeSint64 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_63) && + (value < jspb.BinaryConstants.TWO_TO_63)); + this.writeZigzagVarint_(field, value); +}; + + +/** + * Writes a fixed32 field to the buffer. Numbers outside the range [0,2^32) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeFixed32 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= 0) && + (value < jspb.BinaryConstants.TWO_TO_32)); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED32); + this.rawWriteUint32(value); +}; + + +/** + * Writes a fixed64 field to the buffer. Numbers outside the range [0,2^64) + * will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeFixed64 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= 0) && + (value < jspb.BinaryConstants.TWO_TO_64)); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED64); + this.rawWriteUint64(value); +}; + + +/** + * Writes a sfixed32 field to the buffer. Numbers outside the range + * [-2^31,2^31) will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeSfixed32 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_31) && + (value < jspb.BinaryConstants.TWO_TO_31)); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED32); + this.rawWriteInt32(value); +}; + + +/** + * Writes a sfixed64 field to the buffer. Numbers outside the range + * [-2^63,2^63) will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeSfixed64 = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_63) && + (value < jspb.BinaryConstants.TWO_TO_63)); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED64); + this.rawWriteInt64(value); +}; + + +/** + * Writes a single-precision floating point field to the buffer. Numbers + * requiring more than 32 bits of precision will be truncated. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeFloat = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED32); + this.rawWriteFloat(value); +}; + + +/** + * Writes a double-precision floating point field to the buffer. As this is the + * native format used by JavaScript, no precision will be lost. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeDouble = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED64); + this.rawWriteDouble(value); +}; + + +/** + * Writes a boolean field to the buffer. + * @param {number} field The field number. + * @param {boolean?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeBool = function(field, value) { + if (value == null) return; + goog.asserts.assert(goog.isBoolean(value)); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.temp_.push(~~value); +}; + + +/** + * Writes an enum field to the buffer. + * @param {number} field The field number. + * @param {number?} value The value to write. + */ +jspb.BinaryWriter.prototype.writeEnum = function(field, value) { + if (value == null) return; + goog.asserts.assert((value >= -jspb.BinaryConstants.TWO_TO_31) && + (value < jspb.BinaryConstants.TWO_TO_31)); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteSignedVarint32(value); +}; + + +/** + * Writes a string field to the buffer. + * @param {number} field The field number. + * @param {string?} value The string to write. + */ +jspb.BinaryWriter.prototype.writeString = function(field, value) { + if (value == null) return; + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + + // Conversion loop swiped from goog.crypt.stringToUtf8ByteArray. Note that + // 'bytes' will be at least as long as 'value', but could be longer if we + // need to unpack unicode characters. + var bytes = []; + for (var i = 0; i < value.length; i++) { + var c = value.charCodeAt(i); + if (c < 128) { + bytes.push(c); + } else if (c < 2048) { + bytes.push((c >> 6) | 192); + bytes.push((c & 63) | 128); + } else { + bytes.push((c >> 12) | 224); + bytes.push(((c >> 6) & 63) | 128); + bytes.push((c & 63) | 128); + } + } + + this.rawWriteUnsignedVarint32(bytes.length); + this.appendArray_(bytes); +}; + + +/** + * Writes an arbitrary byte field to the buffer. Note - to match the behavior + * of the C++ implementation, empty byte arrays _are_ serialized. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {jspb.ByteSource} value The array of bytes to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + * @param {boolean=} opt_stringIsRawBytes If `value` is a string, interpret it + * as a series of raw bytes (codepoints 0--255 inclusive) rather than base64 + * data. + */ +jspb.BinaryWriter.prototype.writeBytes = + function(field, value, opt_buffer, opt_start, opt_end, + opt_stringIsRawBytes) { + if (value != null) { + this.rawWriteFieldHeader_(field, + jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length); + this.rawWriteBytes( + jspb.utils.byteSourceToUint8Array(value, opt_stringIsRawBytes)); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an arbitrary byte field to the buffer, with `opt_stringIsRawBytes` + * flag implicitly true. + * @param {number} field + * @param {jspb.ByteSource} value The array of bytes to write. + */ +jspb.BinaryWriter.prototype.writeBytesRawString = function(field, value) { + this.writeBytes(field, value, null, null, null, true); +}; + + +/** + * Writes a message to the buffer. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @template MessageType + * @param {number} field The field number. + * @param {?MessageType} value The message to write. + * @param {!jspb.WriterFunction} writerCallback Will be invoked with the value + * to write and the writer to write it with. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writeMessage = + function(field, value, writerCallback, opt_buffer, opt_start, opt_end) { + if (value !== null) { + var bookmark = this.beginDelimited_(field); + + writerCallback(value, this); + + this.endDelimited_(bookmark); + } else if (opt_buffer && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes a group message to the buffer. + * + * @template MessageType + * @param {number} field The field number. + * @param {?MessageType} value The message to write, wrapped with START_GROUP / + * END_GROUP tags. Will be a no-op if 'value' is null. + * @param {!jspb.WriterFunction} writerCallback Will be invoked with the value + * to write and the writer to write it with. + */ +jspb.BinaryWriter.prototype.writeGroup = + function(field, value, writerCallback) { + if (value) { + this.rawWriteFieldHeader_( + field, jspb.BinaryConstants.WireType.START_GROUP); + writerCallback(value, this); + this.rawWriteFieldHeader_( + field, jspb.BinaryConstants.WireType.END_GROUP); + } +}; + + +/** + * Writes a 64-bit hash string field (8 characters @ 8 bits of data each) to + * the buffer. + * @param {number} field The field number. + * @param {string?} value The hash string. + */ +jspb.BinaryWriter.prototype.writeFixedHash64 = function(field, value) { + if (value == null) return; + goog.asserts.assert(value.length == 8); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.FIXED64); + this.rawWriteFixedHash64(value); +}; + + +/** + * Writes a 64-bit hash string field (8 characters @ 8 bits of data each) to + * the buffer. + * @param {number} field The field number. + * @param {string?} value The hash string. + */ +jspb.BinaryWriter.prototype.writeVarintHash64 = function(field, value) { + if (value == null) return; + goog.asserts.assert(value.length == 8); + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.VARINT); + this.rawWriteVarintHash64(value); +}; + + +/** + * Writes an array of numbers to the buffer as a repeated varint field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @private + */ +jspb.BinaryWriter.prototype.writeRepeatedUnsignedVarint32_ = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeUnsignedVarint32_(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated varint field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @private + */ +jspb.BinaryWriter.prototype.writeRepeatedSignedVarint32_ = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeSignedVarint32_(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated varint field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @private + */ +jspb.BinaryWriter.prototype.writeRepeatedVarint_ = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeVarint_(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated zigzag field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @private + */ +jspb.BinaryWriter.prototype.writeRepeatedZigzag32_ = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeZigzagVarint32_(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated zigzag field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @private + */ +jspb.BinaryWriter.prototype.writeRepeatedZigzag_ = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeZigzagVarint_(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated 32-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedInt32 = + jspb.BinaryWriter.prototype.writeRepeatedSignedVarint32_; + + +/** + * Writes an array of numbers formatted as strings to the buffer as a repeated + * 32-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedInt32String = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeInt32String(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated 64-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedInt64 = + jspb.BinaryWriter.prototype.writeRepeatedVarint_; + + +/** + * Writes an array of numbers formatted as strings to the buffer as a repeated + * 64-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedInt64String = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeInt64String(field, value[i]); + } +}; + + +/** + * Writes an array numbers to the buffer as a repeated unsigned 32-bit int + * field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedUint32 = + jspb.BinaryWriter.prototype.writeRepeatedUnsignedVarint32_; + + +/** + * Writes an array of numbers formatted as strings to the buffer as a repeated + * unsigned 32-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedUint32String = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeUint32String(field, value[i]); + } +}; + + +/** + * Writes an array numbers to the buffer as a repeated unsigned 64-bit int + * field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedUint64 = + jspb.BinaryWriter.prototype.writeRepeatedVarint_; + + +/** + * Writes an array of numbers formatted as strings to the buffer as a repeated + * unsigned 64-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedUint64String = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeUint64String(field, value[i]); + } +}; + + +/** + * Writes an array numbers to the buffer as a repeated signed 32-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedSint32 = + jspb.BinaryWriter.prototype.writeRepeatedZigzag32_; + + +/** + * Writes an array numbers to the buffer as a repeated signed 64-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedSint64 = + jspb.BinaryWriter.prototype.writeRepeatedZigzag_; + + +/** + * Writes an array of numbers to the buffer as a repeated fixed32 field. This + * works for both signed and unsigned fixed32s. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedFixed32 = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeFixed32(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated fixed64 field. This + * works for both signed and unsigned fixed64s. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedFixed64 = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeFixed64(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated sfixed32 field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedSfixed32 = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeSfixed32(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated sfixed64 field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedSfixed64 = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeSfixed64(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated float field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedFloat = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeFloat(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a repeated double field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedDouble = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeDouble(field, value[i]); + } +}; + + +/** + * Writes an array of booleans to the buffer as a repeated bool field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedBool = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeBool(field, value[i]); + } +}; + + +/** + * Writes an array of enums to the buffer as a repeated enum field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedEnum = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeEnum(field, value[i]); + } +}; + + +/** + * Writes an array of strings to the buffer as a repeated string field. + * @param {number} field The field number. + * @param {?Array.} value The array of strings to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedString = function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeString(field, value[i]); + } +}; + + +/** + * Writes an array of arbitrary byte fields to the buffer. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value + * The arrays of arrays of bytes to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + * @param {boolean=} opt_stringIsRawBytes Any values that are strings are + * interpreted as raw bytes rather than base64 data. + */ +jspb.BinaryWriter.prototype.writeRepeatedBytes = + function(field, value, opt_buffer, opt_start, opt_end, + opt_stringIsRawBytes) { + if (value != null) { + for (var i = 0; i < value.length; i++) { + this.writeBytes(field, value[i], null, null, null, opt_stringIsRawBytes); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of arbitrary byte fields to the buffer, with + * `opt_stringIsRawBytes` implicitly true. + * @param {number} field + * @param {?Array.} value + */ +jspb.BinaryWriter.prototype.writeRepeatedBytesRawString = + function(field, value) { + this.writeRepeatedBytes(field, value, null, null, null, true); +}; + + +/** + * Writes an array of messages to the buffer. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @template MessageType + * @param {number} field The field number. + * @param {?Array.} value The array of messages to + * write. + * @param {!jspb.WriterFunction} writerCallback Will be invoked with the value + * to write and the writer to write it with. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writeRepeatedMessage = + function(field, value, writerCallback, opt_buffer, opt_start, opt_end) { + if (value) { + for (var i = 0; i < value.length; i++) { + var bookmark = this.beginDelimited_(field); + + writerCallback(value[i], this); + + this.endDelimited_(bookmark); + } + } else if (opt_buffer && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of group messages to the buffer. + * + * @template MessageType + * @param {number} field The field number. + * @param {?Array.} value The array of messages to + * write. + * @param {!jspb.WriterFunction} writerCallback Will be invoked with the value + * to write and the writer to write it with. + */ +jspb.BinaryWriter.prototype.writeRepeatedGroup = + function(field, value, writerCallback) { + if (value) { + for (var i = 0; i < value.length; i++) { + this.rawWriteFieldHeader_( + field, jspb.BinaryConstants.WireType.START_GROUP); + writerCallback(value[i], this); + this.rawWriteFieldHeader_( + field, jspb.BinaryConstants.WireType.END_GROUP); + } + } +}; + + +/** + * Writes a 64-bit hash string field (8 characters @ 8 bits of data each) to + * the buffer. + * @param {number} field The field number. + * @param {?Array.} value The array of hashes to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedFixedHash64 = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeFixedHash64(field, value[i]); + } +}; + + +/** + * Writes a repeated 64-bit hash string field (8 characters @ 8 bits of data + * each) to the buffer. + * @param {number} field The field number. + * @param {?Array.} value The array of hashes to write. + */ +jspb.BinaryWriter.prototype.writeRepeatedVarintHash64 = + function(field, value) { + if (value == null) return; + for (var i = 0; i < value.length; i++) { + this.writeVarintHash64(field, value[i]); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed varint field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + * @private + */ +jspb.BinaryWriter.prototype.writePackedUnsignedVarint32_ = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteUnsignedVarint32(value[i]); + } + this.endDelimited_(bookmark); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed varint field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + * @private + */ +jspb.BinaryWriter.prototype.writePackedSignedVarint32_ = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteSignedVarint32(value[i]); + } + this.endDelimited_(bookmark); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed varint field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + * @private + */ +jspb.BinaryWriter.prototype.writePackedVarint_ = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteVarint(value[i]); + } + this.endDelimited_(bookmark); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed zigzag field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + * @private + */ +jspb.BinaryWriter.prototype.writePackedZigzag_ = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteZigzagVarint(value[i]); + } + this.endDelimited_(bookmark); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed 32-bit int field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedInt32 = + jspb.BinaryWriter.prototype.writePackedSignedVarint32_; + + +/** + * Writes an array of numbers represented as strings to the buffer as a packed + * 32-bit int field. + * @param {number} field + * @param {?Array.} value + */ +jspb.BinaryWriter.prototype.writePackedInt32String = function(field, value) { + if (value == null || !value.length) return; + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteSignedVarint32(parseInt(value[i], 10)); + } + this.endDelimited_(bookmark); +}; + + +/** + * Writes an array of numbers to the buffer as a packed 64-bit int field. + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedInt64 = + jspb.BinaryWriter.prototype.writePackedVarint_; + + +/** + * Writes an array of numbers represented as strings to the buffer as a packed + * 64-bit int field. + * @param {number} field + * @param {?Array.} value + */ +jspb.BinaryWriter.prototype.writePackedInt64String = + function(field, value) { + if (value == null || !value.length) return; + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + var num = jspb.arith.Int64.fromString(value[i]); + this.rawWriteVarintFromNum(num); + } + this.endDelimited_(bookmark); +}; + + +/** + * Writes an array numbers to the buffer as a packed unsigned 32-bit int field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedUint32 = + jspb.BinaryWriter.prototype.writePackedUnsignedVarint32_; + + +/** + * Writes an array of numbers represented as strings to the buffer as a packed + * unsigned 32-bit int field. + * @param {number} field + * @param {?Array.} value + */ +jspb.BinaryWriter.prototype.writePackedUint32String = + function(field, value) { + if (value == null || !value.length) return; + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteUnsignedVarint32(parseInt(value[i], 10)); + } + this.endDelimited_(bookmark); +}; + + +/** + * Writes an array numbers to the buffer as a packed unsigned 64-bit int field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedUint64 = + jspb.BinaryWriter.prototype.writePackedVarint_; + + +/** + * Writes an array of numbers represented as strings to the buffer as a packed + * unsigned 64-bit int field. + * @param {number} field + * @param {?Array.} value + */ +jspb.BinaryWriter.prototype.writePackedUint64String = + function(field, value) { + if (value == null || !value.length) return; + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + var num = jspb.arith.UInt64.fromString(value[i]); + this.rawWriteVarintFromNum(num); + } + this.endDelimited_(bookmark); +}; + + +/** + * Writes an array numbers to the buffer as a packed signed 32-bit int field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedSint32 = + jspb.BinaryWriter.prototype.writePackedZigzag_; + + +/** + * Writes an array numbers to the buffer as a packed signed 64-bit int field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedSint64 = + jspb.BinaryWriter.prototype.writePackedZigzag_; + + +/** + * Writes an array of numbers to the buffer as a packed fixed32 field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedFixed32 = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 4); + for (var i = 0; i < value.length; i++) { + this.rawWriteUint32(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed fixed64 field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedFixed64 = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 8); + for (var i = 0; i < value.length; i++) { + this.rawWriteUint64(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed sfixed32 field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedSfixed32 = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 4); + for (var i = 0; i < value.length; i++) { + this.rawWriteInt32(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed sfixed64 field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedSfixed64 = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 8); + for (var i = 0; i < value.length; i++) { + this.rawWriteInt64(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed float field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedFloat = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 4); + for (var i = 0; i < value.length; i++) { + this.rawWriteFloat(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of numbers to the buffer as a packed double field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedDouble = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 8); + for (var i = 0; i < value.length; i++) { + this.rawWriteDouble(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of booleans to the buffer as a packed bool field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedBool = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length); + for (var i = 0; i < value.length; i++) { + this.rawWriteBool(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes an array of enums to the buffer as a packed enum field. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of ints to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedEnum = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteEnum(value[i]); + } + this.endDelimited_(bookmark); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes a 64-bit hash string field (8 characters @ 8 bits of data each) to + * the buffer. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of hashes to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedFixedHash64 = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + this.rawWriteFieldHeader_(field, jspb.BinaryConstants.WireType.DELIMITED); + this.rawWriteUnsignedVarint32(value.length * 8); + for (var i = 0; i < value.length; i++) { + this.rawWriteFixedHash64(value[i]); + } + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; + + +/** + * Writes a 64-bit hash string field (8 characters @ 8 bits of data each) to + * the buffer. + * + * If 'value' is null, this method will try and copy the pre-serialized value + * in 'opt_buffer' if present. + * + * @param {number} field The field number. + * @param {?Array.} value The array of hashes to write. + * @param {?Uint8Array=} opt_buffer A buffer containing pre-packed values. + * @param {?number=} opt_start The starting point in the above buffer. + * @param {?number=} opt_end The ending point in the above buffer. + */ +jspb.BinaryWriter.prototype.writePackedVarintHash64 = + function(field, value, opt_buffer, opt_start, opt_end) { + if (value != null && value.length) { + var bookmark = this.beginDelimited_(field); + for (var i = 0; i < value.length; i++) { + this.rawWriteVarintHash64(value[i]); + } + this.endDelimited_(bookmark); + } else if ((opt_buffer != null) && (opt_start != null) && (opt_end != null)) { + this.rawWriteByteRange(opt_buffer, opt_start, opt_end); + } +}; diff --git a/js/binary/writer_test.js b/js/binary/writer_test.js new file mode 100644 index 00000000..54d37a21 --- /dev/null +++ b/js/binary/writer_test.js @@ -0,0 +1,123 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Test cases for jspb's binary protocol buffer writer. In + * practice BinaryWriter is used to drive the Decoder and Reader test cases, + * so only writer-specific tests are here. + * + * Test suite is written using Jasmine -- see http://jasmine.github.io/ + * + * @author aappleby@google.com (Austin Appleby) + */ + +goog.require('goog.crypt'); +goog.require('goog.testing.asserts'); +goog.require('jspb.BinaryWriter'); + + +/** + * @param {function()} func This function should throw an error when run. + */ +function assertFails(func) { + var e = assertThrows(func); + console.log(e); + //assertNotNull(e.toString().match(/Error/)); +} + + +describe('binaryWriterTest', function() { + /** + * Verifies that misuse of the writer class triggers assertions. + */ + it('testWriteErrors', function() { + // Submessages with invalid field indices should assert. + var writer = new jspb.BinaryWriter(); + var dummyMessage = /** @type {!jspb.BinaryMessage} */({}); + + assertFails(function() { + writer.writeMessage(-1, dummyMessage, goog.nullFunction); + }); + + // Writing invalid field indices should assert. + writer = new jspb.BinaryWriter(); + assertFails(function() {writer.writeUint64(-1, 1);}); + + // Writing out-of-range field values should assert. + writer = new jspb.BinaryWriter(); + + assertFails(function() {writer.writeInt32(1, -Infinity);}); + assertFails(function() {writer.writeInt32(1, Infinity);}); + + assertFails(function() {writer.writeInt64(1, -Infinity);}); + assertFails(function() {writer.writeInt64(1, Infinity);}); + + assertFails(function() {writer.writeUint32(1, -1);}); + assertFails(function() {writer.writeUint32(1, Infinity);}); + + assertFails(function() {writer.writeUint64(1, -1);}); + assertFails(function() {writer.writeUint64(1, Infinity);}); + + assertFails(function() {writer.writeSint32(1, -Infinity);}); + assertFails(function() {writer.writeSint32(1, Infinity);}); + + assertFails(function() {writer.writeSint64(1, -Infinity);}); + assertFails(function() {writer.writeSint64(1, Infinity);}); + + assertFails(function() {writer.writeFixed32(1, -1);}); + assertFails(function() {writer.writeFixed32(1, Infinity);}); + + assertFails(function() {writer.writeFixed64(1, -1);}); + assertFails(function() {writer.writeFixed64(1, Infinity);}); + + assertFails(function() {writer.writeSfixed32(1, -Infinity);}); + assertFails(function() {writer.writeSfixed32(1, Infinity);}); + + assertFails(function() {writer.writeSfixed64(1, -Infinity);}); + assertFails(function() {writer.writeSfixed64(1, Infinity);}); + }); + + + /** + * Basic test of retrieving the result as a Uint8Array buffer + */ + it('testGetResultBuffer', function() { + var expected = '0864120b48656c6c6f20776f726c641a0301020320c801'; + + var writer = new jspb.BinaryWriter(); + writer.writeUint32(1, 100); + writer.writeString(2, 'Hello world'); + writer.writeBytes(3, new Uint8Array([1, 2, 3])); + writer.writeUint32(4, 200); + + var buffer = writer.getResultBuffer(); + assertEquals(expected, goog.crypt.byteArrayToHex(buffer)); + }); +}); diff --git a/js/data.proto b/js/data.proto new file mode 100644 index 00000000..74a8a994 --- /dev/null +++ b/js/data.proto @@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: mwr@google.com (Mark Rawling) + +syntax = "proto2"; + +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + +package jspb.test; + +// legacy data, must be nested +message data { + message NestedData { + required string str = 1; + } +} + +// new data, does not require nesting +message UnnestedData { + required string str = 1; +} + diff --git a/js/debug.js b/js/debug.js new file mode 100644 index 00000000..48389118 --- /dev/null +++ b/js/debug.js @@ -0,0 +1,140 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Utilities to debug JSPB based proto objects. + */ + +goog.provide('jspb.debug'); + +goog.require('goog.array'); +goog.require('goog.asserts'); +goog.require('goog.object'); +goog.require('jspb.Message'); + + +/** + * Turns a proto into a human readable object that can i.e. be written to the + * console: {@code console.log(jspb.debug.dump(myProto))}. + * This function makes a best effort and may not work in all cases. It will not + * work in obfuscated and or optimized code. + * Use this in environments where {@see jspb.Message.prototype.toObject} is + * not available for code size reasons. + * @param {jspb.Message} message A jspb.Message. + * @return {Object} + */ +jspb.debug.dump = function(message) { + if (!goog.DEBUG) { + return null; + } + goog.asserts.assert(message instanceof jspb.Message, + 'jspb.Message instance expected'); + /** @type {Object} */ + var object = message; + goog.asserts.assert(object['getExtension'], + 'Only unobfuscated and unoptimized compilation modes supported.'); + return /** @type {Object} */ (jspb.debug.dump_(message)); +}; + + +/** + * Recursively introspects a message and the values its getters return to + * make a best effort in creating a human readable representation of the + * message. + * @param {*} thing A jspb.Message, Array or primitive type to dump. + * @return {*} + * @private + */ +jspb.debug.dump_ = function(thing) { + var type = goog.typeOf(thing); + if (type == 'number' || type == 'string' || type == 'boolean' || + type == 'null' || type == 'undefined') { + return thing; + } + if (type == 'array') { + goog.asserts.assertArray(thing); + return goog.array.map(thing, jspb.debug.dump_); + } + var message = thing; // Copy because we don't want type inference on thing. + goog.asserts.assert(message instanceof jspb.Message, + 'Only messages expected: ' + thing); + var ctor = message.constructor; + var messageName = ctor.name || ctor.displayName; + var object = { + '$name': messageName + }; + for (var name in ctor.prototype) { + var match = /^get([A-Z]\w*)/.exec(name); + if (match && name != 'getExtension' && + name != 'getJsPbMessageId') { + var val = thing[name](); + if (val != null) { + object[jspb.debug.formatFieldName_(match[1])] = jspb.debug.dump_(val); + } + } + } + if (COMPILED && thing['extensionObject_']) { + object['$extensions'] = 'Recursive dumping of extensions not supported ' + + 'in compiled code. Switch to uncompiled or dump extension object ' + + 'directly'; + return object; + } + var extensionsObject; + for (var id in ctor['extensions']) { + if (/^\d+$/.test(id)) { + var ext = ctor['extensions'][id]; + var extVal = thing.getExtension(ext); + var fieldName = goog.object.getKeys(ext.fieldName)[0]; + if (extVal != null) { + if (!extensionsObject) { + extensionsObject = object['$extensions'] = {}; + } + extensionsObject[jspb.debug.formatFieldName_(fieldName)] = + jspb.debug.dump_(extVal); + } + } + } + return object; +}; + + +/** + * Formats a field name for output as camelCase. + * + * @param {string} name Name of the field. + * @return {string} + * @private + */ +jspb.debug.formatFieldName_ = function(name) { + // Name may be in TitleCase. + return name.replace(/^[A-Z]/, function(c) { + return c.toLowerCase(); + }); +}; diff --git a/js/debug_test.js b/js/debug_test.js new file mode 100644 index 00000000..615fc7c6 --- /dev/null +++ b/js/debug_test.js @@ -0,0 +1,101 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +goog.setTestOnly(); + +goog.require('goog.testing.asserts'); +goog.require('jspb.debug'); +goog.require('proto.jspb.test.HasExtensions'); +goog.require('proto.jspb.test.IsExtension'); +goog.require('proto.jspb.test.Simple1'); + + + +describe('debugTest', function() { + it('testSimple1', function() { + if (COMPILED) { + return; + } + var message = new proto.jspb.test.Simple1(); + message.setAString('foo'); + assertObjectEquals({ + $name: 'proto.jspb.test.Simple1', + 'aString': 'foo', + 'aRepeatedStringList': [] + }, jspb.debug.dump(message)); + + message.setABoolean(true); + message.setARepeatedStringList(['1', '2']); + + assertObjectEquals({ + $name: 'proto.jspb.test.Simple1', + 'aString': 'foo', + 'aRepeatedStringList': ['1', '2'], + 'aBoolean': true + }, jspb.debug.dump(message)); + + message.setAString(undefined); + + assertObjectEquals({ + $name: 'proto.jspb.test.Simple1', + 'aRepeatedStringList': ['1', '2'], + 'aBoolean': true + }, jspb.debug.dump(message)); + }); + + + it('testExtensions', function() { + if (COMPILED) { + return; + } + var extension = new proto.jspb.test.IsExtension(); + extension.setExt1('ext1field'); + var extendable = new proto.jspb.test.HasExtensions(); + extendable.setStr1('v1'); + extendable.setStr2('v2'); + extendable.setStr3('v3'); + extendable.setExtension(proto.jspb.test.IsExtension.extField, extension); + + assertObjectEquals({ + '$name': 'proto.jspb.test.HasExtensions', + 'str1': 'v1', + 'str2': 'v2', + 'str3': 'v3', + '$extensions': { + 'extField': { + '$name': 'proto.jspb.test.IsExtension', + 'ext1': 'ext1field' + }, + 'repeatedSimpleList': [] + } + }, jspb.debug.dump(extendable)); + }); + +}); diff --git a/js/message.js b/js/message.js new file mode 100644 index 00000000..cef9aefd --- /dev/null +++ b/js/message.js @@ -0,0 +1,1125 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Definition of jspb.Message. + * + * @author mwr@google.com (Mark Rawling) + */ + +goog.provide('jspb.ExtensionFieldInfo'); +goog.provide('jspb.Message'); + +goog.require('goog.array'); +goog.require('goog.asserts'); +goog.require('goog.json'); +goog.require('goog.object'); + +// Not needed in compilation units that have no protos with xids. +goog.forwardDeclare('xid.String'); + + + +/** + * Stores information for a single extension field. + * + * For example, an extension field defined like so: + * + * extend BaseMessage { + * optional MyMessage my_field = 123; + * } + * + * will result in an ExtensionFieldInfo object with these properties: + * + * { + * fieldIndex: 123, + * fieldName: {my_field_renamed: 0}, + * ctor: proto.example.MyMessage, + * toObjectFn: proto.example.MyMessage.toObject, + * isRepeated: 0 + * } + * + * We include `toObjectFn` to allow the JSCompiler to perform dead-code removal + * on unused toObject() methods. + * + * If an extension field is primitive, ctor and toObjectFn will be null. + * isRepeated should be 0 or 1. + * + * binary{Reader,Writer}Fn and (if message type) binaryMessageSerializeFn are + * always provided. binaryReaderFn and binaryWriterFn are references to the + * appropriate methods on BinaryReader/BinaryWriter to read/write the value of + * this extension, and binaryMessageSerializeFn is a reference to the message + * class's .serializeBinary method, if available. + * + * @param {number} fieldNumber + * @param {Object} fieldName This has the extension field name as a property. + * @param {?function(new: jspb.Message, Array=)} ctor + * @param {?function((boolean|undefined),!jspb.Message):!Object} toObjectFn + * @param {number} isRepeated + * @param {?function(number,?)=} opt_binaryReaderFn + * @param {?function(number,?)|function(number,?,?,?,?,?)=} opt_binaryWriterFn + * @param {?function(?,?)=} opt_binaryMessageSerializeFn + * @param {?function(?,?)=} opt_binaryMessageDeserializeFn + * @param {?boolean=} opt_isPacked + * @constructor + * @struct + * @template T + */ +jspb.ExtensionFieldInfo = function(fieldNumber, fieldName, ctor, toObjectFn, + isRepeated, opt_binaryReaderFn, opt_binaryWriterFn, + opt_binaryMessageSerializeFn, opt_binaryMessageDeserializeFn, + opt_isPacked) { + /** @const */ + this.fieldIndex = fieldNumber; + /** @const */ + this.fieldName = fieldName; + /** @const */ + this.ctor = ctor; + /** @const */ + this.toObjectFn = toObjectFn; + /** @const */ + this.binaryReaderFn = opt_binaryReaderFn; + /** @const */ + this.binaryWriterFn = opt_binaryWriterFn; + /** @const */ + this.binaryMessageSerializeFn = opt_binaryMessageSerializeFn; + /** @const */ + this.binaryMessageDeserializeFn = opt_binaryMessageDeserializeFn; + /** @const */ + this.isRepeated = isRepeated; + /** @const */ + this.isPacked = opt_isPacked; +}; + + +/** + * Base class for all JsPb messages. + * @constructor + * @struct + */ +jspb.Message = function() { +}; + + +/** + * @define {boolean} Whether to generate toObject methods for objects. Turn + * this off, if you do not want toObject to be ever used in your project. + * When turning off this flag, consider adding a conformance test that bans + * calling toObject. Enabling this will disable the JSCompiler's ability to + * dead code eliminate fields used in protocol buffers that are never used + * in an application. + */ +goog.define('jspb.Message.GENERATE_TO_OBJECT', true); + + +/** + * @define {boolean} Whether to generate fromObject methods for objects. Turn + * this off, if you do not want fromObject to be ever used in your project. + * When turning off this flag, consider adding a conformance test that bans + * calling fromObject. Enabling this might disable the JSCompiler's ability + * to dead code eliminate fields used in protocol buffers that are never + * used in an application. + * NOTE: By default no protos actually have a fromObject method. You need to + * add the jspb.generate_from_object options to the proto definition to + * activate the feature. + * By default this is enabled for test code only. + */ +goog.define('jspb.Message.GENERATE_FROM_OBJECT', !goog.DISALLOW_TEST_ONLY_CODE); + + +/** + * @define {boolean} Turning on this flag does NOT change the behavior of JSPB + * and only affects private internal state. It may, however, break some + * tests that use naive deeply-equals algorithms, because using a proto + * mutates its internal state. + * Projects are advised to turn this flag always on. + */ +goog.define('jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS', COMPILED); +// TODO(b/19419436) Turn this on by default. + + +/** + * The internal data array. + * @type {!Array} + * @protected + */ +jspb.Message.prototype.array; + + +/** + * Wrappers are the constructed instances of message-type fields. They are built + * on demand from the raw array data. Includes message fields, repeated message + * fields and extension message fields. Indexed by field number. + * @type {Object} + * @private + */ +jspb.Message.prototype.wrappers_; + + +/** + * The object that contains extension fields, if any. This is an object that + * maps from a proto field number to the field's value. + * @type {Object} + * @private + */ +jspb.Message.prototype.extensionObject_; + + +/** + * Non-extension fields with a field number at or above the pivot are + * stored in the extension object (in addition to all extension fields). + * @type {number} + * @private + */ +jspb.Message.prototype.pivot_; + + +/** + * The JsPb message_id of this proto. + * @type {string|undefined} the message id or undefined if this message + * has no id. + * @private + */ +jspb.Message.prototype.messageId_; + + +/** + * The xid of this proto type (The same for all instances of a proto). Provides + * a way to identify a proto by stable obfuscated name. + * @see {xid}. + * Available if {@link jspb.generate_xid} is added as a Message option to + * a protocol buffer. + * @const {!xid.String|undefined} The xid or undefined if message is + * annotated to generate the xid. + */ +jspb.Message.prototype.messageXid; + + + +/** + * Returns the JsPb message_id of this proto. + * @return {string|undefined} the message id or undefined if this message + * has no id. + */ +jspb.Message.prototype.getJsPbMessageId = function() { + return this.messageId_; +}; + + +/** + * An offset applied to lookups into this.array to account for the presence or + * absence of a messageId at position 0. For response messages, this will be 0. + * Otherwise, it will be -1 so that the first array position is not wasted. + * @type {number} + * @private + */ +jspb.Message.prototype.arrayIndexOffset_; + + +/** + * Returns the index into msg.array at which the proto field with tag number + * fieldNumber will be located. + * @param {!jspb.Message} msg Message for which we're calculating an index. + * @param {number} fieldNumber The field number. + * @return {number} The index. + * @private + */ +jspb.Message.getIndex_ = function(msg, fieldNumber) { + return fieldNumber + msg.arrayIndexOffset_; +}; + + +/** + * Initializes a JsPb Message. + * @param {!jspb.Message} msg The JsPb proto to modify. + * @param {Array|undefined} data An initial data array. + * @param {string|number} messageId For response messages, the message id or '' + * if no message id is specified. For non-response messages, 0. + * @param {number} suggestedPivot The field number at which to start putting + * fields into the extension object. This is only used if data does not + * contain an extension object already. -1 if no extension object is + * required for this message type. + * @param {Array} repeatedFields The message's repeated fields. + * @param {Array>=} opt_oneofFields The fields belonging to + * each of the message's oneof unions. + * @protected + */ +jspb.Message.initialize = function( + msg, data, messageId, suggestedPivot, repeatedFields, opt_oneofFields) { + msg.wrappers_ = jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS ? null : {}; + if (!data) { + data = messageId ? [messageId] : []; + } + msg.messageId_ = messageId ? String(messageId) : undefined; + // If the messageId is 0, this message is not a response message, so we shift + // array indices down by 1 so as not to waste the first position in the array, + // which would otherwise go unused. + msg.arrayIndexOffset_ = messageId === 0 ? -1 : 0; + msg.array = data; + jspb.Message.materializeExtensionObject_(msg, suggestedPivot); + if (repeatedFields) { + for (var i = 0; i < repeatedFields.length; i++) { + var fieldNumber = repeatedFields[i]; + if (fieldNumber < msg.pivot_) { + var index = jspb.Message.getIndex_(msg, fieldNumber); + msg.array[index] = msg.array[index] || + (jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS ? + jspb.Message.EMPTY_LIST_SENTINEL_ : + []); + } else { + msg.extensionObject_[fieldNumber] = + msg.extensionObject_[fieldNumber] || + (jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS ? + jspb.Message.EMPTY_LIST_SENTINEL_ : + []); + } + } + } + + if (opt_oneofFields && opt_oneofFields.length) { + // Compute the oneof case for each union. This ensures only one value is + // set in the union. + goog.array.forEach( + opt_oneofFields, goog.partial(jspb.Message.computeOneofCase, msg)); + } +}; + + +/** + * Used to mark empty repeated fields. Serializes to null when serialized + * to JSON. + * When reading a repeated field readers must check the return value against + * this value and return and replace it with a new empty array if it is + * present. + * @private @const {!Object} + */ +jspb.Message.EMPTY_LIST_SENTINEL_ = goog.DEBUG && Object.freeze ? + Object.freeze([]) : + []; + + +/** + * Ensures that the array contains an extension object if necessary. + * If the array contains an extension object in its last position, then the + * object is kept in place and its position is used as the pivot. If not, then + * create an extension object using suggestedPivot. If suggestedPivot is -1, + * we don't have an extension object at all, in which case all fields are stored + * in the array. + * @param {!jspb.Message} msg The JsPb proto to modify. + * @param {number} suggestedPivot See description for initialize(). + * @private + */ +jspb.Message.materializeExtensionObject_ = function(msg, suggestedPivot) { + if (msg.array.length) { + var foundIndex = msg.array.length - 1; + var obj = msg.array[foundIndex]; + // Normal fields are never objects, so we can be sure that if we find an + // object here, then it's the extension object. However, we must ensure that + // the object is not an array, since arrays are valid field values. + // NOTE(lukestebbing): We avoid looking at .length to avoid a JIT bug + // in Safari on iOS 8. See the description of CL/86511464 for details. + if (obj && typeof obj == 'object' && !goog.isArray(obj)) { + msg.pivot_ = foundIndex - msg.arrayIndexOffset_; + msg.extensionObject_ = obj; + return; + } + } + // This complexity exists because we keep all extension fields in the + // extensionObject_ regardless of proto field number. Changing this would + // simplify the code here, but it would require changing the serialization + // format from the server, which is not backwards compatible. + // TODO(jshneier): Should we just treat extension fields the same as + // non-extension fields, and select whether they appear in the object or in + // the array purely based on tag number? This would allow simplifying all the + // get/setExtension logic, but it would require the breaking change described + // above. + if (suggestedPivot > -1) { + msg.pivot_ = suggestedPivot; + var pivotIndex = jspb.Message.getIndex_(msg, suggestedPivot); + if (!jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS) { + msg.extensionObject_ = msg.array[pivotIndex] = {}; + } else { + // Initialize to null to avoid changing the shape of the proto when it + // gets eventually set. + msg.extensionObject_ = null; + } + } else { + msg.pivot_ = Number.MAX_VALUE; + } +}; + + +/** + * Creates an empty extensionObject_ if non exists. + * @param {!jspb.Message} msg The JsPb proto to modify. + * @private + */ +jspb.Message.maybeInitEmptyExtensionObject_ = function(msg) { + var pivotIndex = jspb.Message.getIndex_(msg, msg.pivot_); + if (!msg.array[pivotIndex]) { + msg.extensionObject_ = msg.array[pivotIndex] = {}; + } +}; + + +/** + * Converts a JsPb repeated message field into an object list. + * @param {!Array} field The repeated message field to be + * converted. + * @param {?function(boolean=): Object| + * function((boolean|undefined),T): Object} toObjectFn The toObject + * function for this field. We need to pass this for effective dead code + * removal. + * @param {boolean=} opt_includeInstance Whether to include the JSPB instance + * for transitional soy proto support: http://goto/soy-param-migration + * @return {!Array} An array of converted message objects. + * @template T + */ +jspb.Message.toObjectList = function(field, toObjectFn, opt_includeInstance) { + // Not using goog.array.map in the generated code to keep it small. + // And not using it here to avoid a function call. + var result = []; + for (var i = 0; i < field.length; i++) { + result[i] = toObjectFn.call(field[i], opt_includeInstance, + /** @type {!jspb.Message} */ (field[i])); + } + return result; +}; + + +/** + * Adds a proto's extension data to a Soy rendering object. + * @param {!jspb.Message} proto The proto whose extensions to convert. + * @param {!Object} obj The Soy object to add converted extension data to. + * @param {!Object} extensions The proto class' registered extensions. + * @param {function(jspb.ExtensionFieldInfo) : *} getExtensionFn The proto + * class' getExtension function. Passed for effective dead code removal. + * @param {boolean=} opt_includeInstance Whether to include the JSPB instance + * for transitional soy proto support: http://goto/soy-param-migration + */ +jspb.Message.toObjectExtension = function(proto, obj, extensions, + getExtensionFn, opt_includeInstance) { + for (var fieldNumber in extensions) { + var fieldInfo = extensions[fieldNumber]; + var value = getExtensionFn.call(proto, fieldInfo); + if (value) { + for (var name in fieldInfo.fieldName) { + if (fieldInfo.fieldName.hasOwnProperty(name)) { + break; // the compiled field name + } + } + if (!fieldInfo.toObjectFn) { + obj[name] = value; + } else { + if (fieldInfo.isRepeated) { + obj[name] = jspb.Message.toObjectList( + /** @type {!Array} */ (value), + fieldInfo.toObjectFn, opt_includeInstance); + } else { + obj[name] = fieldInfo.toObjectFn(opt_includeInstance, value); + } + } + } + } +}; + + +/** + * Writes a proto's extension data to a binary-format output stream. + * @param {!jspb.Message} proto The proto whose extensions to convert. + * @param {*} writer The binary-format writer to write to. + * @param {!Object} extensions The proto class' registered extensions. + * @param {function(jspb.ExtensionFieldInfo) : *} getExtensionFn The proto + * class' getExtension function. Passed for effective dead code removal. + */ +jspb.Message.serializeBinaryExtensions = function(proto, writer, extensions, + getExtensionFn) { + for (var fieldNumber in extensions) { + var fieldInfo = extensions[fieldNumber]; + // The old codegen doesn't add the extra fields to ExtensionFieldInfo, so we + // need to gracefully error-out here rather than produce a null dereference + // below. + if (!fieldInfo.binaryWriterFn) { + throw new Error('Message extension present that was generated ' + + 'without binary serialization support'); + } + var value = getExtensionFn.call(proto, fieldInfo); + if (value) { + if (fieldInfo.ctor) { // is this a message type? + // If the message type of the extension was generated without binary + // support, there may not be a binary message serializer function, and + // we can't know when we codegen the extending message that the extended + // message may require binary support, so we can *only* catch this error + // here, at runtime (and this decoupled codegen is the whole point of + // extensions!). + if (fieldInfo.binaryMessageSerializeFn) { + fieldInfo.binaryWriterFn.call(writer, fieldInfo.fieldIndex, + value, fieldInfo.binaryMessageSerializeFn); + } else { + throw new Error('Message extension present holding submessage ' + + 'without binary support enabled, and message is ' + + 'being serialized to binary format'); + } + } else { + fieldInfo.binaryWriterFn.call(writer, fieldInfo.fieldIndex, value); + } + } + } +}; + + +/** + * Reads an extension field from the given reader and, if a valid extension, + * sets the extension value. + * @param {!jspb.Message} msg A jspb proto. + * @param {{skipField:function(),getFieldNumber:function():number}} reader + * @param {!Object} extensions The extensions object. + * @param {function(jspb.ExtensionFieldInfo)} getExtensionFn + * @param {function(jspb.ExtensionFieldInfo, ?)} setExtensionFn + */ +jspb.Message.readBinaryExtension = function(msg, reader, extensions, + getExtensionFn, setExtensionFn) { + var fieldInfo = extensions[reader.getFieldNumber()]; + if (!fieldInfo) { + reader.skipField(); + return; + } + if (!fieldInfo.binaryReaderFn) { + throw new Error('Deserializing extension whose generated code does not ' + + 'support binary format'); + } + + var value; + if (fieldInfo.ctor) { + // Message type. + value = new fieldInfo.ctor(); + fieldInfo.binaryReaderFn.call( + reader, value, fieldInfo.binaryMessageDeserializeFn); + } else { + // All other types. + value = fieldInfo.binaryReaderFn.call(reader); + } + + if (fieldInfo.isRepeated && !fieldInfo.isPacked) { + var currentList = getExtensionFn.call(msg, fieldInfo); + if (!currentList) { + setExtensionFn.call(msg, fieldInfo, [value]); + } else { + currentList.push(value); + } + } else { + setExtensionFn.call(msg, fieldInfo, value); + } +}; + + +/** + * Gets the value of a non-extension field. + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @return {string|number|boolean|Uint8Array|Array|null|undefined} + * The field's value. + * @protected + */ +jspb.Message.getField = function(msg, fieldNumber) { + if (fieldNumber < msg.pivot_) { + var index = jspb.Message.getIndex_(msg, fieldNumber); + var val = msg.array[index]; + if (val === jspb.Message.EMPTY_LIST_SENTINEL_) { + return msg.array[index] = []; + } + return val; + } else { + var val = msg.extensionObject_[fieldNumber]; + if (val === jspb.Message.EMPTY_LIST_SENTINEL_) { + return msg.extensionObject_[fieldNumber] = []; + } + return val; + } +}; + + +/** + * Gets the value of a non-extension primitive field, with proto3 (non-nullable + * primitives) semantics. Returns `defaultValue` if the field is not otherwise + * set. + * @template T + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @param {T} defaultValue The default value. + * @return {T} The field's value. + * @protected + */ +jspb.Message.getFieldProto3 = function(msg, fieldNumber, defaultValue) { + var value = jspb.Message.getField(msg, fieldNumber); + if (value == null) { + return defaultValue; + } else { + return value; + } +}; + + +/** + * Sets the value of a non-extension field. + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @param {string|number|boolean|Uint8Array|Array|undefined} value New value + * @protected + */ +jspb.Message.setField = function(msg, fieldNumber, value) { + if (fieldNumber < msg.pivot_) { + msg.array[jspb.Message.getIndex_(msg, fieldNumber)] = value; + } else { + msg.extensionObject_[fieldNumber] = value; + } +}; + + +/** + * Sets the value of a field in a oneof union and clears all other fields in + * the union. + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @param {!Array} oneof The fields belonging to the union. + * @param {string|number|boolean|Uint8Array|Array|undefined} value New value + * @protected + */ +jspb.Message.setOneofField = function(msg, fieldNumber, oneof, value) { + var currentCase = jspb.Message.computeOneofCase(msg, oneof); + if (currentCase && currentCase !== fieldNumber && value !== undefined) { + if (msg.wrappers_ && currentCase in msg.wrappers_) { + msg.wrappers_[currentCase] = undefined; + } + jspb.Message.setField(msg, currentCase, undefined); + } + jspb.Message.setField(msg, fieldNumber, value); +}; + + +/** + * Computes the selection in a oneof group for the given message, ensuring + * only one field is set in the process. + * + * According to the protobuf language guide ( + * https://developers.google.com/protocol-buffers/docs/proto#oneof), "if the + * parser encounters multiple members of the same oneof on the wire, only the + * last member seen is used in the parsed message." Since JSPB serializes + * messages to a JSON array, the "last member seen" will always be the field + * with the greatest field number (directly corresponding to the greatest + * array index). + * + * @param {!jspb.Message} msg A jspb proto. + * @param {!Array} oneof The field numbers belonging to the union. + * @return {number} The field number currently set in the union, or 0 if none. + * @protected + */ +jspb.Message.computeOneofCase = function(msg, oneof) { + var oneofField; + var oneofValue; + + goog.array.forEach(oneof, function(fieldNumber) { + var value = jspb.Message.getField(msg, fieldNumber); + if (goog.isDefAndNotNull(value)) { + oneofField = fieldNumber; + oneofValue = value; + jspb.Message.setField(msg, fieldNumber, undefined); + } + }); + + if (oneofField) { + // NB: We know the value is unique, so we can call jspb.Message.setField + // directly instead of jpsb.Message.setOneofField. Also, setOneofField + // calls this function. + jspb.Message.setField(msg, oneofField, oneofValue); + return oneofField; + } + + return 0; +}; + + +/** + * Gets and wraps a proto field on access. + * @param {!jspb.Message} msg A jspb proto. + * @param {function(new:jspb.Message, Array)} ctor Constructor for the field. + * @param {number} fieldNumber The field number. + * @param {number=} opt_required True (1) if this is a required field. + * @return {jspb.Message} The field as a jspb proto. + * @protected + */ +jspb.Message.getWrapperField = function(msg, ctor, fieldNumber, opt_required) { + // TODO(mwr): Consider copying data and/or arrays. + if (!msg.wrappers_) { + msg.wrappers_ = {}; + } + if (!msg.wrappers_[fieldNumber]) { + var data = /** @type {Array} */ (jspb.Message.getField(msg, fieldNumber)); + if (opt_required || data) { + // TODO(mwr): Remove existence test for always valid default protos. + msg.wrappers_[fieldNumber] = new ctor(data); + } + } + return /** @type {jspb.Message} */ (msg.wrappers_[fieldNumber]); +}; + + +/** + * Gets and wraps a repeated proto field on access. + * @param {!jspb.Message} msg A jspb proto. + * @param {function(new:jspb.Message, Array)} ctor Constructor for the field. + * @param {number} fieldNumber The field number. + * @return {Array} The repeated field as an array of protos. + * @protected + */ +jspb.Message.getRepeatedWrapperField = function(msg, ctor, fieldNumber) { + if (!msg.wrappers_) { + msg.wrappers_ = {}; + } + if (!msg.wrappers_[fieldNumber]) { + var data = jspb.Message.getField(msg, fieldNumber); + for (var wrappers = [], i = 0; i < data.length; i++) { + wrappers[i] = new ctor(data[i]); + } + msg.wrappers_[fieldNumber] = wrappers; + } + var val = msg.wrappers_[fieldNumber]; + if (val == jspb.Message.EMPTY_LIST_SENTINEL_) { + val = msg.wrappers_[fieldNumber] = []; + } + return /** @type {Array} */ (val); +}; + + +/** + * Sets a proto field and syncs it to the backing array. + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @param {jspb.Message|undefined} value A new value for this proto field. + * @protected + */ +jspb.Message.setWrapperField = function(msg, fieldNumber, value) { + if (!msg.wrappers_) { + msg.wrappers_ = {}; + } + var data = value ? value.toArray() : value; + msg.wrappers_[fieldNumber] = value; + jspb.Message.setField(msg, fieldNumber, data); +}; + + +/** + * Sets a proto field in a oneof union and syncs it to the backing array. + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @param {!Array} oneof The fields belonging to the union. + * @param {jspb.Message|undefined} value A new value for this proto field. + * @protected + */ +jspb.Message.setOneofWrapperField = function(msg, fieldNumber, oneof, value) { + if (!msg.wrappers_) { + msg.wrappers_ = {}; + } + var data = value ? value.toArray() : value; + msg.wrappers_[fieldNumber] = value; + jspb.Message.setOneofField(msg, fieldNumber, oneof, data); +}; + + +/** + * Sets a repeated proto field and syncs it to the backing array. + * @param {!jspb.Message} msg A jspb proto. + * @param {number} fieldNumber The field number. + * @param {Array|undefined} value An array of protos. + * @protected + */ +jspb.Message.setRepeatedWrapperField = function(msg, fieldNumber, value) { + if (!msg.wrappers_) { + msg.wrappers_ = {}; + } + value = value || []; + for (var data = [], i = 0; i < value.length; i++) { + data[i] = value[i].toArray(); + } + msg.wrappers_[fieldNumber] = value; + jspb.Message.setField(msg, fieldNumber, data); +}; + + +/** + * Converts a JsPb repeated message field into a map. The map will contain + * protos unless an optional toObject function is given, in which case it will + * contain objects suitable for Soy rendering. + * @param {!Array} field The repeated message field to be + * converted. + * @param {function() : string?} mapKeyGetterFn The function to get the key of + * the map. + * @param {?function(boolean=): Object| + * function((boolean|undefined),T): Object} opt_toObjectFn The + * toObject function for this field. We need to pass this for effective + * dead code removal. + * @param {boolean=} opt_includeInstance Whether to include the JSPB instance + * for transitional soy proto support: http://goto/soy-param-migration + * @return {!Object.} A map of proto or Soy objects. + * @template T + */ +jspb.Message.toMap = function( + field, mapKeyGetterFn, opt_toObjectFn, opt_includeInstance) { + var result = {}; + for (var i = 0; i < field.length; i++) { + result[mapKeyGetterFn.call(field[i])] = opt_toObjectFn ? + opt_toObjectFn.call(field[i], opt_includeInstance, + /** @type {!jspb.Message} */ (field[i])) : field[i]; + } + return result; +}; + + +/** + * Returns the internal array of this proto. + *

Note: If you use this array to construct a second proto, the content + * would then be partially shared between the two protos. + * @return {!Array} The proto represented as an array. + */ +jspb.Message.prototype.toArray = function() { + return this.array; +}; + + + + +/** + * Creates a string representation of the internal data array of this proto. + *

NOTE: This string is *not* suitable for use in server requests. + * @return {string} A string representation of this proto. + * @override + */ +jspb.Message.prototype.toString = function() { + return this.array.toString(); +}; + + +/** + * Gets the value of the extension field from the extended object. + * @param {jspb.ExtensionFieldInfo.} fieldInfo Specifies the field to get. + * @return {T} The value of the field. + * @template T + */ +jspb.Message.prototype.getExtension = function(fieldInfo) { + if (!this.extensionObject_) { + return undefined; + } + if (!this.wrappers_) { + this.wrappers_ = {}; + } + var fieldNumber = fieldInfo.fieldIndex; + if (fieldInfo.isRepeated) { + if (fieldInfo.ctor) { + if (!this.wrappers_[fieldNumber]) { + this.wrappers_[fieldNumber] = + goog.array.map(this.extensionObject_[fieldNumber] || [], + function(arr) { + return new fieldInfo.ctor(arr); + }); + } + return this.wrappers_[fieldNumber]; + } else { + return this.extensionObject_[fieldNumber]; + } + } else { + if (fieldInfo.ctor) { + if (!this.wrappers_[fieldNumber] && this.extensionObject_[fieldNumber]) { + this.wrappers_[fieldNumber] = new fieldInfo.ctor( + /** @type {Array|undefined} */ ( + this.extensionObject_[fieldNumber])); + } + return this.wrappers_[fieldNumber]; + } else { + return this.extensionObject_[fieldNumber]; + } + } +}; + + +/** + * Sets the value of the extension field in the extended object. + * @param {jspb.ExtensionFieldInfo} fieldInfo Specifies the field to set. + * @param {jspb.Message|string|number|boolean|Array} value The value to set. + */ +jspb.Message.prototype.setExtension = function(fieldInfo, value) { + if (!this.wrappers_) { + this.wrappers_ = {}; + } + jspb.Message.maybeInitEmptyExtensionObject_(this); + var fieldNumber = fieldInfo.fieldIndex; + if (fieldInfo.isRepeated) { + value = value || []; + if (fieldInfo.ctor) { + this.wrappers_[fieldNumber] = value; + this.extensionObject_[fieldNumber] = goog.array.map( + /** @type {Array} */ (value), function(msg) { + return msg.toArray(); + }); + } else { + this.extensionObject_[fieldNumber] = value; + } + } else { + if (fieldInfo.ctor) { + this.wrappers_[fieldNumber] = value; + this.extensionObject_[fieldNumber] = value ? value.toArray() : value; + } else { + this.extensionObject_[fieldNumber] = value; + } + } +}; + + +/** + * Creates a difference object between two messages. + * + * The result will contain the top-level fields of m2 that differ from those of + * m1 at any level of nesting. No data is cloned, the result object will + * share its top-level elements with m2 (but not with m1). + * + * Note that repeated fields should not have null/undefined elements, but if + * they do, this operation will treat repeated fields of different length as + * the same if the only difference between them is due to trailing + * null/undefined values. + * + * @param {!jspb.Message} m1 The first message object. + * @param {!jspb.Message} m2 The second message object. + * @return {!jspb.Message} The difference returned as a proto message. + * Note that the returned message may be missing required fields. This is + * currently tolerated in Js, but would cause an error if you tried to + * send such a proto to the server. You can access the raw difference + * array with result.toArray(). + * @throws {Error} If the messages are responses with different types. + */ +jspb.Message.difference = function(m1, m2) { + if (!(m1 instanceof m2.constructor)) { + throw new Error('Messages have different types.'); + } + var arr1 = m1.toArray(); + var arr2 = m2.toArray(); + var res = []; + var start = 0; + var length = arr1.length > arr2.length ? arr1.length : arr2.length; + if (m1.getJsPbMessageId()) { + res[0] = m1.getJsPbMessageId(); + start = 1; + } + for (var i = start; i < length; i++) { + if (!jspb.Message.compareFields(arr1[i], arr2[i])) { + res[i] = arr2[i]; + } + } + return new m1.constructor(res); +}; + + +/** + * Tests whether two messages are equal. + * @param {jspb.Message|undefined} m1 The first message object. + * @param {jspb.Message|undefined} m2 The second message object. + * @return {boolean} true if both messages are null/undefined, or if both are + * of the same type and have the same field values. + */ +jspb.Message.equals = function(m1, m2) { + return m1 == m2 || (!!(m1 && m2) && (m1 instanceof m2.constructor) && + jspb.Message.compareFields(m1.toArray(), m2.toArray())); +}; + + +/** + * Compares two message fields recursively. + * @param {*} field1 The first field. + * @param {*} field2 The second field. + * @return {boolean} true if the fields are null/undefined, or otherwise equal. + */ +jspb.Message.compareFields = function(field1, field2) { + if (goog.isObject(field1) && goog.isObject(field2)) { + var keys = {}, name, extensionObject1, extensionObject2; + for (name in field1) { + field1.hasOwnProperty(name) && (keys[name] = 0); + } + for (name in field2) { + field2.hasOwnProperty(name) && (keys[name] = 0); + } + for (name in keys) { + var val1 = field1[name], val2 = field2[name]; + if (goog.isObject(val1) && !goog.isArray(val1)) { + if (extensionObject1 !== undefined) { + throw new Error('invalid jspb state'); + } + extensionObject1 = goog.object.isEmpty(val1) ? undefined : val1; + val1 = undefined; + } + if (goog.isObject(val2) && !goog.isArray(val2)) { + if (extensionObject2 !== undefined) { + throw new Error('invalid jspb state'); + } + extensionObject2 = goog.object.isEmpty(val2) ? undefined : val2; + val2 = undefined; + } + if (!jspb.Message.compareFields(val1, val2)) { + return false; + } + } + if (extensionObject1 || extensionObject2) { + return jspb.Message.compareFields(extensionObject1, extensionObject2); + } + return true; + } + // Primitive fields, null and undefined compare as equal. + // This also forces booleans and 0/1 to compare as equal to ensure + // compatibility with the jspb serializer. + return field1 == field2; +}; + + +/** + * Static clone function. NOTE: A type-safe method called "cloneMessage" exists + * on each generated JsPb class. Do not call this function directly. + * @param {!jspb.Message} msg A message to clone. + * @return {!jspb.Message} A deep clone of the given message. + */ +jspb.Message.clone = function(msg) { + // Although we could include the wrappers, we leave them out here. + return jspb.Message.cloneMessage(msg); +}; + + +/** + * @param {!jspb.Message} msg A message to clone. + * @return {!jspb.Message} A deep clone of the given message. + * @protected + */ +jspb.Message.cloneMessage = function(msg) { + // Although we could include the wrappers, we leave them out here. + return new msg.constructor(jspb.Message.clone_(msg.toArray())); +}; + + +/** + * Takes 2 messages of the same type and copies the contents of the first + * message into the second. After this the 2 messages will equals in terms of + * value semantics but share no state. All data in the destination message will + * be overridden. + * + * @param {MESSAGE} fromMessage Message that will be copied into toMessage. + * @param {MESSAGE} toMessage Message which will receive a copy of fromMessage + * as its contents. + * @template MESSAGE + */ +jspb.Message.copyInto = function(fromMessage, toMessage) { + goog.asserts.assertInstanceof(fromMessage, jspb.Message); + goog.asserts.assertInstanceof(toMessage, jspb.Message); + goog.asserts.assert(fromMessage.constructor == toMessage.constructor, + 'Copy source and target message should have the same type.'); + var copyOfFrom = jspb.Message.clone(fromMessage); + + var to = toMessage.toArray(); + var from = copyOfFrom.toArray(); + + // Empty destination in case it has more values at the end of the array. + to.length = 0; + // and then copy everything from the new to the existing message. + for (var i = 0; i < from.length; i++) { + to[i] = from[i]; + } + + // This is either null or empty for a fresh copy. + toMessage.wrappers_ = copyOfFrom.wrappers_; + // Just a reference into the shared array. + toMessage.extensionObject_ = copyOfFrom.extensionObject_; +}; + + +/** + * Helper for cloning an internal JsPb object. + * @param {!Object} obj A JsPb object, eg, a field, to be cloned. + * @return {!Object} A clone of the input object. + * @private + */ +jspb.Message.clone_ = function(obj) { + var o; + if (goog.isArray(obj)) { + // Allocate array of correct size. + var clonedArray = new Array(obj.length); + // Use array iteration where possible because it is faster than for-in. + for (var i = 0; i < obj.length; i++) { + if ((o = obj[i]) != null) { + clonedArray[i] = typeof o == 'object' ? jspb.Message.clone_(o) : o; + } + } + return clonedArray; + } + var clone = {}; + for (var key in obj) { + if ((o = obj[key]) != null) { + clone[key] = typeof o == 'object' ? jspb.Message.clone_(o) : o; + } + } + return clone; +}; + + +/** + * Registers a JsPb message type id with its constructor. + * @param {string} id The id for this type of message. + * @param {Function} constructor The message constructor. + */ +jspb.Message.registerMessageType = function(id, constructor) { + jspb.Message.registry_[id] = constructor; + // This is needed so we can later access messageId directly on the contructor, + // otherwise it is not available due to 'property collapsing' by the compiler. + constructor.messageId = id; +}; + + +/** + * The registry of message ids to message constructors. + * @private + */ +jspb.Message.registry_ = {}; + + +/** + * The extensions registered on MessageSet. This is a map of extension + * field number to field info object. This should be considered as a + * private API. + * + * This is similar to [jspb class name].extensions object for + * non-MessageSet. We special case MessageSet so that we do not need + * to goog.require MessageSet from classes that extends MessageSet. + * + * @type {!Object.} + */ +jspb.Message.messageSetExtensions = {}; diff --git a/js/message_test.js b/js/message_test.js new file mode 100644 index 00000000..5d3caaa0 --- /dev/null +++ b/js/message_test.js @@ -0,0 +1,970 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Test suite is written using Jasmine -- see http://jasmine.github.io/ + +goog.setTestOnly(); + +goog.require('goog.json'); +goog.require('goog.testing.asserts'); +goog.require('jspb.Message'); +goog.require('proto.jspb.exttest.beta.floatingStrField'); +goog.require('proto.jspb.exttest.floatingMsgField'); +goog.require('proto.jspb.exttest.floatingMsgFieldTwo'); +goog.require('proto.jspb.test.CloneExtension'); +goog.require('proto.jspb.test.Complex'); +goog.require('proto.jspb.test.DefaultValues'); +goog.require('proto.jspb.test.Empty'); +goog.require('proto.jspb.test.EnumContainer'); +goog.require('proto.jspb.test.ExtensionMessage'); +goog.require('proto.jspb.test.floatingMsgField'); +goog.require('proto.jspb.test.floatingStrField'); +goog.require('proto.jspb.test.HasExtensions'); +goog.require('proto.jspb.test.IndirectExtension'); +goog.require('proto.jspb.test.IsExtension'); +goog.require('proto.jspb.test.OptionalFields'); +goog.require('proto.jspb.test.OuterEnum'); +goog.require('proto.jspb.test.simple1'); +goog.require('proto.jspb.test.Simple1'); +goog.require('proto.jspb.test.Simple2'); +goog.require('proto.jspb.test.SpecialCases'); +goog.require('proto.jspb.test.TestClone'); +goog.require('proto.jspb.test.TestExtensionsMessage'); +goog.require('proto.jspb.test.TestGroup'); +goog.require('proto.jspb.test.TestGroup1'); +goog.require('proto.jspb.test.TestMessageWithOneof'); +goog.require('proto.jspb.test.TestReservedNames'); +goog.require('proto.jspb.test.TestReservedNamesExtension'); + + + + +describe('Message test suite', function() { + it('testEmptyProto', function() { + var empty1 = new proto.jspb.test.Empty([]); + var empty2 = new proto.jspb.test.Empty([]); + assertObjectEquals({}, empty1.toObject()); + assertObjectEquals('Message should not be corrupted:', empty2, empty1); + }); + + it('testTopLevelEnum', function() { + var response = new proto.jspb.test.EnumContainer([]); + response.setOuterEnum(proto.jspb.test.OuterEnum.FOO); + assertEquals(proto.jspb.test.OuterEnum.FOO, response.getOuterEnum()); + }); + + it('testByteStrings', function() { + var data = new proto.jspb.test.DefaultValues([]); + data.setBytesField('some_bytes'); + assertEquals('some_bytes', data.getBytesField()); + }); + + it('testComplexConversion', function() { + var data1 = ['a',,, [, 11], [[, 22], [, 33]],, ['s1', 's2'],, 1]; + var data2 = ['a',,, [, 11], [[, 22], [, 33]],, ['s1', 's2'],, 1]; + var foo = new proto.jspb.test.Complex(data1); + var bar = new proto.jspb.test.Complex(data2); + var result = foo.toObject(); + assertObjectEquals({ + aString: 'a', + anOutOfOrderBool: 1, + aNestedMessage: { + anInt: 11 + }, + aRepeatedMessageList: [{anInt: 22}, {anInt: 33}], + aRepeatedStringList: ['s1', 's2'] + }, result); + + // Now test with the jspb instances included. + result = foo.toObject(true /* opt_includeInstance */); + assertObjectEquals({ + aString: 'a', + anOutOfOrderBool: 1, + aNestedMessage: { + anInt: 11, + $jspbMessageInstance: foo.getANestedMessage() + }, + aRepeatedMessageList: [ + {anInt: 22, $jspbMessageInstance: foo.getARepeatedMessageList()[0]}, + {anInt: 33, $jspbMessageInstance: foo.getARepeatedMessageList()[1]} + ], + aRepeatedStringList: ['s1', 's2'], + $jspbMessageInstance: foo + }, result); + + }); + + it('testMissingFields', function() { + var foo = new proto.jspb.test.Complex([ + undefined, undefined, undefined, [], + undefined, undefined, undefined, undefined]); + var bar = new proto.jspb.test.Complex([ + undefined, undefined, undefined, [], + undefined, undefined, undefined, undefined]); + var result = foo.toObject(); + assertObjectEquals({ + aString: undefined, + anOutOfOrderBool: undefined, + aNestedMessage: { + anInt: undefined + }, + // Note: JsPb converts undefined repeated fields to empty arrays. + aRepeatedMessageList: [], + aRepeatedStringList: [] + }, result); + + }); + + it('testSpecialCases', function() { + // Note: Some property names are reserved in JavaScript. + // These names are converted to the Js property named pb_. + var special = + new proto.jspb.test.SpecialCases(['normal', 'default', 'function', + 'var']); + var result = special.toObject(); + assertObjectEquals({ + normal: 'normal', + pb_default: 'default', + pb_function: 'function', + pb_var: 'var' + }, result); + }); + + it('testDefaultValues', function() { + var defaultString = "default<>\'\"abc"; + var response = new proto.jspb.test.DefaultValues(); + + // Test toObject + var expectedObject = { + stringField: defaultString, + boolField: true, + intField: 11, + enumField: 13, + emptyField: '', + bytesField: 'bW9v' + }; + assertObjectEquals(expectedObject, response.toObject()); + + + // Test getters + response = new proto.jspb.test.DefaultValues(); + assertEquals(defaultString, response.getStringField()); + assertEquals(true, response.getBoolField()); + assertEquals(11, response.getIntField()); + assertEquals(13, response.getEnumField()); + assertEquals('', response.getEmptyField()); + assertEquals('bW9v', response.getBytesField()); + + function makeDefault(values) { + return new proto.jspb.test.DefaultValues(values); + } + + // Test with undefined values, + // Use push to workaround IE treating undefined array elements as holes. + response = makeDefault([undefined, undefined, undefined, undefined]); + assertEquals(defaultString, response.getStringField()); + assertEquals(true, response.getBoolField()); + assertEquals(11, response.getIntField()); + assertEquals(13, response.getEnumField()); + + // Test with null values, as would be returned by a JSON serializer. + response = makeDefault([null, null, null, null]); + assertEquals(defaultString, response.getStringField()); + assertEquals(true, response.getBoolField()); + assertEquals(11, response.getIntField()); + assertEquals(13, response.getEnumField()); + + // Test with false-like values. + response = makeDefault(['', false, 0, 0]); + assertEquals('', response.getStringField()); + assertEquals(false, response.getBoolField()); + assertEquals(true, response.getIntField() == 0); + assertEquals(true, response.getEnumField() == 0); + + // Test that clearing the values reverts them to the default state. + response = makeDefault(['blah', false, 111, 77]); + response.clearStringField(); response.clearBoolField(); + response.clearIntField(); response.clearEnumField(); + assertEquals(defaultString, response.getStringField()); + assertEquals(true, response.getBoolField()); + assertEquals(11, response.getIntField()); + assertEquals(13, response.getEnumField()); + + // Test that setFoo(null) clears the values. + response = makeDefault(['blah', false, 111, 77]); + response.setStringField(null); response.setBoolField(null); + response.setIntField(undefined); response.setEnumField(undefined); + assertEquals(defaultString, response.getStringField()); + assertEquals(true, response.getBoolField()); + assertEquals(11, response.getIntField()); + assertEquals(13, response.getEnumField()); + }); + + it('testMessageRegistration', function() { + // goog.require(SomeResponse) will include its library, which will in + // turn add SomeResponse to the message registry. + assertEquals(jspb.Message.registry_['res'], proto.jspb.test.SomeResponse); + }); + + it('testClearFields', function() { + // We don't set 'proper' defaults, rather, bools, strings, + // etc, are cleared to undefined or null and take on the Javascript + // meaning for that value. Repeated fields are set to [] when cleared. + var data = ['str', true, [11], [[22], [33]], ['s1', 's2']]; + var foo = new proto.jspb.test.OptionalFields(data); + foo.clearAString(); + foo.clearABool(); + foo.clearANestedMessage(); + foo.clearARepeatedMessageList(); + foo.clearARepeatedStringList(); + assertUndefined(foo.getAString()); + assertUndefined(foo.getABool()); + assertUndefined(foo.getANestedMessage()); + assertObjectEquals([], foo.getARepeatedMessageList()); + assertObjectEquals([], foo.getARepeatedStringList()); + // NOTE: We want the missing fields in 'expected' to be undefined, + // but we actually get a sparse array instead. We could use something + // like [1,undefined,2] to avoid this, except that this is still + // sparse on IE. No comment... + var expected = [,,, [], []]; + expected[0] = expected[1] = expected[2] = undefined; + assertObjectEquals(expected, foo.toArray()); + + // Test set(null). We could deprecated this in favor of clear(), but + // it's also convenient to have. + data = ['str', true, [11], [[22], [33]], ['s1', 's2']]; + foo = new proto.jspb.test.OptionalFields(data); + foo.setAString(null); + foo.setABool(null); + foo.setANestedMessage(null); + foo.setARepeatedMessageList(null); + foo.setARepeatedStringList(null); + assertNull(foo.getAString()); + assertNull(foo.getABool()); + assertNull(foo.getANestedMessage()); + assertObjectEquals([], foo.getARepeatedMessageList()); + assertObjectEquals([], foo.getARepeatedStringList()); + assertObjectEquals([null, null, null, [], []], foo.toArray()); + + // Test set(undefined). Again, not something we really need, and not + // supported directly by our typing, but it should 'do the right thing'. + data = ['str', true, [11], [[22], [33]], ['s1', 's2']]; + foo = new proto.jspb.test.OptionalFields(data); + foo.setAString(undefined); + foo.setABool(undefined); + foo.setANestedMessage(undefined); + foo.setARepeatedMessageList(undefined); + foo.setARepeatedStringList(undefined); + assertUndefined(foo.getAString()); + assertUndefined(foo.getABool()); + assertUndefined(foo.getANestedMessage()); + assertObjectEquals([], foo.getARepeatedMessageList()); + assertObjectEquals([], foo.getARepeatedStringList()); + expected = [,,, [], []]; + expected[0] = expected[1] = expected[2] = undefined; + assertObjectEquals(expected, foo.toArray()); + }); + + it('testDifferenceRawObject', function() { + var p1 = new proto.jspb.test.HasExtensions(['hi', 'diff', {}]); + var p2 = new proto.jspb.test.HasExtensions(['hi', 'what', + {1000: 'unique'}]); + var diff = /** @type {proto.jspb.test.HasExtensions} */ + (jspb.Message.difference(p1, p2)); + assertUndefined(diff.getStr1()); + assertEquals('what', diff.getStr2()); + assertUndefined(diff.getStr3()); + assertEquals('unique', diff.extensionObject_[1000]); + }); + + it('testEqualsSimple', function() { + var s1 = new proto.jspb.test.Simple1(['hi']); + assertTrue(jspb.Message.equals(s1, new proto.jspb.test.Simple1(['hi']))); + assertFalse(jspb.Message.equals(s1, new proto.jspb.test.Simple1(['bye']))); + var s1b = new proto.jspb.test.Simple1(['hi', ['hello']]); + assertTrue(jspb.Message.equals(s1b, + new proto.jspb.test.Simple1(['hi', ['hello']]))); + assertTrue(jspb.Message.equals(s1b, + new proto.jspb.test.Simple1(['hi', ['hello', undefined, + undefined, undefined]]))); + assertFalse(jspb.Message.equals(s1b, + new proto.jspb.test.Simple1(['no', ['hello']]))); + // Test with messages of different types + var s2 = new proto.jspb.test.Simple2(['hi']); + assertFalse(jspb.Message.equals(s1, s2)); + }); + + it('testEquals_softComparison', function() { + var s1 = new proto.jspb.test.Simple1(['hi', [], null]); + assertTrue(jspb.Message.equals(s1, + new proto.jspb.test.Simple1(['hi', []]))); + + var s1b = new proto.jspb.test.Simple1(['hi', [], true]); + assertTrue(jspb.Message.equals(s1b, + new proto.jspb.test.Simple1(['hi', [], 1]))); + }); + + it('testEqualsComplex', function() { + var data1 = ['a',,, [, 11], [[, 22], [, 33]],, ['s1', 's2'],, 1]; + var data2 = ['a',,, [, 11], [[, 22], [, 34]],, ['s1', 's2'],, 1]; + var data3 = ['a',,, [, 11], [[, 22]],, ['s1', 's2'],, 1]; + var data4 = ['hi']; + var c1a = new proto.jspb.test.Complex(data1); + var c1b = new proto.jspb.test.Complex(data1); + var c2 = new proto.jspb.test.Complex(data2); + var c3 = new proto.jspb.test.Complex(data3); + var s1 = new proto.jspb.test.Simple1(data4); + + assertTrue(jspb.Message.equals(c1a, c1b)); + assertFalse(jspb.Message.equals(c1a, c2)); + assertFalse(jspb.Message.equals(c2, c3)); + assertFalse(jspb.Message.equals(c1a, s1)); + }); + + it('testEqualsExtensionsConstructed', function() { + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions([]), + new proto.jspb.test.HasExtensions([{}]) + )); + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions(['hi', {100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions(['hi', {100: [{200: 'a'}]}]) + )); + assertFalse(jspb.Message.equals( + new proto.jspb.test.HasExtensions(['hi', {100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions(['hi', {100: [{200: 'b'}]}]) + )); + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions([{100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions([{100: [{200: 'a'}]}]) + )); + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions([{100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions([,,, {100: [{200: 'a'}]}]) + )); + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions([,,, {100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions([{100: [{200: 'a'}]}]) + )); + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions(['hi', {100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions(['hi',,, {100: [{200: 'a'}]}]) + )); + assertTrue(jspb.Message.equals( + new proto.jspb.test.HasExtensions(['hi',,, {100: [{200: 'a'}]}]), + new proto.jspb.test.HasExtensions(['hi', {100: [{200: 'a'}]}]) + )); + }); + + it('testEqualsExtensionsUnconstructed', function() { + assertTrue(jspb.Message.compareFields([], [{}])); + assertTrue(jspb.Message.compareFields([,,, {}], [])); + assertTrue(jspb.Message.compareFields([,,, {}], [,, {}])); + assertTrue(jspb.Message.compareFields( + ['hi', {100: [{200: 'a'}]}], ['hi', {100: [{200: 'a'}]}])); + assertFalse(jspb.Message.compareFields( + ['hi', {100: [{200: 'a'}]}], ['hi', {100: [{200: 'b'}]}])); + assertTrue(jspb.Message.compareFields( + [{100: [{200: 'a'}]}], [{100: [{200: 'a'}]}])); + assertTrue(jspb.Message.compareFields( + [{100: [{200: 'a'}]}], [,,, {100: [{200: 'a'}]}])); + assertTrue(jspb.Message.compareFields( + [,,, {100: [{200: 'a'}]}], [{100: [{200: 'a'}]}])); + assertTrue(jspb.Message.compareFields( + ['hi', {100: [{200: 'a'}]}], ['hi',,, {100: [{200: 'a'}]}])); + assertTrue(jspb.Message.compareFields( + ['hi',,, {100: [{200: 'a'}]}], ['hi', {100: [{200: 'a'}]}])); + }); + + it('testToMap', function() { + var p1 = new proto.jspb.test.Simple1(['k', ['v']]); + var p2 = new proto.jspb.test.Simple1(['k1', ['v1', 'v2']]); + var soymap = jspb.Message.toMap([p1, p2], + proto.jspb.test.Simple1.prototype.getAString, + proto.jspb.test.Simple1.prototype.toObject); + assertEquals('k', soymap['k'].aString); + assertArrayEquals(['v'], soymap['k'].aRepeatedStringList); + var protomap = jspb.Message.toMap([p1, p2], + proto.jspb.test.Simple1.prototype.getAString); + assertEquals('k', protomap['k'].getAString()); + assertArrayEquals(['v'], protomap['k'].getARepeatedStringList()); + }); + + it('testClone', function() { + var original = new proto.jspb.test.TestClone(); + original.setStr('v1'); + var simple1 = new proto.jspb.test.Simple1(['x1', ['y1', 'z1']]); + var simple2 = new proto.jspb.test.Simple1(['x2', ['y2', 'z2']]); + var simple3 = new proto.jspb.test.Simple1(['x3', ['y3', 'z3']]); + original.setSimple1(simple1); + original.setSimple2List([simple2, simple3]); + var extension = new proto.jspb.test.CloneExtension(); + extension.setExt('e1'); + original.setExtension(proto.jspb.test.IsExtension.extField, extension); + var clone = original.cloneMessage(); + assertArrayEquals(['v1',, ['x1', ['y1', 'z1']],, + [['x2', ['y2', 'z2']], ['x3', ['y3', 'z3']]],,, { 100: [, 'e1'] }], + clone.toArray()); + clone.setStr('v2'); + var simple4 = new proto.jspb.test.Simple1(['a1', ['b1', 'c1']]); + var simple5 = new proto.jspb.test.Simple1(['a2', ['b2', 'c2']]); + var simple6 = new proto.jspb.test.Simple1(['a3', ['b3', 'c3']]); + clone.setSimple1(simple4); + clone.setSimple2List([simple5, simple6]); + var newExtension = new proto.jspb.test.CloneExtension(); + newExtension.setExt('e2'); + clone.setExtension(proto.jspb.test.CloneExtension.extField, newExtension); + assertArrayEquals(['v2',, ['a1', ['b1', 'c1']],, + [['a2', ['b2', 'c2']], ['a3', ['b3', 'c3']]],,, { 100: [, 'e2'] }], + clone.toArray()); + assertArrayEquals(['v1',, ['x1', ['y1', 'z1']],, + [['x2', ['y2', 'z2']], ['x3', ['y3', 'z3']]],,, { 100: [, 'e1'] }], + original.toArray()); + }); + + it('testCopyInto', function() { + var original = new proto.jspb.test.TestClone(); + original.setStr('v1'); + var dest = new proto.jspb.test.TestClone(); + dest.setStr('override'); + var simple1 = new proto.jspb.test.Simple1(['x1', ['y1', 'z1']]); + var simple2 = new proto.jspb.test.Simple1(['x2', ['y2', 'z2']]); + var simple3 = new proto.jspb.test.Simple1(['x3', ['y3', 'z3']]); + var destSimple1 = new proto.jspb.test.Simple1(['ox1', ['oy1', 'oz1']]); + var destSimple2 = new proto.jspb.test.Simple1(['ox2', ['oy2', 'oz2']]); + var destSimple3 = new proto.jspb.test.Simple1(['ox3', ['oy3', 'oz3']]); + original.setSimple1(simple1); + original.setSimple2List([simple2, simple3]); + dest.setSimple1(destSimple1); + dest.setSimple2List([destSimple2, destSimple3]); + var extension = new proto.jspb.test.CloneExtension(); + extension.setExt('e1'); + original.setExtension(proto.jspb.test.CloneExtension.extField, extension); + + jspb.Message.copyInto(original, dest); + assertArrayEquals(original.toArray(), dest.toArray()); + assertEquals('x1', dest.getSimple1().getAString()); + assertEquals('e1', + dest.getExtension(proto.jspb.test.CloneExtension.extField).getExt()); + dest.getSimple1().setAString('new value'); + assertNotEquals(dest.getSimple1().getAString(), + original.getSimple1().getAString()); + dest.getExtension(proto.jspb.test.CloneExtension.extField). + setExt('new value'); + assertNotEquals( + dest.getExtension(proto.jspb.test.CloneExtension.extField).getExt(), + original.getExtension( + proto.jspb.test.CloneExtension.extField).getExt()); + }); + + it('testCopyInto_notSameType', function() { + var a = new proto.jspb.test.TestClone(); + var b = new proto.jspb.test.Simple1(['str', ['s1', 's2']]); + + var e = assertThrows(function() { + jspb.Message.copyInto(a, b); + }); + assertContains('should have the same type', e.message); + }); + + it('testExtensions', function() { + var extension1 = new proto.jspb.test.IsExtension(['ext1field']); + var extension2 = new proto.jspb.test.Simple1(['str', ['s1', 's2']]); + var extendable = new proto.jspb.test.HasExtensions(['v1', 'v2', 'v3']); + extendable.setExtension(proto.jspb.test.IsExtension.extField, extension1); + extendable.setExtension(proto.jspb.test.IndirectExtension.simple, + extension2); + extendable.setExtension(proto.jspb.test.IndirectExtension.str, 'xyzzy'); + extendable.setExtension(proto.jspb.test.IndirectExtension.repeatedStrList, + ['a', 'b']); + var s1 = new proto.jspb.test.Simple1(['foo', ['s1', 's2']]); + var s2 = new proto.jspb.test.Simple1(['bar', ['t1', 't2']]); + extendable.setExtension( + proto.jspb.test.IndirectExtension.repeatedSimpleList, + [s1, s2]); + assertObjectEquals(extension1, + extendable.getExtension(proto.jspb.test.IsExtension.extField)); + assertObjectEquals(extension2, + extendable.getExtension(proto.jspb.test.IndirectExtension.simple)); + assertObjectEquals('xyzzy', + extendable.getExtension(proto.jspb.test.IndirectExtension.str)); + assertObjectEquals(['a', 'b'], extendable.getExtension( + proto.jspb.test.IndirectExtension.repeatedStrList)); + assertObjectEquals([s1, s2], extendable.getExtension( + proto.jspb.test.IndirectExtension.repeatedSimpleList)); + // Not supported yet, but it should work... + extendable.setExtension(proto.jspb.test.IndirectExtension.simple, null); + assertNull( + extendable.getExtension(proto.jspb.test.IndirectExtension.simple)); + extendable.setExtension(proto.jspb.test.IndirectExtension.str, null); + assertNull(extendable.getExtension(proto.jspb.test.IndirectExtension.str)); + + // These assertions will only work properly in uncompiled mode. + // Extension fields defined on proto2 Descriptor messages are filtered out. + assertUndefined(proto.jspb.test.IsExtension['simpleOption']); + // Extension fields with jspb.ignore = true are ignored. + assertUndefined(proto.jspb.test.IndirectExtension['ignored']); + assertUndefined(proto.jspb.test.HasExtensions['ignoredFloating']); + }); + + it('testFloatingExtensions', function() { + // From an autogenerated container. + var extendable = new proto.jspb.test.HasExtensions(['v1', 'v2', 'v3']); + var extension = new proto.jspb.test.Simple1(['foo', ['s1', 's2']]); + extendable.setExtension(proto.jspb.test.simple1, extension); + assertObjectEquals(extension, + extendable.getExtension(proto.jspb.test.simple1)); + + // From _lib mode. + extension = new proto.jspb.test.ExtensionMessage(['s1']); + extendable = new proto.jspb.test.TestExtensionsMessage([16]); + extendable.setExtension(proto.jspb.test.floatingMsgField, extension); + extendable.setExtension(proto.jspb.test.floatingStrField, 's2'); + assertObjectEquals(extension, + extendable.getExtension(proto.jspb.test.floatingMsgField)); + assertObjectEquals('s2', + extendable.getExtension(proto.jspb.test.floatingStrField)); + assertNotUndefined(proto.jspb.exttest.floatingMsgField); + assertNotUndefined(proto.jspb.exttest.floatingMsgFieldTwo); + assertNotUndefined(proto.jspb.exttest.beta.floatingStrField); + }); + + it('testToObject_extendedObject', function() { + var extension1 = new proto.jspb.test.IsExtension(['ext1field']); + var extension2 = new proto.jspb.test.Simple1(['str', ['s1', 's2'], true]); + var extendable = new proto.jspb.test.HasExtensions(['v1', 'v2', 'v3']); + extendable.setExtension(proto.jspb.test.IsExtension.extField, extension1); + extendable.setExtension(proto.jspb.test.IndirectExtension.simple, + extension2); + extendable.setExtension(proto.jspb.test.IndirectExtension.str, 'xyzzy'); + extendable.setExtension(proto.jspb.test.IndirectExtension.repeatedStrList, + ['a', 'b']); + var s1 = new proto.jspb.test.Simple1(['foo', ['s1', 's2'], true]); + var s2 = new proto.jspb.test.Simple1(['bar', ['t1', 't2'], false]); + extendable.setExtension( + proto.jspb.test.IndirectExtension.repeatedSimpleList, + [s1, s2]); + assertObjectEquals({ + str1: 'v1', str2: 'v2', str3: 'v3', + extField: { ext1: 'ext1field' }, + simple: { + aString: 'str', aRepeatedStringList: ['s1', 's2'], aBoolean: true + }, + str: 'xyzzy', + repeatedStrList: ['a', 'b'], + repeatedSimpleList: [ + { aString: 'foo', aRepeatedStringList: ['s1', 's2'], aBoolean: true}, + { aString: 'bar', aRepeatedStringList: ['t1', 't2'], aBoolean: false} + ] + }, extendable.toObject()); + + // Now, with instances included. + assertObjectEquals({ + str1: 'v1', str2: 'v2', str3: 'v3', + extField: { + ext1: 'ext1field', + $jspbMessageInstance: + extendable.getExtension(proto.jspb.test.IsExtension.extField) + }, + simple: { + aString: 'str', + aRepeatedStringList: ['s1', 's2'], + aBoolean: true, + $jspbMessageInstance: + extendable.getExtension(proto.jspb.test.IndirectExtension.simple) + }, + str: 'xyzzy', + repeatedStrList: ['a', 'b'], + repeatedSimpleList: [{ + aString: 'foo', + aRepeatedStringList: ['s1', 's2'], + aBoolean: true, + $jspbMessageInstance: s1 + }, { + aString: 'bar', + aRepeatedStringList: ['t1', 't2'], + aBoolean: false, + $jspbMessageInstance: s2 + }], + $jspbMessageInstance: extendable + }, extendable.toObject(true /* opt_includeInstance */)); + }); + + it('testInitialization_emptyArray', function() { + var msg = new proto.jspb.test.HasExtensions([]); + if (jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS) { + assertArrayEquals([], msg.toArray()); + } else { + // Extension object is created past all regular fields. + assertArrayEquals([,,, {}], msg.toArray()); + } + }); + + it('testInitialization_justExtensionObject', function() { + var msg = new proto.jspb.test.Empty([{1: 'hi'}]); + // The extensionObject is not moved from its original location. + assertArrayEquals([{1: 'hi'}], msg.toArray()); + }); + + it('testInitialization_incompleteList', function() { + var msg = new proto.jspb.test.Empty([1, {4: 'hi'}]); + // The extensionObject is not moved from its original location. + assertArrayEquals([1, {4: 'hi'}], msg.toArray()); + }); + + it('testInitialization_forwardCompatible', function() { + var msg = new proto.jspb.test.Empty([1, 2, 3, {1: 'hi'}]); + assertArrayEquals([1, 2, 3, {1: 'hi'}], msg.toArray()); + }); + + it('testExtendedMessageEnsureObject', function() { + var data = new proto.jspb.test.HasExtensions(['str1', + {'a_key': 'an_object'}]); + assertEquals('an_object', data.extensionObject_['a_key']); + }); + + it('testToObject_hasExtensionField', function() { + var data = new proto.jspb.test.HasExtensions(['str1', {100: ['ext1']}]); + var obj = data.toObject(); + assertEquals('str1', obj.str1); + assertEquals('ext1', obj.extField.ext1); + }); + + it('testGetExtension', function() { + var data = new proto.jspb.test.HasExtensions(['str1', {100: ['ext1']}]); + assertEquals('str1', data.getStr1()); + var extension = data.getExtension(proto.jspb.test.IsExtension.extField); + assertNotNull(extension); + assertEquals('ext1', extension.getExt1()); + }); + + it('testSetExtension', function() { + var data = new proto.jspb.test.HasExtensions(); + var extensionMessage = new proto.jspb.test.IsExtension(['is_extension']); + data.setExtension(proto.jspb.test.IsExtension.extField, extensionMessage); + var obj = data.toObject(); + assertNotNull( + data.getExtension(proto.jspb.test.IsExtension.extField)); + assertEquals('is_extension', obj.extField.ext1); + }); + + /** + * Note that group is long deprecated, we only support it because JsPb has + * a goal of being able to generate JS classes for all proto descriptors. + */ + it('testGroups', function() { + var group = new proto.jspb.test.TestGroup(); + var someGroup = new proto.jspb.test.TestGroup.RepeatedGroup(); + someGroup.setId('g1'); + someGroup.setSomeBoolList([true, false]); + group.setRepeatedGroupList([someGroup]); + var groups = group.getRepeatedGroupList(); + assertEquals('g1', groups[0].getId()); + assertObjectEquals([true, false], groups[0].getSomeBoolList()); + assertObjectEquals({id: 'g1', someBoolList: [true, false]}, + groups[0].toObject()); + assertObjectEquals({ + repeatedGroupList: [{id: 'g1', someBoolList: [true, false]}], + requiredGroup: {id: undefined}, + optionalGroup: undefined, + requiredSimple: {aRepeatedStringList: [], aString: undefined}, + optionalSimple: undefined, + id: undefined + }, group.toObject()); + var group1 = new proto.jspb.test.TestGroup1(); + group1.setGroup(someGroup); + assertEquals(someGroup, group1.getGroup()); + }); + + it('testNonExtensionFieldsAfterExtensionRange', function() { + var data = [{'1': 'a_string'}]; + var message = new proto.jspb.test.Complex(data); + assertArrayEquals([], message.getARepeatedStringList()); + }); + + it('testReservedGetterNames', function() { + var message = new proto.jspb.test.TestReservedNames(); + message.setExtension$(11); + message.setExtension(proto.jspb.test.TestReservedNamesExtension.foo, 12); + assertEquals(11, message.getExtension$()); + assertEquals(12, message.getExtension( + proto.jspb.test.TestReservedNamesExtension.foo)); + assertObjectEquals({extension: 11, foo: 12}, message.toObject()); + }); + + it('testInitializeMessageWithUnsetOneof', function() { + var message = new proto.jspb.test.TestMessageWithOneof([]); + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase. + PARTIAL_ONEOF_NOT_SET, + message.getPartialOneofCase()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.RecursiveOneofCase. + RECURSIVE_ONEOF_NOT_SET, + message.getRecursiveOneofCase()); + }); + + it('testInitializeMessageWithSingleValueSetInOneof', function() { + var message = new proto.jspb.test.TestMessageWithOneof([,, 'x']); + + assertEquals('x', message.getPone()); + assertUndefined(message.getPthree()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase.PONE, + message.getPartialOneofCase()); + }); + + it('testKeepsLastWireValueSetInUnion_multipleValues', function() { + var message = new proto.jspb.test.TestMessageWithOneof([,, 'x',, 'y']); + + assertUndefined('x', message.getPone()); + assertEquals('y', message.getPthree()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase.PTHREE, + message.getPartialOneofCase()); + }); + + it('testSettingOneofFieldClearsOthers', function() { + var message = new proto.jspb.test.TestMessageWithOneof; + assertUndefined(message.getPone()); + assertUndefined(message.getPthree()); + + message.setPone('hi'); + assertEquals('hi', message.getPone()); + assertUndefined(message.getPthree()); + + message.setPthree('bye'); + assertUndefined(message.getPone()); + assertEquals('bye', message.getPthree()); + }); + + it('testSettingOneofFieldDoesNotClearFieldsFromOtherUnions', function() { + var other = new proto.jspb.test.TestMessageWithOneof; + var message = new proto.jspb.test.TestMessageWithOneof; + assertUndefined(message.getPone()); + assertUndefined(message.getPthree()); + assertUndefined(message.getRone()); + + message.setPone('hi'); + message.setRone(other); + assertEquals('hi', message.getPone()); + assertUndefined(message.getPthree()); + assertEquals(other, message.getRone()); + + message.setPthree('bye'); + assertUndefined(message.getPone()); + assertEquals('bye', message.getPthree()); + assertEquals(other, message.getRone()); + }); + + it('testUnsetsOneofCaseWhenFieldIsCleared', function() { + var message = new proto.jspb.test.TestMessageWithOneof; + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase. + PARTIAL_ONEOF_NOT_SET, + message.getPartialOneofCase()); + + message.setPone('hi'); + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase.PONE, + message.getPartialOneofCase()); + + message.clearPone(); + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase. + PARTIAL_ONEOF_NOT_SET, + message.getPartialOneofCase()); + }); + + it('testMessageWithDefaultOneofValues', function() { + var message = new proto.jspb.test.TestMessageWithOneof; + assertEquals(1234, message.getAone()); + assertUndefined(message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase + .DEFAULT_ONEOF_A_NOT_SET, + message.getDefaultOneofACase()); + + message.setAone(567); + assertEquals(567, message.getAone()); + assertUndefined(message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase.AONE, + message.getDefaultOneofACase()); + + message.setAtwo(890); + assertEquals(1234, message.getAone()); + assertEquals(890, message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase.ATWO, + message.getDefaultOneofACase()); + + message.clearAtwo(); + assertEquals(1234, message.getAone()); + assertUndefined(message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase + .DEFAULT_ONEOF_A_NOT_SET, + message.getDefaultOneofACase()); + }); + + it('testMessageWithDefaultOneofValues_defaultNotOnFirstField', function() { + var message = new proto.jspb.test.TestMessageWithOneof; + assertUndefined(message.getBone()); + assertEquals(1234, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase + .DEFAULT_ONEOF_B_NOT_SET, + message.getDefaultOneofBCase()); + + message.setBone(2); + assertEquals(2, message.getBone()); + assertEquals(1234, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BONE, + message.getDefaultOneofBCase()); + + message.setBtwo(3); + assertUndefined(message.getBone()); + assertEquals(3, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BTWO, + message.getDefaultOneofBCase()); + + message.clearBtwo(); + assertUndefined(message.getBone()); + assertEquals(1234, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase + .DEFAULT_ONEOF_B_NOT_SET, + message.getDefaultOneofBCase()); + }); + + it('testInitializeMessageWithOneofDefaults', function() { + var message = + new proto.jspb.test.TestMessageWithOneof(new Array(9).concat(567)); + assertEquals(567, message.getAone()); + assertUndefined(message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase.AONE, + message.getDefaultOneofACase()); + + message = + new proto.jspb.test.TestMessageWithOneof(new Array(10).concat(890)); + assertEquals(1234, message.getAone()); + assertEquals(890, message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase.ATWO, + message.getDefaultOneofACase()); + + message = + new proto.jspb.test.TestMessageWithOneof(new Array(9).concat(567,890)); + assertEquals(1234, message.getAone()); + assertEquals(890, message.getAtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofACase.ATWO, + message.getDefaultOneofACase()); + }); + + it('testInitializeMessageWithOneofDefaults_defaultNotSetOnFirstField', + function() { + var message; + + message = + new proto.jspb.test.TestMessageWithOneof(new Array(11).concat(567)); + assertEquals(567, message.getBone()); + assertEquals(1234, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BONE, + message.getDefaultOneofBCase()); + + message = + new proto.jspb.test.TestMessageWithOneof(new Array(12).concat(890)); + assertUndefined(message.getBone()); + assertEquals(890, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BTWO, + message.getDefaultOneofBCase()); + + message = new proto.jspb.test.TestMessageWithOneof( + new Array(11).concat(567,890)); + assertUndefined(message.getBone()); + assertEquals(890, message.getBtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.DefaultOneofBCase.BTWO, + message.getDefaultOneofBCase()); + }); + + it('testOneofContainingAnotherMessage', function() { + var message = new proto.jspb.test.TestMessageWithOneof; + assertEquals( + proto.jspb.test.TestMessageWithOneof.RecursiveOneofCase. + RECURSIVE_ONEOF_NOT_SET, + message.getRecursiveOneofCase()); + + var other = new proto.jspb.test.TestMessageWithOneof; + message.setRone(other); + assertEquals(other, message.getRone()); + assertUndefined(message.getRtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.RecursiveOneofCase.RONE, + message.getRecursiveOneofCase()); + + message.setRtwo('hi'); + assertUndefined(message.getRone()); + assertEquals('hi', message.getRtwo()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.RecursiveOneofCase.RTWO, + message.getRecursiveOneofCase()); + }); + + it('testQueryingOneofCaseEnsuresOnlyOneFieldIsSetInUnderlyingArray', + function() { + var message = new proto.jspb.test.TestMessageWithOneof; + message.setPone('x'); + assertEquals('x', message.getPone()); + assertUndefined(message.getPthree()); + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase.PONE, + message.getPartialOneofCase()); + + var array = message.toArray(); + assertEquals('x', array[2]); + assertUndefined(array[4]); + array[4] = 'y'; + + assertEquals( + proto.jspb.test.TestMessageWithOneof.PartialOneofCase.PTHREE, + message.getPartialOneofCase()); + assertUndefined(array[2]); + assertEquals('y', array[4]); + }); + +}); diff --git a/js/proto3_test.js b/js/proto3_test.js new file mode 100644 index 00000000..8102bab6 --- /dev/null +++ b/js/proto3_test.js @@ -0,0 +1,279 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +goog.require('goog.testing.asserts'); +goog.require('proto.jspb.test.ForeignMessage'); +goog.require('proto.jspb.test.Proto3Enum'); +goog.require('proto.jspb.test.TestProto3'); + +/** + * Helper: compare a bytes field to a string with codepoints 0--255. + * @param {Uint8Array|string} arr + * @param {string} str + * @return {boolean} + */ +function bytesCompare(arr, str) { + if (arr.length != str.length) { + return false; + } + if (typeof arr == 'string') { + for (var i = 0; i < arr.length; i++) { + if (arr.charCodeAt(i) != str.charCodeAt(i)) { + return false; + } + } + return true; + } else { + for (var i = 0; i < arr.length; i++) { + if (arr[i] != str.charCodeAt(i)) { + return false; + } + } + return true; + } +} + + +describe('proto3Test', function() { + /** + * Test defaults for proto3 message fields. + */ + it('testProto3FieldDefaults', function() { + var msg = new proto.jspb.test.TestProto3(); + + assertEquals(msg.getOptionalInt32(), 0); + assertEquals(msg.getOptionalInt64(), 0); + assertEquals(msg.getOptionalUint32(), 0); + assertEquals(msg.getOptionalUint64(), 0); + assertEquals(msg.getOptionalSint32(), 0); + assertEquals(msg.getOptionalSint64(), 0); + assertEquals(msg.getOptionalFixed32(), 0); + assertEquals(msg.getOptionalFixed64(), 0); + assertEquals(msg.getOptionalSfixed32(), 0); + assertEquals(msg.getOptionalSfixed64(), 0); + assertEquals(msg.getOptionalFloat(), 0); + assertEquals(msg.getOptionalDouble(), 0); + assertEquals(msg.getOptionalString(), ''); + + // If/when we change bytes fields to return Uint8Array, we'll want to switch + // to this assertion instead: + //assertEquals(msg.getOptionalBytes() instanceof Uint8Array, true); + assertEquals(typeof msg.getOptionalBytes(), 'string'); + + assertEquals(msg.getOptionalBytes().length, 0); + assertEquals(msg.getOptionalForeignEnum(), proto.jspb.test.Proto3Enum.PROTO3_FOO); + assertEquals(msg.getOptionalForeignMessage(), undefined); + assertEquals(msg.getOptionalForeignMessage(), undefined); + + assertEquals(msg.getRepeatedInt32List().length, 0); + assertEquals(msg.getRepeatedInt64List().length, 0); + assertEquals(msg.getRepeatedUint32List().length, 0); + assertEquals(msg.getRepeatedUint64List().length, 0); + assertEquals(msg.getRepeatedSint32List().length, 0); + assertEquals(msg.getRepeatedSint64List().length, 0); + assertEquals(msg.getRepeatedFixed32List().length, 0); + assertEquals(msg.getRepeatedFixed64List().length, 0); + assertEquals(msg.getRepeatedSfixed32List().length, 0); + assertEquals(msg.getRepeatedSfixed64List().length, 0); + assertEquals(msg.getRepeatedFloatList().length, 0); + assertEquals(msg.getRepeatedDoubleList().length, 0); + assertEquals(msg.getRepeatedStringList().length, 0); + assertEquals(msg.getRepeatedBytesList().length, 0); + assertEquals(msg.getRepeatedForeignEnumList().length, 0); + assertEquals(msg.getRepeatedForeignMessageList().length, 0); + + }); + + + /** + * Test that all fields can be set and read via a serialization roundtrip. + */ + it('testProto3FieldSetGet', function() { + var msg = new proto.jspb.test.TestProto3(); + + msg.setOptionalInt32(-42); + msg.setOptionalInt64(-0x7fffffff00000000); + msg.setOptionalUint32(0x80000000); + msg.setOptionalUint64(0xf000000000000000); + msg.setOptionalSint32(-100); + msg.setOptionalSint64(-0x8000000000000000); + msg.setOptionalFixed32(1234); + msg.setOptionalFixed64(0x1234567800000000); + msg.setOptionalSfixed32(-1234); + msg.setOptionalSfixed64(-0x1234567800000000); + msg.setOptionalFloat(1.5); + msg.setOptionalDouble(-1.5); + msg.setOptionalBool(true); + msg.setOptionalString('hello world'); + msg.setOptionalBytes('bytes'); + var submsg = new proto.jspb.test.ForeignMessage(); + submsg.setC(16); + msg.setOptionalForeignMessage(submsg); + msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR); + + msg.setRepeatedInt32List([-42]); + msg.setRepeatedInt64List([-0x7fffffff00000000]); + msg.setRepeatedUint32List([0x80000000]); + msg.setRepeatedUint64List([0xf000000000000000]); + msg.setRepeatedSint32List([-100]); + msg.setRepeatedSint64List([-0x8000000000000000]); + msg.setRepeatedFixed32List([1234]); + msg.setRepeatedFixed64List([0x1234567800000000]); + msg.setRepeatedSfixed32List([-1234]); + msg.setRepeatedSfixed64List([-0x1234567800000000]); + msg.setRepeatedFloatList([1.5]); + msg.setRepeatedDoubleList([-1.5]); + msg.setRepeatedBoolList([true]); + msg.setRepeatedStringList(['hello world']); + msg.setRepeatedBytesList(['bytes']); + submsg = new proto.jspb.test.ForeignMessage(); + submsg.setC(1000); + msg.setRepeatedForeignMessageList([submsg]); + msg.setRepeatedForeignEnumList([proto.jspb.test.Proto3Enum.PROTO3_BAR]); + + msg.setOneofString('asdf'); + + var serialized = msg.serializeBinary(); + msg = proto.jspb.test.TestProto3.deserializeBinary(serialized); + + assertEquals(msg.getOptionalInt32(), -42); + assertEquals(msg.getOptionalInt64(), -0x7fffffff00000000); + assertEquals(msg.getOptionalUint32(), 0x80000000); + assertEquals(msg.getOptionalUint64(), 0xf000000000000000); + assertEquals(msg.getOptionalSint32(), -100); + assertEquals(msg.getOptionalSint64(), -0x8000000000000000); + assertEquals(msg.getOptionalFixed32(), 1234); + assertEquals(msg.getOptionalFixed64(), 0x1234567800000000); + assertEquals(msg.getOptionalSfixed32(), -1234); + assertEquals(msg.getOptionalSfixed64(), -0x1234567800000000); + assertEquals(msg.getOptionalFloat(), 1.5); + assertEquals(msg.getOptionalDouble(), -1.5); + assertEquals(msg.getOptionalBool(), true); + assertEquals(msg.getOptionalString(), 'hello world'); + assertEquals(true, bytesCompare(msg.getOptionalBytes(), 'bytes')); + assertEquals(msg.getOptionalForeignMessage().getC(), 16); + assertEquals(msg.getOptionalForeignEnum(), + proto.jspb.test.Proto3Enum.PROTO3_BAR); + + assertElementsEquals(msg.getRepeatedInt32List(), [-42]); + assertElementsEquals(msg.getRepeatedInt64List(), [-0x7fffffff00000000]); + assertElementsEquals(msg.getRepeatedUint32List(), [0x80000000]); + assertElementsEquals(msg.getRepeatedUint64List(), [0xf000000000000000]); + assertElementsEquals(msg.getRepeatedSint32List(), [-100]); + assertElementsEquals(msg.getRepeatedSint64List(), [-0x8000000000000000]); + assertElementsEquals(msg.getRepeatedFixed32List(), [1234]); + assertElementsEquals(msg.getRepeatedFixed64List(), [0x1234567800000000]); + assertElementsEquals(msg.getRepeatedSfixed32List(), [-1234]); + assertElementsEquals(msg.getRepeatedSfixed64List(), [-0x1234567800000000]); + assertElementsEquals(msg.getRepeatedFloatList(), [1.5]); + assertElementsEquals(msg.getRepeatedDoubleList(), [-1.5]); + assertElementsEquals(msg.getRepeatedBoolList(), [true]); + assertElementsEquals(msg.getRepeatedStringList(), ['hello world']); + assertEquals(msg.getRepeatedBytesList().length, 1); + assertEquals(true, bytesCompare(msg.getRepeatedBytesList()[0], 'bytes')); + assertEquals(msg.getRepeatedForeignMessageList().length, 1); + assertEquals(msg.getRepeatedForeignMessageList()[0].getC(), 1000); + assertElementsEquals(msg.getRepeatedForeignEnumList(), + [proto.jspb.test.Proto3Enum.PROTO3_BAR]); + + assertEquals(msg.getOneofString(), 'asdf'); + }); + + + /** + * Test that oneofs continue to have a notion of field presence. + */ + it('testOneofs', function() { + var msg = new proto.jspb.test.TestProto3(); + + assertEquals(msg.getOneofUint32(), undefined); + assertEquals(msg.getOneofForeignMessage(), undefined); + assertEquals(msg.getOneofString(), undefined); + assertEquals(msg.getOneofBytes(), undefined); + + msg.setOneofUint32(42); + assertEquals(msg.getOneofUint32(), 42); + assertEquals(msg.getOneofForeignMessage(), undefined); + assertEquals(msg.getOneofString(), undefined); + assertEquals(msg.getOneofBytes(), undefined); + + + var submsg = new proto.jspb.test.ForeignMessage(); + msg.setOneofForeignMessage(submsg); + assertEquals(msg.getOneofUint32(), undefined); + assertEquals(msg.getOneofForeignMessage(), submsg); + assertEquals(msg.getOneofString(), undefined); + assertEquals(msg.getOneofBytes(), undefined); + + msg.setOneofString('hello'); + assertEquals(msg.getOneofUint32(), undefined); + assertEquals(msg.getOneofForeignMessage(), undefined); + assertEquals(msg.getOneofString(), 'hello'); + assertEquals(msg.getOneofBytes(), undefined); + + msg.setOneofBytes('\u00FF\u00FF'); + assertEquals(msg.getOneofUint32(), undefined); + assertEquals(msg.getOneofForeignMessage(), undefined); + assertEquals(msg.getOneofString(), undefined); + assertEquals(msg.getOneofBytes(), '\u00FF\u00FF'); + }); + + + /** + * Test that "default"-valued primitive fields are not emitted on the wire. + */ + it('testNoSerializeDefaults', function() { + var msg = new proto.jspb.test.TestProto3(); + + // Set each primitive to a non-default value, then back to its default, to + // ensure that the serialization is actually checking the value and not just + // whether it has ever been set. + msg.setOptionalInt32(42); + msg.setOptionalInt32(0); + msg.setOptionalDouble(3.14); + msg.setOptionalDouble(0.0); + msg.setOptionalBool(true); + msg.setOptionalBool(false); + msg.setOptionalString('hello world'); + msg.setOptionalString(''); + msg.setOptionalBytes('\u00FF\u00FF'); + msg.setOptionalBytes(''); + msg.setOptionalForeignMessage(new proto.jspb.test.ForeignMessage()); + msg.setOptionalForeignMessage(null); + msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_BAR); + msg.setOptionalForeignEnum(proto.jspb.test.Proto3Enum.PROTO3_FOO); + msg.setOneofUint32(32); + msg.setOneofUint32(null); + + + var serialized = msg.serializeBinary(); + assertEquals(0, serialized.length); + }); +}); diff --git a/js/proto3_test.proto b/js/proto3_test.proto new file mode 100644 index 00000000..acb67164 --- /dev/null +++ b/js/proto3_test.proto @@ -0,0 +1,89 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +import "testbinary.proto"; + +package jspb.test; + +message TestProto3 { + int32 optional_int32 = 1; + int64 optional_int64 = 2; + uint32 optional_uint32 = 3; + uint64 optional_uint64 = 4; + sint32 optional_sint32 = 5; + sint64 optional_sint64 = 6; + fixed32 optional_fixed32 = 7; + fixed64 optional_fixed64 = 8; + sfixed32 optional_sfixed32 = 9; + sfixed64 optional_sfixed64 = 10; + float optional_float = 11; + double optional_double = 12; + bool optional_bool = 13; + string optional_string = 14; + bytes optional_bytes = 15; + + ForeignMessage optional_foreign_message = 19; + Proto3Enum optional_foreign_enum = 22; + + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; + repeated sfixed32 repeated_sfixed32 = 39; + repeated sfixed64 repeated_sfixed64 = 40; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; + + repeated ForeignMessage repeated_foreign_message = 49; + repeated Proto3Enum repeated_foreign_enum = 52; + + + oneof oneof_field { + uint32 oneof_uint32 = 111; + ForeignMessage oneof_foreign_message = 112; + string oneof_string = 113; + bytes oneof_bytes = 114; + } +} + +enum Proto3Enum { + PROTO3_FOO = 0; + PROTO3_BAR = 1; + PROTO3_BAZ = 2; +} diff --git a/js/test.proto b/js/test.proto new file mode 100644 index 00000000..5f9078ef --- /dev/null +++ b/js/test.proto @@ -0,0 +1,212 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: mwr@google.com (Mark Rawling) + +syntax = "proto2"; + +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + +import "google/protobuf/descriptor.proto"; + +package jspb.test; + +message Empty { +} + +enum OuterEnum { + FOO = 1; + BAR = 2; +} + +message EnumContainer { + optional OuterEnum outer_enum = 1; +} + +message Simple1 { + required string a_string = 1; + repeated string a_repeated_string = 2; + optional bool a_boolean = 3; +} + +// A message that differs from Simple1 only by name +message Simple2 { + required string a_string = 1; + repeated string a_repeated_string = 2; +} + +message SpecialCases { + required string normal = 1; + // Examples of Js reserved names that are converted to pb_. + required string default = 2; + required string function = 3; + required string var = 4; +} + +message OptionalFields { + message Nested { + optional int32 an_int = 1; + } + optional string a_string = 1; + required bool a_bool = 2; + optional Nested a_nested_message = 3; + repeated Nested a_repeated_message = 4; + repeated string a_repeated_string = 5; +} + +message HasExtensions { + optional string str1 = 1; + optional string str2 = 2; + optional string str3 = 3; + extensions 10 to max; +} + +message Complex { + message Nested { + required int32 an_int = 2; + } + required string a_string = 1; + required bool an_out_of_order_bool = 9; + optional Nested a_nested_message = 4; + repeated Nested a_repeated_message = 5; + repeated string a_repeated_string = 7; +} + +message IsExtension { + extend HasExtensions { + optional IsExtension ext_field = 100; + } + optional string ext1 = 1; + + // Extensions of proto2 Descriptor messages will be ignored. + extend google.protobuf.EnumOptions { + optional string simple_option = 42113038; + } +} + +message IndirectExtension { + extend HasExtensions { + optional Simple1 simple = 101; + optional string str = 102; + repeated string repeated_str = 103; + repeated Simple1 repeated_simple = 104; + } +} + +extend HasExtensions { + optional Simple1 simple1 = 105; +} + +message DefaultValues { + enum Enum { + E1 = 13; + E2 = 77; + } + optional string string_field = 1 [default="default<>\'\"abc"]; + optional bool bool_field = 2 [default=true]; + optional int64 int_field = 3 [default=11]; + optional Enum enum_field = 4 [default=E1]; + optional string empty_field = 6 [default=""]; + optional bytes bytes_field = 8 [default="moo"]; // Base64 encoding is "bW9v" +} + +message TestClone { + optional string str = 1; + optional Simple1 simple1 = 3; + repeated Simple1 simple2 = 5; + optional string unused = 7; + extensions 10 to max; +} + +message CloneExtension { + extend TestClone { + optional CloneExtension ext_field = 100; + } + optional string ext = 2; +} + +message TestGroup { + repeated group RepeatedGroup = 1 { + required string id = 1; + repeated bool some_bool = 2; + } + required group RequiredGroup = 2 { + required string id = 1; + } + optional group OptionalGroup = 3 { + required string id = 1; + } + optional string id = 4; + required Simple2 required_simple = 5; + optional Simple2 optional_simple = 6; +} + +message TestGroup1 { + optional TestGroup.RepeatedGroup group = 1; +} + +message TestReservedNames { + optional int32 extension = 1; + extensions 10 to max; +} + +message TestReservedNamesExtension { + extend TestReservedNames { + optional int32 foo = 10; + } +} + +message TestMessageWithOneof { + + oneof partial_oneof { + string pone = 3; + string pthree = 5; + } + + oneof recursive_oneof { + TestMessageWithOneof rone = 6; + string rtwo = 7; + } + + optional bool normal_field = 8; + repeated string repeated_field = 9; + + oneof default_oneof_a { + int32 aone = 10 [default = 1234]; + int32 atwo = 11; + } + + oneof default_oneof_b { + int32 bone = 12; + int32 btwo = 13 [default = 1234]; + } +} + diff --git a/js/test2.proto b/js/test2.proto new file mode 100644 index 00000000..44e55eff --- /dev/null +++ b/js/test2.proto @@ -0,0 +1,54 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + +package jspb.test; + +message TestExtensionsMessage { + optional int32 intfield = 1; + extensions 100 to max; +} + +message ExtensionMessage { + extend TestExtensionsMessage { + optional ExtensionMessage ext_field = 100; + } + optional string ext1 = 1; +} + +// Floating extensions are only supported when generating a _lib.js library. +extend TestExtensionsMessage { + optional ExtensionMessage floating_msg_field = 101; + optional string floating_str_field = 102; +} diff --git a/js/test3.proto b/js/test3.proto new file mode 100644 index 00000000..940a552e --- /dev/null +++ b/js/test3.proto @@ -0,0 +1,53 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + +package jspb.exttest; + +message TestExtensionsMessage { + optional int32 intfield = 1; + extensions 100 to max; +} + +message ExtensionMessage { + extend TestExtensionsMessage { + optional ExtensionMessage ext_field = 100; + } + optional string ext1 = 1; +} + +extend TestExtensionsMessage { + optional ExtensionMessage floating_msg_field = 101; + optional string floating_str_field = 102; +} diff --git a/js/test4.proto b/js/test4.proto new file mode 100644 index 00000000..cf2451e9 --- /dev/null +++ b/js/test4.proto @@ -0,0 +1,42 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + +package jspb.exttest; + +import "test3.proto"; + +extend TestExtensionsMessage { + optional ExtensionMessage floating_msg_field_two = 103; +} diff --git a/js/test5.proto b/js/test5.proto new file mode 100644 index 00000000..34979517 --- /dev/null +++ b/js/test5.proto @@ -0,0 +1,44 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +option java_package = "com.google.apps.jspb.proto"; +option java_multiple_files = true; + +package jspb.exttest.beta; + +message TestBetaExtensionsMessage { + extensions 100 to max; +} + +extend TestBetaExtensionsMessage { + optional string floating_str_field = 101; +} diff --git a/js/test_bootstrap.js b/js/test_bootstrap.js new file mode 100644 index 00000000..9d00a1c4 --- /dev/null +++ b/js/test_bootstrap.js @@ -0,0 +1,41 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/** + * @fileoverview Sets flags for uncompiled JSUnit tests. + */ + +/** + * Set uncompiled flags. + */ +var CLOSURE_DEFINES = { + // Enable the fromObject method on the message class. + 'jspb.Message.GENERATE_FROM_OBJECT': true +}; diff --git a/js/testbinary.proto b/js/testbinary.proto new file mode 100644 index 00000000..60c70190 --- /dev/null +++ b/js/testbinary.proto @@ -0,0 +1,185 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// LINT: ALLOW_GROUPS + +syntax = "proto2"; + + +package jspb.test; + +// These types are borrowed from `unittest.proto` in the protobuf tree. We want +// to ensure that the binary-format support will handle all field types +// properly. +message TestAllTypes { + optional int32 optional_int32 = 1; + optional int64 optional_int64 = 2; + optional uint32 optional_uint32 = 3; + optional uint64 optional_uint64 = 4; + optional sint32 optional_sint32 = 5; + optional sint64 optional_sint64 = 6; + optional fixed32 optional_fixed32 = 7; + optional fixed64 optional_fixed64 = 8; + optional sfixed32 optional_sfixed32 = 9; + optional sfixed64 optional_sfixed64 = 10; + optional float optional_float = 11; + optional double optional_double = 12; + optional bool optional_bool = 13; + optional string optional_string = 14; + optional bytes optional_bytes = 15; + optional group OptionalGroup = 16 { + optional int32 a = 17; + } + + optional ForeignMessage optional_foreign_message = 19; + optional ForeignEnum optional_foreign_enum = 22; + + // Repeated + repeated int32 repeated_int32 = 31; + repeated int64 repeated_int64 = 32; + repeated uint32 repeated_uint32 = 33; + repeated uint64 repeated_uint64 = 34; + repeated sint32 repeated_sint32 = 35; + repeated sint64 repeated_sint64 = 36; + repeated fixed32 repeated_fixed32 = 37; + repeated fixed64 repeated_fixed64 = 38; + repeated sfixed32 repeated_sfixed32 = 39; + repeated sfixed64 repeated_sfixed64 = 40; + repeated float repeated_float = 41; + repeated double repeated_double = 42; + repeated bool repeated_bool = 43; + repeated string repeated_string = 44; + repeated bytes repeated_bytes = 45; + + repeated group RepeatedGroup = 46 { + optional int32 a = 47; + } + + repeated ForeignMessage repeated_foreign_message = 49; + repeated ForeignEnum repeated_foreign_enum = 52; + + // Packed repeated + repeated int32 packed_repeated_int32 = 61 [packed=true]; + repeated int64 packed_repeated_int64 = 62 [packed=true]; + repeated uint32 packed_repeated_uint32 = 63 [packed=true]; + repeated uint64 packed_repeated_uint64 = 64 [packed=true]; + repeated sint32 packed_repeated_sint32 = 65 [packed=true]; + repeated sint64 packed_repeated_sint64 = 66 [packed=true]; + repeated fixed32 packed_repeated_fixed32 = 67 [packed=true]; + repeated fixed64 packed_repeated_fixed64 = 68 [packed=true]; + repeated sfixed32 packed_repeated_sfixed32 = 69 [packed=true]; + repeated sfixed64 packed_repeated_sfixed64 = 70 [packed=true]; + repeated float packed_repeated_float = 71 [packed=true]; + repeated double packed_repeated_double = 72 [packed=true]; + repeated bool packed_repeated_bool = 73 [packed=true]; + + oneof oneof_field { + uint32 oneof_uint32 = 111; + ForeignMessage oneof_foreign_message = 112; + string oneof_string = 113; + bytes oneof_bytes = 114; + } + +} + +message ForeignMessage { + optional int32 c = 1; +} + +enum ForeignEnum { + FOREIGN_FOO = 4; + FOREIGN_BAR = 5; + FOREIGN_BAZ = 6; +} + +message TestExtendable { + extensions 1 to max; +} + +message ExtendsWithMessage { + extend TestExtendable { + optional ExtendsWithMessage optional_extension = 19; + repeated ExtendsWithMessage repeated_extension = 49; + } + optional int32 foo = 1; +} + +extend TestExtendable { + optional int32 extend_optional_int32 = 1; + optional int64 extend_optional_int64 = 2; + optional uint32 extend_optional_uint32 = 3; + optional uint64 extend_optional_uint64 = 4; + optional sint32 extend_optional_sint32 = 5; + optional sint64 extend_optional_sint64 = 6; + optional fixed32 extend_optional_fixed32 = 7; + optional fixed64 extend_optional_fixed64 = 8; + optional sfixed32 extend_optional_sfixed32 = 9; + optional sfixed64 extend_optional_sfixed64 = 10; + optional float extend_optional_float = 11; + optional double extend_optional_double = 12; + optional bool extend_optional_bool = 13; + optional string extend_optional_string = 14; + optional bytes extend_optional_bytes = 15; + optional ForeignEnum extend_optional_foreign_enum = 22; + + repeated int32 extend_repeated_int32 = 31; + repeated int64 extend_repeated_int64 = 32; + repeated uint32 extend_repeated_uint32 = 33; + repeated uint64 extend_repeated_uint64 = 34; + repeated sint32 extend_repeated_sint32 = 35; + repeated sint64 extend_repeated_sint64 = 36; + repeated fixed32 extend_repeated_fixed32 = 37; + repeated fixed64 extend_repeated_fixed64 = 38; + repeated sfixed32 extend_repeated_sfixed32 = 39; + repeated sfixed64 extend_repeated_sfixed64 = 40; + repeated float extend_repeated_float = 41; + repeated double extend_repeated_double = 42; + repeated bool extend_repeated_bool = 43; + repeated string extend_repeated_string = 44; + repeated bytes extend_repeated_bytes = 45; + repeated ForeignEnum extend_repeated_foreign_enum = 52; + + repeated int32 extend_packed_repeated_int32 = 61 [packed=true]; + repeated int64 extend_packed_repeated_int64 = 62 [packed=true]; + repeated uint32 extend_packed_repeated_uint32 = 63 [packed=true]; + repeated uint64 extend_packed_repeated_uint64 = 64 [packed=true]; + repeated sint32 extend_packed_repeated_sint32 = 65 [packed=true]; + repeated sint64 extend_packed_repeated_sint64 = 66 [packed=true]; + repeated fixed32 extend_packed_repeated_fixed32 = 67 [packed=true]; + repeated fixed64 extend_packed_repeated_fixed64 = 68 [packed=true]; + repeated sfixed32 extend_packed_repeated_sfixed32 = 69 [packed=true]; + repeated sfixed64 extend_packed_repeated_sfixed64 = 70 [packed=true]; + repeated float extend_packed_repeated_float = 71 [packed=true]; + repeated double extend_packed_repeated_double = 72 [packed=true]; + repeated bool extend_packed_repeated_bool = 73 [packed=true]; + repeated ForeignEnum extend_packed_repeated_foreign_enum = 82 + [packed=true]; + +} diff --git a/js/testempty.proto b/js/testempty.proto new file mode 100644 index 00000000..960bce4e --- /dev/null +++ b/js/testempty.proto @@ -0,0 +1,34 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto2"; + +package javatests.com.google.apps.jspb; + diff --git a/objectivec/google/protobuf/Any.pbobjc.h b/objectivec/google/protobuf/Any.pbobjc.h index d2261ee4..9866b5b1 100644 --- a/objectivec/google/protobuf/Any.pbobjc.h +++ b/objectivec/google/protobuf/Any.pbobjc.h @@ -34,6 +34,7 @@ typedef GPB_ENUM(GPBAny_FieldNumber) { // `Any` contains an arbitrary serialized message along with a URL // that describes the type of the serialized message. // +// // JSON // ==== // The JSON representation of an `Any` value uses the regular @@ -54,8 +55,8 @@ typedef GPB_ENUM(GPBAny_FieldNumber) { // // If the embedded message type is well-known and has a custom JSON // representation, that representation will be embedded adding a field -// `value` which holds the custom JSON in addition to the the `@type` -// field. Example (for message [google.protobuf.Duration][google.protobuf.Duration]): +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): // // { // "@type": "type.googleapis.com/google.protobuf.Duration", @@ -72,7 +73,7 @@ typedef GPB_ENUM(GPBAny_FieldNumber) { // * If no schema is provided, `https` is assumed. // * The last segment of the URL's path must represent the fully // qualified name of the type (as in `path/google.protobuf.Duration`). -// * An HTTP GET on the URL must yield a [google.protobuf.Type][google.protobuf.Type] +// * An HTTP GET on the URL must yield a [google.protobuf.Type][] // value in binary format, or produce an error. // * Applications are allowed to cache lookup results based on the // URL, or have them precompiled into a binary to avoid any diff --git a/objectivec/google/protobuf/Api.pbobjc.h b/objectivec/google/protobuf/Api.pbobjc.h index c9dacfee..c3cf8e94 100644 --- a/objectivec/google/protobuf/Api.pbobjc.h +++ b/objectivec/google/protobuf/Api.pbobjc.h @@ -174,7 +174,6 @@ typedef GPB_ENUM(GPBMixin_FieldNumber) { // // package google.storage.v2; // service Storage { -// // (-- see AccessControl.GetAcl --) // rpc GetAcl(GetAclRequest) returns (Acl); // // // Get a data record. diff --git a/objectivec/google/protobuf/FieldMask.pbobjc.h b/objectivec/google/protobuf/FieldMask.pbobjc.h index 67cea4d6..4e4ec387 100644 --- a/objectivec/google/protobuf/FieldMask.pbobjc.h +++ b/objectivec/google/protobuf/FieldMask.pbobjc.h @@ -61,7 +61,7 @@ typedef GPB_ENUM(GPBFieldMask_FieldNumber) { // z: 8 // // The result will not contain specific values for fields x,y and z -// (there value will be set to the default, and omitted in proto text +// (their value will be set to the default, and omitted in proto text // output): // // diff --git a/objectivec/google/protobuf/Type.pbobjc.h b/objectivec/google/protobuf/Type.pbobjc.h index 65f1da46..e4c7a251 100644 --- a/objectivec/google/protobuf/Type.pbobjc.h +++ b/objectivec/google/protobuf/Type.pbobjc.h @@ -18,13 +18,13 @@ NS_ASSUME_NONNULL_BEGIN #pragma mark - Enum GPBSyntax -// Syntax specifies the syntax in which a service element was defined. +// The syntax in which a protocol buffer element is defined. typedef GPB_ENUM(GPBSyntax) { GPBSyntax_GPBUnrecognizedEnumeratorValue = kGPBUnrecognizedEnumeratorValue, - // Syntax "proto2" + // Syntax `proto2`. GPBSyntax_SyntaxProto2 = 0, - // Syntax "proto3" + // Syntax `proto3`. GPBSyntax_SyntaxProto3 = 1, }; @@ -34,7 +34,7 @@ BOOL GPBSyntax_IsValidValue(int32_t value); #pragma mark - Enum GPBField_Kind -// Kind represents a basic field type. +// Basic field types. typedef GPB_ENUM(GPBField_Kind) { GPBField_Kind_GPBUnrecognizedEnumeratorValue = kGPBUnrecognizedEnumeratorValue, // Field type unknown. @@ -67,7 +67,7 @@ typedef GPB_ENUM(GPBField_Kind) { // Field type string. GPBField_Kind_TypeString = 9, - // Field type group (deprecated proto2 type) + // Field type group. Proto2 syntax only, and deprecated. GPBField_Kind_TypeGroup = 10, // Field type message. @@ -101,17 +101,16 @@ BOOL GPBField_Kind_IsValidValue(int32_t value); #pragma mark - Enum GPBField_Cardinality -// Cardinality represents whether a field is optional, required, or -// repeated. +// Whether a field is optional, required, or repeated. typedef GPB_ENUM(GPBField_Cardinality) { GPBField_Cardinality_GPBUnrecognizedEnumeratorValue = kGPBUnrecognizedEnumeratorValue, - // The field cardinality is unknown. Typically an error condition. + // For fields with unknown cardinality. GPBField_Cardinality_CardinalityUnknown = 0, // For optional fields. GPBField_Cardinality_CardinalityOptional = 1, - // For required fields. Not used for proto3. + // For required fields. Proto2 syntax only. GPBField_Cardinality_CardinalityRequired = 2, // For repeated fields. @@ -144,7 +143,7 @@ typedef GPB_ENUM(GPBType_FieldNumber) { GPBType_FieldNumber_Syntax = 6, }; -// A light-weight descriptor for a proto message type. +// A protocol buffer message type. @interface GPBType : GPBMessage // The fully qualified message name. @@ -155,12 +154,12 @@ typedef GPB_ENUM(GPBType_FieldNumber) { @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *fieldsArray; @property(nonatomic, readonly) NSUInteger fieldsArray_Count; -// The list of oneof definitions. +// The list of types appearing in `oneof` definitions in this type. // |oneofsArray| contains |NSString| @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *oneofsArray; @property(nonatomic, readonly) NSUInteger oneofsArray_Count; -// The proto options. +// The protocol buffer options. // |optionsArray| contains |GPBOption| @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; @property(nonatomic, readonly) NSUInteger optionsArray_Count; @@ -189,41 +188,46 @@ typedef GPB_ENUM(GPBField_FieldNumber) { GPBField_FieldNumber_Packed = 8, GPBField_FieldNumber_OptionsArray = 9, GPBField_FieldNumber_JsonName = 10, + GPBField_FieldNumber_DefaultValue = 11, }; -// Field represents a single field of a message type. +// A single field of a message type. @interface GPBField : GPBMessage -// The field kind. +// The field type. @property(nonatomic, readwrite) GPBField_Kind kind; -// The field cardinality, i.e. optional/required/repeated. +// The field cardinality. @property(nonatomic, readwrite) GPBField_Cardinality cardinality; -// The proto field number. +// The field number. @property(nonatomic, readwrite) int32_t number; // The field name. @property(nonatomic, readwrite, copy, null_resettable) NSString *name; -// The type URL (without the scheme) when the type is MESSAGE or ENUM, -// such as `type.googleapis.com/google.protobuf.Empty`. +// The field type URL, without the scheme, for message or enumeration +// types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. @property(nonatomic, readwrite, copy, null_resettable) NSString *typeURL; -// Index in Type.oneofs. Starts at 1. Zero means no oneof mapping. +// The index of the field type in `Type.oneofs`, for message or enumeration +// types. The first type has index 1; zero means the type is not in the list. @property(nonatomic, readwrite) int32_t oneofIndex; // Whether to use alternative packed wire representation. @property(nonatomic, readwrite) BOOL packed; -// The proto options. +// The protocol buffer options. // |optionsArray| contains |GPBOption| @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; @property(nonatomic, readonly) NSUInteger optionsArray_Count; -// The JSON name for this field. +// The field JSON name. @property(nonatomic, readwrite, copy, null_resettable) NSString *jsonName; +// The string value of the default value of this field. Proto2 syntax only. +@property(nonatomic, readwrite, copy, null_resettable) NSString *defaultValue; + @end int32_t GPBField_Kind_RawValue(GPBField *message); @@ -253,7 +257,7 @@ typedef GPB_ENUM(GPBEnum_FieldNumber) { @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *enumvalueArray; @property(nonatomic, readonly) NSUInteger enumvalueArray_Count; -// Proto options for the enum type. +// Protocol buffer options. // |optionsArray| contains |GPBOption| @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; @property(nonatomic, readonly) NSUInteger optionsArray_Count; @@ -287,7 +291,7 @@ typedef GPB_ENUM(GPBEnumValue_FieldNumber) { // Enum value number. @property(nonatomic, readwrite) int32_t number; -// Proto options for the enum value. +// Protocol buffer options. // |optionsArray| contains |GPBOption| @property(nonatomic, readwrite, strong, null_resettable) NSMutableArray *optionsArray; @property(nonatomic, readonly) NSUInteger optionsArray_Count; @@ -301,13 +305,14 @@ typedef GPB_ENUM(GPBOption_FieldNumber) { GPBOption_FieldNumber_Value = 2, }; -// Proto option attached to messages/fields/enums etc. +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. @interface GPBOption : GPBMessage -// Proto option name. +// The option's name. For example, `"java_package"`. @property(nonatomic, readwrite, copy, null_resettable) NSString *name; -// Proto option value. +// The option's value. For example, `"com.google.protobuf"`. @property(nonatomic, readwrite) BOOL hasValue; @property(nonatomic, readwrite, strong, null_resettable) GPBAny *value; diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index 2bf36532..5f613c88 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -786,25 +786,33 @@ class FileDescriptor(DescriptorBase): message_types_by_name: Dict of message names of their descriptors. enum_types_by_name: Dict of enum names and their descriptors. extensions_by_name: Dict of extension names and their descriptors. + pool: the DescriptorPool this descriptor belongs to. When not passed to the + constructor, the global default pool is used. """ if _USE_C_DESCRIPTORS: _C_DESCRIPTOR_CLASS = _message.FileDescriptor def __new__(cls, name, package, options=None, serialized_pb=None, - dependencies=None, syntax=None): + dependencies=None, syntax=None, pool=None): # FileDescriptor() is called from various places, not only from generated # files, to register dynamic proto files and messages. if serialized_pb: + # TODO(amauryfa): use the pool passed as argument. This will work only + # for C++-implemented DescriptorPools. return _message.default_pool.AddSerializedFile(serialized_pb) else: return super(FileDescriptor, cls).__new__(cls) def __init__(self, name, package, options=None, serialized_pb=None, - dependencies=None, syntax=None): + dependencies=None, syntax=None, pool=None): """Constructor.""" super(FileDescriptor, self).__init__(options, 'FileOptions') + if pool is None: + from google.protobuf import descriptor_pool + pool = descriptor_pool.Default() + self.pool = pool self.message_types_by_name = {} self.name = name self.package = package diff --git a/python/google/protobuf/descriptor_database.py b/python/google/protobuf/descriptor_database.py index b10021e9..1333f996 100644 --- a/python/google/protobuf/descriptor_database.py +++ b/python/google/protobuf/descriptor_database.py @@ -65,6 +65,7 @@ class DescriptorDatabase(object): raise DescriptorDatabaseConflictingDefinitionError( '%s already added, but with different descriptor.' % proto_name) + # Add the top-level Message, Enum and Extension descriptors to the index. package = file_desc_proto.package for message in file_desc_proto.message_type: self._file_desc_protos_by_symbol.update( @@ -72,6 +73,9 @@ class DescriptorDatabase(object): for enum in file_desc_proto.enum_type: self._file_desc_protos_by_symbol[ '.'.join((package, enum.name))] = file_desc_proto + for extension in file_desc_proto.extension: + self._file_desc_protos_by_symbol[ + '.'.join((package, extension.name))] = file_desc_proto def FindFileByName(self, name): """Finds the file descriptor proto by file name. diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index 6a1b4b5e..3e80795c 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -83,6 +83,12 @@ def _NormalizeFullyQualifiedName(name): class DescriptorPool(object): """A collection of protobufs dynamically constructed by descriptor protos.""" + if _USE_C_DESCRIPTORS: + + def __new__(cls, descriptor_db=None): + # pylint: disable=protected-access + return descriptor._message.DescriptorPool(descriptor_db) + def __init__(self, descriptor_db=None): """Initializes a Pool of proto buffs. @@ -264,6 +270,39 @@ class DescriptorPool(object): self.FindFileContainingSymbol(full_name) return self._enum_descriptors[full_name] + def FindFieldByName(self, full_name): + """Loads the named field descriptor from the pool. + + Args: + full_name: The full name of the field descriptor to load. + + Returns: + The field descriptor for the named field. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + message_name, _, field_name = full_name.rpartition('.') + message_descriptor = self.FindMessageTypeByName(message_name) + return message_descriptor.fields_by_name[field_name] + + def FindExtensionByName(self, full_name): + """Loads the named extension descriptor from the pool. + + Args: + full_name: The full name of the extension descriptor to load. + + Returns: + A FieldDescriptor, describing the named extension. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + message_name, _, extension_name = full_name.rpartition('.') + try: + # Most extensions are nested inside a message. + scope = self.FindMessageTypeByName(message_name) + except KeyError: + # Some extensions are defined at file scope. + scope = self.FindFileContainingSymbol(full_name) + return scope.extensions_by_name[extension_name] + def _ConvertFileProtoToFileDescriptor(self, file_proto): """Creates a FileDescriptor from a proto or returns a cached copy. @@ -282,6 +321,7 @@ class DescriptorPool(object): direct_deps = [self.FindFileByName(n) for n in file_proto.dependency] file_descriptor = descriptor.FileDescriptor( + pool=self, name=file_proto.name, package=file_proto.package, syntax=file_proto.syntax, @@ -598,10 +638,24 @@ class DescriptorPool(object): field_desc.default_value = text_encoding.CUnescape( field_proto.default_value) else: + # All other types are of the "int" type. field_desc.default_value = int(field_proto.default_value) else: field_desc.has_default_value = False - field_desc.default_value = None + if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or + field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT): + field_desc.default_value = 0.0 + elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING: + field_desc.default_value = u'' + elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL: + field_desc.default_value = False + elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM: + field_desc.default_value = field_desc.enum_type.values[0].number + elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES: + field_desc.default_value = b'' + else: + # All other types are of the "int" type. + field_desc.default_value = 0 field_desc.type = field_proto.type @@ -680,3 +734,16 @@ class DescriptorPool(object): def _PrefixWithDot(name): return name if name.startswith('.') else '.%s' % name + + +if _USE_C_DESCRIPTORS: + # TODO(amauryfa): This pool could be constructed from Python code, when we + # support a flag like 'use_cpp_generated_pool=True'. + # pylint: disable=protected-access + _DEFAULT = descriptor._message.default_pool +else: + _DEFAULT = DescriptorPool() + + +def Default(): + return _DEFAULT diff --git a/python/google/protobuf/internal/any_test.proto b/python/google/protobuf/internal/any_test.proto new file mode 100644 index 00000000..cd641ca0 --- /dev/null +++ b/python/google/protobuf/internal/any_test.proto @@ -0,0 +1,42 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: jieluo@google.com (Jie Luo) + +syntax = "proto3"; + +package google.protobuf.internal; + +import "google/protobuf/any.proto"; + +message TestAny { + google.protobuf.Any value = 1; + int32 int_value = 2; +} diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py index 9c8275eb..97cdd848 100755 --- a/python/google/protobuf/internal/containers.py +++ b/python/google/protobuf/internal/containers.py @@ -464,6 +464,9 @@ class ScalarMap(MutableMapping): return val def __contains__(self, item): + # We check the key's type to match the strong-typing flavor of the API. + # Also this makes it easier to match the behavior of the C++ implementation. + self._key_checker.CheckValue(item) return item in self._values # We need to override this explicitly, because our defaultdict-like behavior @@ -491,10 +494,20 @@ class ScalarMap(MutableMapping): def __iter__(self): return iter(self._values) + def __repr__(self): + return repr(self._values) + def MergeFrom(self, other): self._values.update(other._values) self._message_listener.Modified() + def InvalidateIterators(self): + # It appears that the only way to reliably invalidate iterators to + # self._values is to ensure that its size changes. + original = self._values + self._values = original.copy() + original[None] = None + # This is defined in the abstract base, but we can do it much more cheaply. def clear(self): self._values.clear() @@ -576,12 +589,22 @@ class MessageMap(MutableMapping): def __iter__(self): return iter(self._values) + def __repr__(self): + return repr(self._values) + def MergeFrom(self, other): for key in other: self[key].MergeFrom(other[key]) # self._message_listener.Modified() not required here, because # mutations to submessages already propagate. + def InvalidateIterators(self): + # It appears that the only way to reliably invalidate iterators to + # self._values is to ensure that its size changes. + original = self._values + self._values = original.copy() + original[None] = None + # This is defined in the abstract base, but we can do it much more cheaply. def clear(self): self._values.clear() diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index da9a78db..f1d6bf99 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -40,6 +40,8 @@ try: import unittest2 as unittest except ImportError: import unittest +from google.protobuf import unittest_import_pb2 +from google.protobuf import unittest_import_public_pb2 from google.protobuf import unittest_pb2 from google.protobuf import descriptor_pb2 from google.protobuf.internal import api_implementation @@ -51,13 +53,17 @@ from google.protobuf.internal import test_util from google.protobuf import descriptor from google.protobuf import descriptor_database from google.protobuf import descriptor_pool +from google.protobuf import message_factory from google.protobuf import symbol_database class DescriptorPoolTest(unittest.TestCase): + def CreatePool(self): + return descriptor_pool.DescriptorPool() + def setUp(self): - self.pool = descriptor_pool.DescriptorPool() + self.pool = self.CreatePool() self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( factory_test1_pb2.DESCRIPTOR.serialized_pb) self.factory_test2_fd = descriptor_pb2.FileDescriptorProto.FromString( @@ -89,7 +95,7 @@ class DescriptorPoolTest(unittest.TestCase): 'google.protobuf.python.internal.Factory1Message') self.assertIsInstance(file_desc1, descriptor.FileDescriptor) self.assertEqual('google/protobuf/internal/factory_test1.proto', - file_desc1.name) + file_desc1.name) self.assertEqual('google.protobuf.python.internal', file_desc1.package) self.assertIn('Factory1Message', file_desc1.message_types_by_name) @@ -97,7 +103,7 @@ class DescriptorPoolTest(unittest.TestCase): 'google.protobuf.python.internal.Factory2Message') self.assertIsInstance(file_desc2, descriptor.FileDescriptor) self.assertEqual('google/protobuf/internal/factory_test2.proto', - file_desc2.name) + file_desc2.name) self.assertEqual('google.protobuf.python.internal', file_desc2.package) self.assertIn('Factory2Message', file_desc2.message_types_by_name) @@ -111,7 +117,7 @@ class DescriptorPoolTest(unittest.TestCase): self.assertIsInstance(msg1, descriptor.Descriptor) self.assertEqual('Factory1Message', msg1.name) self.assertEqual('google.protobuf.python.internal.Factory1Message', - msg1.full_name) + msg1.full_name) self.assertEqual(None, msg1.containing_type) nested_msg1 = msg1.nested_types[0] @@ -132,7 +138,7 @@ class DescriptorPoolTest(unittest.TestCase): self.assertIsInstance(msg2, descriptor.Descriptor) self.assertEqual('Factory2Message', msg2.name) self.assertEqual('google.protobuf.python.internal.Factory2Message', - msg2.full_name) + msg2.full_name) self.assertIsNone(msg2.containing_type) nested_msg2 = msg2.nested_types[0] @@ -223,6 +229,37 @@ class DescriptorPoolTest(unittest.TestCase): with self.assertRaises(KeyError): self.pool.FindEnumTypeByName('Does not exist') + def testFindFieldByName(self): + field = self.pool.FindFieldByName( + 'google.protobuf.python.internal.Factory1Message.list_value') + self.assertEqual(field.name, 'list_value') + self.assertEqual(field.label, field.LABEL_REPEATED) + with self.assertRaises(KeyError): + self.pool.FindFieldByName('Does not exist') + + def testFindExtensionByName(self): + # An extension defined in a message. + extension = self.pool.FindExtensionByName( + 'google.protobuf.python.internal.Factory2Message.one_more_field') + self.assertEqual(extension.name, 'one_more_field') + # An extension defined at file scope. + extension = self.pool.FindExtensionByName( + 'google.protobuf.python.internal.another_field') + self.assertEqual(extension.name, 'another_field') + self.assertEqual(extension.number, 1002) + with self.assertRaises(KeyError): + self.pool.FindFieldByName('Does not exist') + + def testExtensionsAreNotFields(self): + with self.assertRaises(KeyError): + self.pool.FindFieldByName('google.protobuf.python.internal.another_field') + with self.assertRaises(KeyError): + self.pool.FindFieldByName( + 'google.protobuf.python.internal.Factory2Message.one_more_field') + with self.assertRaises(KeyError): + self.pool.FindExtensionByName( + 'google.protobuf.python.internal.Factory1Message.list_value') + def testUserDefinedDB(self): db = descriptor_database.DescriptorDatabase() self.pool = descriptor_pool.DescriptorPool(db) @@ -231,8 +268,7 @@ class DescriptorPoolTest(unittest.TestCase): self.testFindMessageTypeByName() def testAddSerializedFile(self): - db = descriptor_database.DescriptorDatabase() - self.pool = descriptor_pool.DescriptorPool(db) + self.pool = descriptor_pool.DescriptorPool() self.pool.AddSerializedFile(self.factory_test1_fd.SerializeToString()) self.pool.AddSerializedFile(self.factory_test2_fd.SerializeToString()) self.testFindMessageTypeByName() @@ -274,6 +310,56 @@ class DescriptorPoolTest(unittest.TestCase): 'google/protobuf/internal/descriptor_pool_test1.proto') _CheckDefaultValue(file_descriptor) + def testDefaultValueForCustomMessages(self): + """Check the value returned by non-existent fields.""" + def _CheckValueAndType(value, expected_value, expected_type): + self.assertEqual(value, expected_value) + self.assertIsInstance(value, expected_type) + + def _CheckDefaultValues(msg): + try: + int64 = long + except NameError: # Python3 + int64 = int + try: + unicode_type = unicode + except NameError: # Python3 + unicode_type = str + _CheckValueAndType(msg.optional_int32, 0, int) + _CheckValueAndType(msg.optional_uint64, 0, (int64, int)) + _CheckValueAndType(msg.optional_float, 0, (float, int)) + _CheckValueAndType(msg.optional_double, 0, (float, int)) + _CheckValueAndType(msg.optional_bool, False, bool) + _CheckValueAndType(msg.optional_string, u'', unicode_type) + _CheckValueAndType(msg.optional_bytes, b'', bytes) + _CheckValueAndType(msg.optional_nested_enum, msg.FOO, int) + # First for the generated message + _CheckDefaultValues(unittest_pb2.TestAllTypes()) + # Then for a message built with from the DescriptorPool. + pool = descriptor_pool.DescriptorPool() + pool.Add(descriptor_pb2.FileDescriptorProto.FromString( + unittest_import_public_pb2.DESCRIPTOR.serialized_pb)) + pool.Add(descriptor_pb2.FileDescriptorProto.FromString( + unittest_import_pb2.DESCRIPTOR.serialized_pb)) + pool.Add(descriptor_pb2.FileDescriptorProto.FromString( + unittest_pb2.DESCRIPTOR.serialized_pb)) + message_class = message_factory.MessageFactory(pool).GetPrototype( + pool.FindMessageTypeByName( + unittest_pb2.TestAllTypes.DESCRIPTOR.full_name)) + _CheckDefaultValues(message_class()) + + +@unittest.skipIf(api_implementation.Type() != 'cpp', + 'explicit tests of the C++ implementation') +class CppDescriptorPoolTest(DescriptorPoolTest): + # TODO(amauryfa): remove when descriptor_pool.DescriptorPool() creates true + # C++ descriptor pool object for C++ implementation. + + def CreatePool(self): + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + return _message.DescriptorPool() + class ProtoFile(object): @@ -468,6 +554,8 @@ class AddDescriptorTest(unittest.TestCase): pool.FindFileContainingSymbol( prefix + 'protobuf_unittest.TestAllTypes.NestedMessage').name) + @unittest.skipIf(api_implementation.Type() == 'cpp', + 'With the cpp implementation, Add() must be called first') def testMessage(self): self._TestMessage('') self._TestMessage('.') @@ -502,10 +590,14 @@ class AddDescriptorTest(unittest.TestCase): pool.FindFileContainingSymbol( prefix + 'protobuf_unittest.TestAllTypes.NestedEnum').name) + @unittest.skipIf(api_implementation.Type() == 'cpp', + 'With the cpp implementation, Add() must be called first') def testEnum(self): self._TestEnum('') self._TestEnum('.') + @unittest.skipIf(api_implementation.Type() == 'cpp', + 'With the cpp implementation, Add() must be called first') def testFile(self): pool = descriptor_pool.DescriptorPool() pool.AddFileDescriptor(unittest_pb2.DESCRIPTOR) @@ -520,6 +612,76 @@ class AddDescriptorTest(unittest.TestCase): pool.FindFileContainingSymbol( 'protobuf_unittest.TestAllTypes') + def _GetDescriptorPoolClass(self): + # Test with both implementations of descriptor pools. + if api_implementation.Type() == 'cpp': + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + return _message.DescriptorPool + else: + return descriptor_pool.DescriptorPool + + def testEmptyDescriptorPool(self): + # Check that an empty DescriptorPool() contains no message. + pool = self._GetDescriptorPoolClass()() + proto_file_name = descriptor_pb2.DESCRIPTOR.name + self.assertRaises(KeyError, pool.FindFileByName, proto_file_name) + # Add the above file to the pool + file_descriptor = descriptor_pb2.FileDescriptorProto() + descriptor_pb2.DESCRIPTOR.CopyToProto(file_descriptor) + pool.Add(file_descriptor) + # Now it exists. + self.assertTrue(pool.FindFileByName(proto_file_name)) + + def testCustomDescriptorPool(self): + # Create a new pool, and add a file descriptor. + pool = self._GetDescriptorPoolClass()() + file_desc = descriptor_pb2.FileDescriptorProto( + name='some/file.proto', package='package') + file_desc.message_type.add(name='Message') + pool.Add(file_desc) + self.assertEqual(pool.FindFileByName('some/file.proto').name, + 'some/file.proto') + self.assertEqual(pool.FindMessageTypeByName('package.Message').name, + 'Message') + + +@unittest.skipIf( + api_implementation.Type() != 'cpp', + 'default_pool is only supported by the C++ implementation') +class DefaultPoolTest(unittest.TestCase): + + def testFindMethods(self): + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + pool = _message.default_pool + self.assertIs( + pool.FindFileByName('google/protobuf/unittest.proto'), + unittest_pb2.DESCRIPTOR) + self.assertIs( + pool.FindMessageTypeByName('protobuf_unittest.TestAllTypes'), + unittest_pb2.TestAllTypes.DESCRIPTOR) + self.assertIs( + pool.FindFieldByName('protobuf_unittest.TestAllTypes.optional_int32'), + unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name['optional_int32']) + self.assertIs( + pool.FindExtensionByName('protobuf_unittest.optional_int32_extension'), + unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension']) + self.assertIs( + pool.FindEnumTypeByName('protobuf_unittest.ForeignEnum'), + unittest_pb2.ForeignEnum.DESCRIPTOR) + self.assertIs( + pool.FindOneofByName('protobuf_unittest.TestAllTypes.oneof_field'), + unittest_pb2.TestAllTypes.DESCRIPTOR.oneofs_by_name['oneof_field']) + + def testAddFileDescriptor(self): + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + pool = _message.default_pool + file_desc = descriptor_pb2.FileDescriptorProto(name='some/file.proto') + pool.Add(file_desc) + pool.AddSerializedFile(file_desc.SerializeToString()) + TEST1_FILE = ProtoFile( 'google/protobuf/internal/descriptor_pool_test1.proto', diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 99afee63..fee09a56 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -47,6 +47,7 @@ from google.protobuf import descriptor_pb2 from google.protobuf.internal import api_implementation from google.protobuf.internal import test_util from google.protobuf import descriptor +from google.protobuf import descriptor_pool from google.protobuf import symbol_database from google.protobuf import text_format @@ -75,9 +76,9 @@ class DescriptorTest(unittest.TestCase): enum_proto.value.add(name='FOREIGN_BAR', number=5) enum_proto.value.add(name='FOREIGN_BAZ', number=6) - descriptor_pool = symbol_database.Default().pool - descriptor_pool.Add(file_proto) - self.my_file = descriptor_pool.FindFileByName(file_proto.name) + self.pool = self.GetDescriptorPool() + self.pool.Add(file_proto) + self.my_file = self.pool.FindFileByName(file_proto.name) self.my_message = self.my_file.message_types_by_name[message_proto.name] self.my_enum = self.my_message.enum_types_by_name[enum_proto.name] @@ -97,6 +98,9 @@ class DescriptorTest(unittest.TestCase): self.my_method ]) + def GetDescriptorPool(self): + return symbol_database.Default().pool + def testEnumValueName(self): self.assertEqual(self.my_message.EnumValueName('ForeignEnum', 4), 'FOREIGN_FOO') @@ -393,6 +397,9 @@ class DescriptorTest(unittest.TestCase): def testFileDescriptor(self): self.assertEqual(self.my_file.name, 'some/filename/some.proto') self.assertEqual(self.my_file.package, 'protobuf_unittest') + self.assertEqual(self.my_file.pool, self.pool) + # Generated modules also belong to the default pool. + self.assertEqual(unittest_pb2.DESCRIPTOR.pool, descriptor_pool.Default()) @unittest.skipIf( api_implementation.Type() != 'cpp' or api_implementation.Version() != 2, @@ -407,6 +414,13 @@ class DescriptorTest(unittest.TestCase): message_descriptor.fields.append(None) +class NewDescriptorTest(DescriptorTest): + """Redo the same tests as above, but with a separate DescriptorPool.""" + + def GetDescriptorPool(self): + return descriptor_pool.DescriptorPool() + + class GeneratedDescriptorTest(unittest.TestCase): """Tests for the properties of descriptors in generated code.""" diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index 69197865..be3ad11a 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py @@ -42,6 +42,7 @@ try: import unittest2 as unittest except ImportError: import unittest +from google.protobuf.internal import well_known_types from google.protobuf import json_format from google.protobuf.util import json_format_proto3_pb2 @@ -269,15 +270,15 @@ class JsonFormatTest(JsonFormatBase): '}')) parsed_message = json_format_proto3_pb2.TestTimestamp() self.CheckParseBack(message, parsed_message) - text = (r'{"value": "1972-01-01T01:00:00.01+08:00",' + text = (r'{"value": "1970-01-01T00:00:00.01+08:00",' r'"repeatedValue":[' - r' "1972-01-01T01:00:00.01+08:30",' - r' "1972-01-01T01:00:00.01-01:23"]}') + r' "1970-01-01T00:00:00.01+08:30",' + r' "1970-01-01T00:00:00.01-01:23"]}') json_format.Parse(text, parsed_message) - self.assertEqual(parsed_message.value.seconds, 63104400) + self.assertEqual(parsed_message.value.seconds, -8 * 3600) self.assertEqual(parsed_message.value.nanos, 10000000) - self.assertEqual(parsed_message.repeated_value[0].seconds, 63106200) - self.assertEqual(parsed_message.repeated_value[1].seconds, 63070620) + self.assertEqual(parsed_message.repeated_value[0].seconds, -8.5 * 3600) + self.assertEqual(parsed_message.repeated_value[1].seconds, 3600 + 23 * 60) def testDurationMessage(self): message = json_format_proto3_pb2.TestDuration() @@ -389,7 +390,7 @@ class JsonFormatTest(JsonFormatBase): def testParseEmptyText(self): self.CheckError('', - r'Failed to load JSON: (Expecting value)|(No JSON)') + r'Failed to load JSON: (Expecting value)|(No JSON).') def testParseBadEnumValue(self): self.CheckError( @@ -414,7 +415,7 @@ class JsonFormatTest(JsonFormatBase): if sys.version_info < (2, 7): return self.CheckError('{"int32Value": 1,\n"int32Value":2}', - 'Failed to load JSON: duplicate key int32Value') + 'Failed to load JSON: duplicate key int32Value.') def testInvalidBoolValue(self): self.CheckError('{"boolValue": 1}', @@ -431,39 +432,43 @@ class JsonFormatTest(JsonFormatBase): json_format.Parse, text, message) self.CheckError('{"int32Value": 012345}', (r'Failed to load JSON: Expecting \'?,\'? delimiter: ' - r'line 1')) + r'line 1.')) self.CheckError('{"int32Value": 1.0}', 'Failed to parse int32Value field: ' - 'Couldn\'t parse integer: 1.0') + 'Couldn\'t parse integer: 1.0.') self.CheckError('{"int32Value": " 1 "}', 'Failed to parse int32Value field: ' - 'Couldn\'t parse integer: " 1 "') + 'Couldn\'t parse integer: " 1 ".') + self.CheckError('{"int32Value": "1 "}', + 'Failed to parse int32Value field: ' + 'Couldn\'t parse integer: "1 ".') self.CheckError('{"int32Value": 12345678901234567890}', 'Failed to parse int32Value field: Value out of range: ' - '12345678901234567890') + '12345678901234567890.') self.CheckError('{"int32Value": 1e5}', 'Failed to parse int32Value field: ' - 'Couldn\'t parse integer: 100000.0') + 'Couldn\'t parse integer: 100000.0.') self.CheckError('{"uint32Value": -1}', - 'Failed to parse uint32Value field: Value out of range: -1') + 'Failed to parse uint32Value field: ' + 'Value out of range: -1.') def testInvalidFloatValue(self): self.CheckError('{"floatValue": "nan"}', 'Failed to parse floatValue field: Couldn\'t ' - 'parse float "nan", use "NaN" instead') + 'parse float "nan", use "NaN" instead.') def testInvalidBytesValue(self): self.CheckError('{"bytesValue": "AQI"}', - 'Failed to parse bytesValue field: Incorrect padding') + 'Failed to parse bytesValue field: Incorrect padding.') self.CheckError('{"bytesValue": "AQI*"}', - 'Failed to parse bytesValue field: Incorrect padding') + 'Failed to parse bytesValue field: Incorrect padding.') def testInvalidMap(self): message = json_format_proto3_pb2.TestMap() text = '{"int32Map": {"null": 2, "2": 3}}' self.assertRaisesRegexp( json_format.ParseError, - 'Failed to parse int32Map field: Couldn\'t parse integer: "null"', + 'Failed to parse int32Map field: invalid literal', json_format.Parse, text, message) text = '{"int32Map": {1: 2, "2": 3}}' self.assertRaisesRegexp( @@ -474,7 +479,7 @@ class JsonFormatTest(JsonFormatBase): text = '{"boolMap": {"null": 1}}' self.assertRaisesRegexp( json_format.ParseError, - 'Failed to parse boolMap field: Expect "true" or "false", not null.', + 'Failed to parse boolMap field: Expected "true" or "false", not null.', json_format.Parse, text, message) if sys.version_info < (2, 7): return @@ -490,30 +495,29 @@ class JsonFormatTest(JsonFormatBase): self.assertRaisesRegexp( json_format.ParseError, 'time data \'10000-01-01T00:00:00\' does not match' - ' format \'%Y-%m-%dT%H:%M:%S\'', + ' format \'%Y-%m-%dT%H:%M:%S\'.', json_format.Parse, text, message) text = '{"value": "1970-01-01T00:00:00.0123456789012Z"}' self.assertRaisesRegexp( - json_format.ParseError, - 'Failed to parse value field: Failed to parse Timestamp: ' + well_known_types.ParseError, 'nanos 0123456789012 more than 9 fractional digits.', json_format.Parse, text, message) text = '{"value": "1972-01-01T01:00:00.01+08"}' self.assertRaisesRegexp( - json_format.ParseError, - (r'Failed to parse value field: Invalid timezone offset value: \+08'), + well_known_types.ParseError, + (r'Invalid timezone offset value: \+08.'), json_format.Parse, text, message) # Time smaller than minimum time. text = '{"value": "0000-01-01T00:00:00Z"}' self.assertRaisesRegexp( json_format.ParseError, - 'Failed to parse value field: year is out of range', + 'Failed to parse value field: year is out of range.', json_format.Parse, text, message) # Time bigger than maxinum time. message.value.seconds = 253402300800 self.assertRaisesRegexp( - json_format.SerializeToJsonError, - 'Failed to serialize value field: year is out of range', + OverflowError, + 'date value out of range', json_format.MessageToJson, message) def testInvalidOneof(self): diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index d760b898..2fbe5ea7 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -45,6 +45,7 @@ from google.protobuf import descriptor_database from google.protobuf import descriptor_pool from google.protobuf import message_factory + class MessageFactoryTest(unittest.TestCase): def setUp(self): @@ -104,8 +105,8 @@ class MessageFactoryTest(unittest.TestCase): def testGetMessages(self): # performed twice because multiple calls with the same input must be allowed for _ in range(2): - messages = message_factory.GetMessages([self.factory_test2_fd, - self.factory_test1_fd]) + messages = message_factory.GetMessages([self.factory_test1_fd, + self.factory_test2_fd]) self.assertTrue( set(['google.protobuf.python.internal.Factory2Message', 'google.protobuf.python.internal.Factory1Message'], @@ -116,7 +117,7 @@ class MessageFactoryTest(unittest.TestCase): set(['google.protobuf.python.internal.Factory2Message.one_more_field', 'google.protobuf.python.internal.another_field'], ).issubset( - set(messages['google.protobuf.python.internal.Factory1Message'] + set(messages['google.protobuf.python.internal.Factory1Message'] ._extensions_by_name.keys()))) factory_msg1 = messages['google.protobuf.python.internal.Factory1Message'] msg1 = messages['google.protobuf.python.internal.Factory1Message']() diff --git a/python/google/protobuf/internal/message_set_extensions.proto b/python/google/protobuf/internal/message_set_extensions.proto index 702c8d07..14e5f193 100644 --- a/python/google/protobuf/internal/message_set_extensions.proto +++ b/python/google/protobuf/internal/message_set_extensions.proto @@ -54,6 +54,14 @@ message TestMessageSetExtension2 { optional string str = 25; } +message TestMessageSetExtension3 { + optional string text = 35; +} + +extend TestMessageSet { + optional TestMessageSetExtension3 message_set_extension3 = 98418655; +} + // This message was used to generate // //net/proto2/python/internal/testdata/message_set_message, but is commented // out since it must not actually exist in code, to simulate an "unknown" diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 13c3caa6..d03f2d25 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -60,6 +60,7 @@ from google.protobuf.internal import _parameterized from google.protobuf import map_unittest_pb2 from google.protobuf import unittest_pb2 from google.protobuf import unittest_proto3_arena_pb2 +from google.protobuf.internal import any_test_pb2 from google.protobuf.internal import api_implementation from google.protobuf.internal import packed_field_test_pb2 from google.protobuf.internal import test_util @@ -1279,12 +1280,13 @@ class Proto3Test(unittest.TestCase): self.assertIsInstance(msg.map_string_string['abc'], six.text_type) - # Accessing an unset key still throws TypeError of the type of the key + # Accessing an unset key still throws TypeError if the type of the key # is incorrect. with self.assertRaises(TypeError): msg.map_string_string[123] - self.assertFalse(123 in msg.map_string_string) + with self.assertRaises(TypeError): + 123 in msg.map_string_string def testMapGet(self): # Need to test that get() properly returns the default, even though the dict @@ -1591,31 +1593,49 @@ class Proto3Test(unittest.TestCase): # For the C++ implementation this tests the correctness of # ScalarMapContainer::Release() msg = map_unittest_pb2.TestMap() - map = msg.map_int32_int32 + int32_map = msg.map_int32_int32 - map[2] = 4 - map[3] = 6 - map[4] = 8 + int32_map[2] = 4 + int32_map[3] = 6 + int32_map[4] = 8 msg.ClearField('map_int32_int32') + self.assertEqual(b'', msg.SerializeToString()) matching_dict = {2: 4, 3: 6, 4: 8} - self.assertMapIterEquals(map.items(), matching_dict) + self.assertMapIterEquals(int32_map.items(), matching_dict) - def testMapIterValidAfterFieldCleared(self): - # Map iterator needs to work even if field is cleared. + def testMessageMapValidAfterFieldCleared(self): + # Map needs to work even if field is cleared. # For the C++ implementation this tests the correctness of # ScalarMapContainer::Release() msg = map_unittest_pb2.TestMap() + int32_foreign_message = msg.map_int32_foreign_message - msg.map_int32_int32[2] = 4 - msg.map_int32_int32[3] = 6 - msg.map_int32_int32[4] = 8 + int32_foreign_message[2].c = 5 - it = msg.map_int32_int32.items() + msg.ClearField('map_int32_foreign_message') + self.assertEqual(b'', msg.SerializeToString()) + self.assertTrue(2 in int32_foreign_message.keys()) + + def testMapIterInvalidatedByClearField(self): + # Map iterator is invalidated when field is cleared. + # But this case does need to not crash the interpreter. + # For the C++ implementation this tests the correctness of + # ScalarMapContainer::Release() + msg = map_unittest_pb2.TestMap() + + it = iter(msg.map_int32_int32) msg.ClearField('map_int32_int32') - matching_dict = {2: 4, 3: 6, 4: 8} - self.assertMapIterEquals(it, matching_dict) + with self.assertRaises(RuntimeError): + for _ in it: + pass + + it = iter(msg.map_int32_foreign_message) + msg.ClearField('map_int32_foreign_message') + with self.assertRaises(RuntimeError): + for _ in it: + pass def testMapDelete(self): msg = map_unittest_pb2.TestMap() @@ -1646,6 +1666,37 @@ class Proto3Test(unittest.TestCase): msg.map_string_foreign_message['foo'].c = 5 self.assertEqual(0, len(msg.FindInitializationErrors())) + def testAnyMessage(self): + # Creates and sets message. + msg = any_test_pb2.TestAny() + msg_descriptor = msg.DESCRIPTOR + all_types = unittest_pb2.TestAllTypes() + all_descriptor = all_types.DESCRIPTOR + all_types.repeated_string.append(u'\u00fc\ua71f') + # Packs to Any. + msg.value.Pack(all_types) + self.assertEqual(msg.value.type_url, + 'type.googleapis.com/%s' % all_descriptor.full_name) + self.assertEqual(msg.value.value, + all_types.SerializeToString()) + # Tests Is() method. + self.assertTrue(msg.value.Is(all_descriptor)) + self.assertFalse(msg.value.Is(msg_descriptor)) + # Unpacks Any. + unpacked_message = unittest_pb2.TestAllTypes() + self.assertTrue(msg.value.Unpack(unpacked_message)) + self.assertEqual(all_types, unpacked_message) + # Unpacks to different type. + self.assertFalse(msg.value.Unpack(msg)) + # Only Any messages have Pack method. + try: + msg.Pack(all_types) + except AttributeError: + pass + else: + raise AttributeError('%s should not have Pack method.' % + msg_descriptor.full_name) + class ValidTypeNamesTest(unittest.TestCase): diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 2b87f704..87f60666 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -65,6 +65,7 @@ from google.protobuf.internal import encoder from google.protobuf.internal import enum_type_wrapper from google.protobuf.internal import message_listener as message_listener_mod from google.protobuf.internal import type_checkers +from google.protobuf.internal import well_known_types from google.protobuf.internal import wire_format from google.protobuf import descriptor as descriptor_mod from google.protobuf import message as message_mod @@ -72,6 +73,7 @@ from google.protobuf import symbol_database from google.protobuf import text_format _FieldDescriptor = descriptor_mod.FieldDescriptor +_AnyFullTypeName = 'google.protobuf.Any' class GeneratedProtocolMessageType(type): @@ -127,6 +129,8 @@ class GeneratedProtocolMessageType(type): Newly-allocated class. """ descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY] + if descriptor.full_name in well_known_types.WKTBASES: + bases += (well_known_types.WKTBASES[descriptor.full_name],) _AddClassAttributesForNestedExtensions(descriptor, dictionary) _AddSlots(descriptor, dictionary) @@ -261,7 +265,6 @@ def _IsMessageSetExtension(field): field.containing_type.has_options and field.containing_type.GetOptions().message_set_wire_format and field.type == _FieldDescriptor.TYPE_MESSAGE and - field.message_type == field.extension_scope and field.label == _FieldDescriptor.LABEL_OPTIONAL) @@ -543,7 +546,8 @@ def _GetFieldByName(message_descriptor, field_name): try: return message_descriptor.fields_by_name[field_name] except KeyError: - raise ValueError('Protocol message has no "%s" field.' % field_name) + raise ValueError('Protocol message %s has no "%s" field.' % + (message_descriptor.name, field_name)) def _AddPropertiesForFields(descriptor, cls): @@ -848,9 +852,15 @@ def _AddClearFieldMethod(message_descriptor, cls): else: return except KeyError: - raise ValueError('Protocol message has no "%s" field.' % field_name) + raise ValueError('Protocol message %s() has no "%s" field.' % + (message_descriptor.name, field_name)) if field in self._fields: + # To match the C++ implementation, we need to invalidate iterators + # for map fields when ClearField() happens. + if hasattr(self._fields[field], 'InvalidateIterators'): + self._fields[field].InvalidateIterators() + # Note: If the field is a sub-message, its listener will still point # at us. That's fine, because the worst than can happen is that it # will call _Modified() and invalidate our byte size. Big deal. @@ -904,7 +914,19 @@ def _AddHasExtensionMethod(cls): return extension_handle in self._fields cls.HasExtension = HasExtension -def _UnpackAny(msg): +def _InternalUnpackAny(msg): + """Unpacks Any message and returns the unpacked message. + + This internal method is differnt from public Any Unpack method which takes + the target message as argument. _InternalUnpackAny method does not have + target message type and need to find the message type in descriptor pool. + + Args: + msg: An Any message to be unpacked. + + Returns: + The unpacked message. + """ type_url = msg.type_url db = symbol_database.Default() @@ -935,9 +957,9 @@ def _AddEqualsMethod(message_descriptor, cls): if self is other: return True - if self.DESCRIPTOR.full_name == "google.protobuf.Any": - any_a = _UnpackAny(self) - any_b = _UnpackAny(other) + if self.DESCRIPTOR.full_name == _AnyFullTypeName: + any_a = _InternalUnpackAny(self) + any_b = _InternalUnpackAny(other) if any_a and any_b: return any_a == any_b @@ -962,6 +984,13 @@ def _AddStrMethod(message_descriptor, cls): cls.__str__ = __str__ +def _AddReprMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + def __repr__(self): + return text_format.MessageToString(self) + cls.__repr__ = __repr__ + + def _AddUnicodeMethod(unused_message_descriptor, cls): """Helper for _AddMessageMethods().""" @@ -1270,6 +1299,7 @@ def _AddMessageMethods(message_descriptor, cls): _AddClearMethod(message_descriptor, cls) _AddEqualsMethod(message_descriptor, cls) _AddStrMethod(message_descriptor, cls) + _AddReprMethod(message_descriptor, cls) _AddUnicodeMethod(message_descriptor, cls) _AddSetListenerMethod(cls) _AddByteSizeMethod(message_descriptor, cls) @@ -1280,6 +1310,7 @@ def _AddMessageMethods(message_descriptor, cls): _AddMergeFromMethod(cls) _AddWhichOneofMethod(message_descriptor, cls) + def _AddPrivateHelperMethods(message_descriptor, cls): """Adds implementation of private helper methods to cls.""" diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 8621d61e..752f2f5d 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -2394,8 +2394,10 @@ class SerializationTest(unittest.TestCase): extension_message2 = message_set_extensions_pb2.TestMessageSetExtension2 extension1 = extension_message1.message_set_extension extension2 = extension_message2.message_set_extension + extension3 = message_set_extensions_pb2.message_set_extension3 proto.Extensions[extension1].i = 123 proto.Extensions[extension2].str = 'foo' + proto.Extensions[extension3].text = 'bar' # Serialize using the MessageSet wire format (this is specified in the # .proto file). @@ -2407,7 +2409,7 @@ class SerializationTest(unittest.TestCase): self.assertEqual( len(serialized), raw.MergeFromString(serialized)) - self.assertEqual(2, len(raw.item)) + self.assertEqual(3, len(raw.item)) message1 = message_set_extensions_pb2.TestMessageSetExtension1() self.assertEqual( @@ -2421,6 +2423,12 @@ class SerializationTest(unittest.TestCase): message2.MergeFromString(raw.item[1].message)) self.assertEqual('foo', message2.str) + message3 = message_set_extensions_pb2.TestMessageSetExtension3() + self.assertEqual( + len(raw.item[2].message), + message3.MergeFromString(raw.item[2].message)) + self.assertEqual('bar', message3.text) + # Deserialize using the MessageSet wire format. proto2 = message_set_extensions_pb2.TestMessageSet() self.assertEqual( @@ -2428,6 +2436,7 @@ class SerializationTest(unittest.TestCase): proto2.MergeFromString(serialized)) self.assertEqual(123, proto2.Extensions[extension1].i) self.assertEqual('foo', proto2.Extensions[extension2].str) + self.assertEqual('bar', proto2.Extensions[extension3].text) # Check byte size. self.assertEqual(proto2.ByteSize(), len(serialized)) @@ -2757,9 +2766,10 @@ class SerializationTest(unittest.TestCase): def testInitArgsUnknownFieldName(self): def InitalizeEmptyMessageWithExtraKeywordArg(): unused_proto = unittest_pb2.TestEmptyMessage(unknown='unknown') - self._CheckRaises(ValueError, - InitalizeEmptyMessageWithExtraKeywordArg, - 'Protocol message has no "unknown" field.') + self._CheckRaises( + ValueError, + InitalizeEmptyMessageWithExtraKeywordArg, + 'Protocol message TestEmptyMessage has no "unknown" field.') def testInitRequiredKwargs(self): proto = unittest_pb2.TestRequired(a=1, b=1, c=1) diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index cca0ee63..0e14556c 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -51,8 +51,22 @@ from google.protobuf import unittest_pb2 from google.protobuf import unittest_proto3_arena_pb2 from google.protobuf.internal import api_implementation from google.protobuf.internal import test_util +from google.protobuf.internal import message_set_extensions_pb2 from google.protobuf import text_format + +# Low-level nuts-n-bolts tests. +class SimpleTextFormatTests(unittest.TestCase): + + # The members of _QUOTES are formatted into a regexp template that + # expects single characters. Therefore it's an error (in addition to being + # non-sensical in the first place) to try to specify a "quote mark" that is + # more than one character. + def TestQuoteMarksAreSingleChars(self): + for quote in text_format._QUOTES: + self.assertEqual(1, len(quote)) + + # Base class with some common functionality. class TextFormatBase(unittest.TestCase): @@ -287,6 +301,19 @@ class TextFormatTest(TextFormatBase): self.assertEqual(u'one', message.repeated_string[0]) self.assertEqual(u'two', message.repeated_string[1]) + def testParseRepeatedScalarShortFormat(self, message_module): + message = message_module.TestAllTypes() + text = ('repeated_int64: [100, 200];\n' + 'repeated_int64: 300,\n' + 'repeated_string: ["one", "two"];\n') + text_format.Parse(text, message) + + self.assertEqual(100, message.repeated_int64[0]) + self.assertEqual(200, message.repeated_int64[1]) + self.assertEqual(300, message.repeated_int64[2]) + self.assertEqual(u'one', message.repeated_string[0]) + self.assertEqual(u'two', message.repeated_string[1]) + def testParseEmptyText(self, message_module): message = message_module.TestAllTypes() text = '' @@ -301,7 +328,7 @@ class TextFormatTest(TextFormatBase): def testParseSingleWord(self, message_module): message = message_module.TestAllTypes() text = 'foo' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' r'"foo".'), @@ -310,7 +337,7 @@ class TextFormatTest(TextFormatBase): def testParseUnknownField(self, message_module): message = message_module.TestAllTypes() text = 'unknown_field: 8\n' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, (r'1:1 : Message type "\w+.TestAllTypes" has no field named ' r'"unknown_field".'), @@ -319,7 +346,7 @@ class TextFormatTest(TextFormatBase): def testParseBadEnumValue(self, message_module): message = message_module.TestAllTypes() text = 'optional_nested_enum: BARR' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" ' r'has no value named BARR.'), @@ -327,7 +354,7 @@ class TextFormatTest(TextFormatBase): message = message_module.TestAllTypes() text = 'optional_nested_enum: 100' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, (r'1:23 : Enum type "\w+.TestAllTypes.NestedEnum" ' r'has no value with number 100.'), @@ -336,7 +363,7 @@ class TextFormatTest(TextFormatBase): def testParseBadIntValue(self, message_module): message = message_module.TestAllTypes() text = 'optional_int32: bork' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, ('1:17 : Couldn\'t parse integer: bork'), text_format.Parse, text, message) @@ -553,6 +580,15 @@ class Proto2Tests(TextFormatBase): ' }\n' '}\n') + message = message_set_extensions_pb2.TestMessageSet() + ext = message_set_extensions_pb2.message_set_extension3 + message.Extensions[ext].text = 'bar' + self.CompareToGoldenText( + text_format.MessageToString(message), + '[google.protobuf.internal.TestMessageSetExtension3] {\n' + ' text: \"bar\"\n' + '}\n') + def testPrintMessageSetAsOneLine(self): message = unittest_mset_pb2.TestMessageSetContainer() ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension @@ -627,15 +663,125 @@ class Proto2Tests(TextFormatBase): text_format.Parse(ascii_text, parsed_message) self.assertEqual(message, parsed_message) + def testParseAllowedUnknownExtension(self): + # Skip over unknown extension correctly. + message = unittest_mset_pb2.TestMessageSetContainer() + text = ('message_set {\n' + ' [unknown_extension] {\n' + ' i: 23\n' + ' [nested_unknown_ext]: {\n' + ' i: 23\n' + ' test: "test_string"\n' + ' floaty_float: -0.315\n' + ' num: -inf\n' + ' multiline_str: "abc"\n' + ' "def"\n' + ' "xyz."\n' + ' [nested_unknown_ext]: <\n' + ' i: 23\n' + ' i: 24\n' + ' pointfloat: .3\n' + ' test: "test_string"\n' + ' floaty_float: -0.315\n' + ' num: -inf\n' + ' long_string: "test" "test2" \n' + ' >\n' + ' }\n' + ' }\n' + ' [unknown_extension]: 5\n' + '}\n') + text_format.Parse(text, message, allow_unknown_extension=True) + golden = 'message_set {\n}\n' + self.CompareToGoldenText(text_format.MessageToString(message), golden) + + # Catch parse errors in unknown extension. + message = unittest_mset_pb2.TestMessageSetContainer() + malformed = ('message_set {\n' + ' [unknown_extension] {\n' + ' i:\n' # Missing value. + ' }\n' + '}\n') + six.assertRaisesRegex(self, + text_format.ParseError, + 'Invalid field value: }', + text_format.Parse, malformed, message, + allow_unknown_extension=True) + + message = unittest_mset_pb2.TestMessageSetContainer() + malformed = ('message_set {\n' + ' [unknown_extension] {\n' + ' str: "malformed string\n' # Missing closing quote. + ' }\n' + '}\n') + six.assertRaisesRegex(self, + text_format.ParseError, + 'Invalid field value: "', + text_format.Parse, malformed, message, + allow_unknown_extension=True) + + message = unittest_mset_pb2.TestMessageSetContainer() + malformed = ('message_set {\n' + ' [unknown_extension] {\n' + ' str: "malformed\n multiline\n string\n' + ' }\n' + '}\n') + six.assertRaisesRegex(self, + text_format.ParseError, + 'Invalid field value: "', + text_format.Parse, malformed, message, + allow_unknown_extension=True) + + message = unittest_mset_pb2.TestMessageSetContainer() + malformed = ('message_set {\n' + ' [malformed_extension] <\n' + ' i: -5\n' + ' \n' # Missing '>' here. + '}\n') + six.assertRaisesRegex(self, + text_format.ParseError, + '5:1 : Expected ">".', + text_format.Parse, malformed, message, + allow_unknown_extension=True) + + # Don't allow unknown fields with allow_unknown_extension=True. + message = unittest_mset_pb2.TestMessageSetContainer() + malformed = ('message_set {\n' + ' unknown_field: true\n' + ' \n' # Missing '>' here. + '}\n') + six.assertRaisesRegex(self, + text_format.ParseError, + ('2:3 : Message type ' + '"proto2_wireformat_unittest.TestMessageSet" has no' + ' field named "unknown_field".'), + text_format.Parse, malformed, message, + allow_unknown_extension=True) + + # Parse known extension correcty. + message = unittest_mset_pb2.TestMessageSetContainer() + text = ('message_set {\n' + ' [protobuf_unittest.TestMessageSetExtension1] {\n' + ' i: 23\n' + ' }\n' + ' [protobuf_unittest.TestMessageSetExtension2] {\n' + ' str: \"foo\"\n' + ' }\n' + '}\n') + text_format.Parse(text, message, allow_unknown_extension=True) + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + self.assertEqual(23, message.message_set.Extensions[ext1].i) + self.assertEqual('foo', message.message_set.Extensions[ext2].str) + def testParseBadExtension(self): message = unittest_pb2.TestAllExtensions() text = '[unknown_extension]: 8\n' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, '1:2 : Extension "unknown_extension" not registered.', text_format.Parse, text, message) message = unittest_pb2.TestAllTypes() - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' 'extensions.'), @@ -654,7 +800,7 @@ class Proto2Tests(TextFormatBase): message = unittest_pb2.TestAllExtensions() text = ('[protobuf_unittest.optional_int32_extension]: 42 ' '[protobuf_unittest.optional_int32_extension]: 67') - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, ('1:96 : Message type "protobuf_unittest.TestAllExtensions" ' 'should not have multiple ' @@ -665,7 +811,7 @@ class Proto2Tests(TextFormatBase): message = unittest_pb2.TestAllTypes() text = ('optional_nested_message { bb: 1 } ' 'optional_nested_message { bb: 2 }') - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, ('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" ' 'should not have multiple "bb" fields.'), @@ -675,7 +821,7 @@ class Proto2Tests(TextFormatBase): message = unittest_pb2.TestAllTypes() text = ('optional_int32: 42 ' 'optional_int32: 67') - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, ('1:36 : Message type "protobuf_unittest.TestAllTypes" should not ' 'have multiple "optional_int32" fields.'), @@ -684,11 +830,11 @@ class Proto2Tests(TextFormatBase): def testParseGroupNotClosed(self): message = unittest_pb2.TestAllTypes() text = 'RepeatedGroup: <' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected ">".', text_format.Parse, text, message) text = 'RepeatedGroup: {' - six.assertRaisesRegex(self, + six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected "}".', text_format.Parse, text, message) diff --git a/python/google/protobuf/internal/well_known_types.py b/python/google/protobuf/internal/well_known_types.py new file mode 100644 index 00000000..d3de9831 --- /dev/null +++ b/python/google/protobuf/internal/well_known_types.py @@ -0,0 +1,622 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Contains well known classes. + +This files defines well known classes which need extra maintenance including: + - Any + - Duration + - FieldMask + - Timestamp +""" + +__author__ = 'jieluo@google.com (Jie Luo)' + +from datetime import datetime +from datetime import timedelta + +from google.protobuf.descriptor import FieldDescriptor + +_TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' +_NANOS_PER_SECOND = 1000000000 +_NANOS_PER_MILLISECOND = 1000000 +_NANOS_PER_MICROSECOND = 1000 +_MILLIS_PER_SECOND = 1000 +_MICROS_PER_SECOND = 1000000 +_SECONDS_PER_DAY = 24 * 3600 + + +class Error(Exception): + """Top-level module error.""" + + +class ParseError(Error): + """Thrown in case of parsing error.""" + + +class Any(object): + """Class for Any Message type.""" + + def Pack(self, msg): + """Packs the specified message into current Any message.""" + self.type_url = 'type.googleapis.com/%s' % msg.DESCRIPTOR.full_name + self.value = msg.SerializeToString() + + def Unpack(self, msg): + """Unpacks the current Any message into specified message.""" + descriptor = msg.DESCRIPTOR + if not self.Is(descriptor): + return False + msg.ParseFromString(self.value) + return True + + def Is(self, descriptor): + """Checks if this Any represents the given protobuf type.""" + # Only last part is to be used: b/25630112 + return self.type_url.split('/')[-1] == descriptor.full_name + + +class Timestamp(object): + """Class for Timestamp message type.""" + + def ToJsonString(self): + """Converts Timestamp to RFC 3339 date string format. + + Returns: + A string converted from timestamp. The string is always Z-normalized + and uses 3, 6 or 9 fractional digits as required to represent the + exact time. Example of the return format: '1972-01-01T10:00:20.021Z' + """ + nanos = self.nanos % _NANOS_PER_SECOND + total_sec = self.seconds + (self.nanos - nanos) // _NANOS_PER_SECOND + seconds = total_sec % _SECONDS_PER_DAY + days = (total_sec - seconds) // _SECONDS_PER_DAY + dt = datetime(1970, 1, 1) + timedelta(days, seconds) + + result = dt.isoformat() + if (nanos % 1e9) == 0: + # If there are 0 fractional digits, the fractional + # point '.' should be omitted when serializing. + return result + 'Z' + if (nanos % 1e6) == 0: + # Serialize 3 fractional digits. + return result + '.%03dZ' % (nanos / 1e6) + if (nanos % 1e3) == 0: + # Serialize 6 fractional digits. + return result + '.%06dZ' % (nanos / 1e3) + # Serialize 9 fractional digits. + return result + '.%09dZ' % nanos + + def FromJsonString(self, value): + """Parse a RFC 3339 date string format to Timestamp. + + Args: + value: A date string. Any fractional digits (or none) and any offset are + accepted as long as they fit into nano-seconds precision. + Example of accepted format: '1972-01-01T10:00:20.021-05:00' + + Raises: + ParseError: On parsing problems. + """ + timezone_offset = value.find('Z') + if timezone_offset == -1: + timezone_offset = value.find('+') + if timezone_offset == -1: + timezone_offset = value.rfind('-') + if timezone_offset == -1: + raise ParseError( + 'Failed to parse timestamp: missing valid timezone offset.') + time_value = value[0:timezone_offset] + # Parse datetime and nanos. + point_position = time_value.find('.') + if point_position == -1: + second_value = time_value + nano_value = '' + else: + second_value = time_value[:point_position] + nano_value = time_value[point_position + 1:] + date_object = datetime.strptime(second_value, _TIMESTAMPFOMAT) + td = date_object - datetime(1970, 1, 1) + seconds = td.seconds + td.days * _SECONDS_PER_DAY + if len(nano_value) > 9: + raise ParseError( + 'Failed to parse Timestamp: nanos {0} more than ' + '9 fractional digits.'.format(nano_value)) + if nano_value: + nanos = round(float('0.' + nano_value) * 1e9) + else: + nanos = 0 + # Parse timezone offsets. + if value[timezone_offset] == 'Z': + if len(value) != timezone_offset + 1: + raise ParseError('Failed to parse timestamp: invalid trailing' + ' data {0}.'.format(value)) + else: + timezone = value[timezone_offset:] + pos = timezone.find(':') + if pos == -1: + raise ParseError( + 'Invalid timezone offset value: {0}.'.format(timezone)) + if timezone[0] == '+': + seconds -= (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60 + else: + seconds += (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60 + # Set seconds and nanos + self.seconds = int(seconds) + self.nanos = int(nanos) + + def GetCurrentTime(self): + """Get the current UTC into Timestamp.""" + self.FromDatetime(datetime.utcnow()) + + def ToNanoseconds(self): + """Converts Timestamp to nanoseconds since epoch.""" + return self.seconds * _NANOS_PER_SECOND + self.nanos + + def ToMicroseconds(self): + """Converts Timestamp to microseconds since epoch.""" + return (self.seconds * _MICROS_PER_SECOND + + self.nanos // _NANOS_PER_MICROSECOND) + + def ToMilliseconds(self): + """Converts Timestamp to milliseconds since epoch.""" + return (self.seconds * _MILLIS_PER_SECOND + + self.nanos // _NANOS_PER_MILLISECOND) + + def ToSeconds(self): + """Converts Timestamp to seconds since epoch.""" + return self.seconds + + def FromNanoseconds(self, nanos): + """Converts nanoseconds since epoch to Timestamp.""" + self.seconds = nanos // _NANOS_PER_SECOND + self.nanos = nanos % _NANOS_PER_SECOND + + def FromMicroseconds(self, micros): + """Converts microseconds since epoch to Timestamp.""" + self.seconds = micros // _MICROS_PER_SECOND + self.nanos = (micros % _MICROS_PER_SECOND) * _NANOS_PER_MICROSECOND + + def FromMilliseconds(self, millis): + """Converts milliseconds since epoch to Timestamp.""" + self.seconds = millis // _MILLIS_PER_SECOND + self.nanos = (millis % _MILLIS_PER_SECOND) * _NANOS_PER_MILLISECOND + + def FromSeconds(self, seconds): + """Converts seconds since epoch to Timestamp.""" + self.seconds = seconds + self.nanos = 0 + + def ToDatetime(self): + """Converts Timestamp to datetime.""" + return datetime.utcfromtimestamp( + self.seconds + self.nanos / float(_NANOS_PER_SECOND)) + + def FromDatetime(self, dt): + """Converts datetime to Timestamp.""" + td = dt - datetime(1970, 1, 1) + self.seconds = td.seconds + td.days * _SECONDS_PER_DAY + self.nanos = td.microseconds * _NANOS_PER_MICROSECOND + + +class Duration(object): + """Class for Duration message type.""" + + def ToJsonString(self): + """Converts Duration to string format. + + Returns: + A string converted from self. The string format will contains + 3, 6, or 9 fractional digits depending on the precision required to + represent the exact Duration value. For example: "1s", "1.010s", + "1.000000100s", "-3.100s" + """ + if self.seconds < 0 or self.nanos < 0: + result = '-' + seconds = - self.seconds + int((0 - self.nanos) // 1e9) + nanos = (0 - self.nanos) % 1e9 + else: + result = '' + seconds = self.seconds + int(self.nanos // 1e9) + nanos = self.nanos % 1e9 + result += '%d' % seconds + if (nanos % 1e9) == 0: + # If there are 0 fractional digits, the fractional + # point '.' should be omitted when serializing. + return result + 's' + if (nanos % 1e6) == 0: + # Serialize 3 fractional digits. + return result + '.%03ds' % (nanos / 1e6) + if (nanos % 1e3) == 0: + # Serialize 6 fractional digits. + return result + '.%06ds' % (nanos / 1e3) + # Serialize 9 fractional digits. + return result + '.%09ds' % nanos + + def FromJsonString(self, value): + """Converts a string to Duration. + + Args: + value: A string to be converted. The string must end with 's'. Any + fractional digits (or none) are accepted as long as they fit into + precision. For example: "1s", "1.01s", "1.0000001s", "-3.100s + + Raises: + ParseError: On parsing problems. + """ + if len(value) < 1 or value[-1] != 's': + raise ParseError( + 'Duration must end with letter "s": {0}.'.format(value)) + try: + pos = value.find('.') + if pos == -1: + self.seconds = int(value[:-1]) + self.nanos = 0 + else: + self.seconds = int(value[:pos]) + if value[0] == '-': + self.nanos = int(round(float('-0{0}'.format(value[pos: -1])) *1e9)) + else: + self.nanos = int(round(float('0{0}'.format(value[pos: -1])) *1e9)) + except ValueError: + raise ParseError( + 'Couldn\'t parse duration: {0}.'.format(value)) + + def ToNanoseconds(self): + """Converts a Duration to nanoseconds.""" + return self.seconds * _NANOS_PER_SECOND + self.nanos + + def ToMicroseconds(self): + """Converts a Duration to microseconds.""" + micros = _RoundTowardZero(self.nanos, _NANOS_PER_MICROSECOND) + return self.seconds * _MICROS_PER_SECOND + micros + + def ToMilliseconds(self): + """Converts a Duration to milliseconds.""" + millis = _RoundTowardZero(self.nanos, _NANOS_PER_MILLISECOND) + return self.seconds * _MILLIS_PER_SECOND + millis + + def ToSeconds(self): + """Converts a Duration to seconds.""" + return self.seconds + + def FromNanoseconds(self, nanos): + """Converts nanoseconds to Duration.""" + self._NormalizeDuration(nanos // _NANOS_PER_SECOND, + nanos % _NANOS_PER_SECOND) + + def FromMicroseconds(self, micros): + """Converts microseconds to Duration.""" + self._NormalizeDuration( + micros // _MICROS_PER_SECOND, + (micros % _MICROS_PER_SECOND) * _NANOS_PER_MICROSECOND) + + def FromMilliseconds(self, millis): + """Converts milliseconds to Duration.""" + self._NormalizeDuration( + millis // _MILLIS_PER_SECOND, + (millis % _MILLIS_PER_SECOND) * _NANOS_PER_MILLISECOND) + + def FromSeconds(self, seconds): + """Converts seconds to Duration.""" + self.seconds = seconds + self.nanos = 0 + + def ToTimedelta(self): + """Converts Duration to timedelta.""" + return timedelta( + seconds=self.seconds, microseconds=_RoundTowardZero( + self.nanos, _NANOS_PER_MICROSECOND)) + + def FromTimedelta(self, td): + """Convertd timedelta to Duration.""" + self._NormalizeDuration(td.seconds + td.days * _SECONDS_PER_DAY, + td.microseconds * _NANOS_PER_MICROSECOND) + + def _NormalizeDuration(self, seconds, nanos): + """Set Duration by seconds and nonas.""" + # Force nanos to be negative if the duration is negative. + if seconds < 0 and nanos > 0: + seconds += 1 + nanos -= _NANOS_PER_SECOND + self.seconds = seconds + self.nanos = nanos + + +def _RoundTowardZero(value, divider): + """Truncates the remainder part after division.""" + # For some languanges, the sign of the remainder is implementation + # dependent if any of the operands is negative. Here we enforce + # "rounded toward zero" semantics. For example, for (-5) / 2 an + # implementation may give -3 as the result with the remainder being + # 1. This function ensures we always return -2 (closer to zero). + result = value // divider + remainder = value % divider + if result < 0 and remainder > 0: + return result + 1 + else: + return result + + +class FieldMask(object): + """Class for FieldMask message type.""" + + def ToJsonString(self): + """Converts FieldMask to string according to proto3 JSON spec.""" + return ','.join(self.paths) + + def FromJsonString(self, value): + """Converts string to FieldMask according to proto3 JSON spec.""" + self.Clear() + for path in value.split(','): + self.paths.append(path) + + def IsValidForDescriptor(self, message_descriptor): + """Checks whether the FieldMask is valid for Message Descriptor.""" + for path in self.paths: + if not _IsValidPath(message_descriptor, path): + return False + return True + + def AllFieldsFromDescriptor(self, message_descriptor): + """Gets all direct fields of Message Descriptor to FieldMask.""" + self.Clear() + for field in message_descriptor.fields: + self.paths.append(field.name) + + def CanonicalFormFromMask(self, mask): + """Converts a FieldMask to the canonical form. + + Removes paths that are covered by another path. For example, + "foo.bar" is covered by "foo" and will be removed if "foo" + is also in the FieldMask. Then sorts all paths in alphabetical order. + + Args: + mask: The original FieldMask to be converted. + """ + tree = _FieldMaskTree(mask) + tree.ToFieldMask(self) + + def Union(self, mask1, mask2): + """Merges mask1 and mask2 into this FieldMask.""" + _CheckFieldMaskMessage(mask1) + _CheckFieldMaskMessage(mask2) + tree = _FieldMaskTree(mask1) + tree.MergeFromFieldMask(mask2) + tree.ToFieldMask(self) + + def Intersect(self, mask1, mask2): + """Intersects mask1 and mask2 into this FieldMask.""" + _CheckFieldMaskMessage(mask1) + _CheckFieldMaskMessage(mask2) + tree = _FieldMaskTree(mask1) + intersection = _FieldMaskTree() + for path in mask2.paths: + tree.IntersectPath(path, intersection) + intersection.ToFieldMask(self) + + def MergeMessage( + self, source, destination, + replace_message_field=False, replace_repeated_field=False): + """Merges fields specified in FieldMask from source to destination. + + Args: + source: Source message. + destination: The destination message to be merged into. + replace_message_field: Replace message field if True. Merge message + field if False. + replace_repeated_field: Replace repeated field if True. Append + elements of repeated field if False. + """ + tree = _FieldMaskTree(self) + tree.MergeMessage( + source, destination, replace_message_field, replace_repeated_field) + + +def _IsValidPath(message_descriptor, path): + """Checks whether the path is valid for Message Descriptor.""" + parts = path.split('.') + last = parts.pop() + for name in parts: + field = message_descriptor.fields_by_name[name] + if (field is None or + field.label == FieldDescriptor.LABEL_REPEATED or + field.type != FieldDescriptor.TYPE_MESSAGE): + return False + message_descriptor = field.message_type + return last in message_descriptor.fields_by_name + + +def _CheckFieldMaskMessage(message): + """Raises ValueError if message is not a FieldMask.""" + message_descriptor = message.DESCRIPTOR + if (message_descriptor.name != 'FieldMask' or + message_descriptor.file.name != 'google/protobuf/field_mask.proto'): + raise ValueError('Message {0} is not a FieldMask.'.format( + message_descriptor.full_name)) + + +class _FieldMaskTree(object): + """Represents a FieldMask in a tree structure. + + For example, given a FieldMask "foo.bar,foo.baz,bar.baz", + the FieldMaskTree will be: + [_root] -+- foo -+- bar + | | + | +- baz + | + +- bar --- baz + In the tree, each leaf node represents a field path. + """ + + def __init__(self, field_mask=None): + """Initializes the tree by FieldMask.""" + self._root = {} + if field_mask: + self.MergeFromFieldMask(field_mask) + + def MergeFromFieldMask(self, field_mask): + """Merges a FieldMask to the tree.""" + for path in field_mask.paths: + self.AddPath(path) + + def AddPath(self, path): + """Adds a field path into the tree. + + If the field path to add is a sub-path of an existing field path + in the tree (i.e., a leaf node), it means the tree already matches + the given path so nothing will be added to the tree. If the path + matches an existing non-leaf node in the tree, that non-leaf node + will be turned into a leaf node with all its children removed because + the path matches all the node's children. Otherwise, a new path will + be added. + + Args: + path: The field path to add. + """ + node = self._root + for name in path.split('.'): + if name not in node: + node[name] = {} + elif not node[name]: + # Pre-existing empty node implies we already have this entire tree. + return + node = node[name] + # Remove any sub-trees we might have had. + node.clear() + + def ToFieldMask(self, field_mask): + """Converts the tree to a FieldMask.""" + field_mask.Clear() + _AddFieldPaths(self._root, '', field_mask) + + def IntersectPath(self, path, intersection): + """Calculates the intersection part of a field path with this tree. + + Args: + path: The field path to calculates. + intersection: The out tree to record the intersection part. + """ + node = self._root + for name in path.split('.'): + if name not in node: + return + elif not node[name]: + intersection.AddPath(path) + return + node = node[name] + intersection.AddLeafNodes(path, node) + + def AddLeafNodes(self, prefix, node): + """Adds leaf nodes begin with prefix to this tree.""" + if not node: + self.AddPath(prefix) + for name in node: + child_path = prefix + '.' + name + self.AddLeafNodes(child_path, node[name]) + + def MergeMessage( + self, source, destination, + replace_message, replace_repeated): + """Merge all fields specified by this tree from source to destination.""" + _MergeMessage( + self._root, source, destination, replace_message, replace_repeated) + + +def _StrConvert(value): + """Converts value to str if it is not.""" + # This file is imported by c extension and some methods like ClearField + # requires string for the field name. py2/py3 has different text + # type and may use unicode. + if not isinstance(value, str): + return value.encode('utf-8') + return value + + +def _MergeMessage( + node, source, destination, replace_message, replace_repeated): + """Merge all fields specified by a sub-tree from source to destination.""" + source_descriptor = source.DESCRIPTOR + for name in node: + child = node[name] + field = source_descriptor.fields_by_name[name] + if field is None: + raise ValueError('Error: Can\'t find field {0} in message {1}.'.format( + name, source_descriptor.full_name)) + if child: + # Sub-paths are only allowed for singular message fields. + if (field.label == FieldDescriptor.LABEL_REPEATED or + field.cpp_type != FieldDescriptor.CPPTYPE_MESSAGE): + raise ValueError('Error: Field {0} in message {1} is not a singular ' + 'message field and cannot have sub-fields.'.format( + name, source_descriptor.full_name)) + _MergeMessage( + child, getattr(source, name), getattr(destination, name), + replace_message, replace_repeated) + continue + if field.label == FieldDescriptor.LABEL_REPEATED: + if replace_repeated: + destination.ClearField(_StrConvert(name)) + repeated_source = getattr(source, name) + repeated_destination = getattr(destination, name) + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + for item in repeated_source: + repeated_destination.add().MergeFrom(item) + else: + repeated_destination.extend(repeated_source) + else: + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + if replace_message: + destination.ClearField(_StrConvert(name)) + if source.HasField(name): + getattr(destination, name).MergeFrom(getattr(source, name)) + else: + setattr(destination, name, getattr(source, name)) + + +def _AddFieldPaths(node, prefix, field_mask): + """Adds the field paths descended from node to field_mask.""" + if not node: + field_mask.paths.append(prefix) + return + for name in sorted(node): + if prefix: + child_path = prefix + '.' + name + else: + child_path = name + _AddFieldPaths(node[name], child_path, field_mask) + + +WKTBASES = { + 'google.protobuf.Any': Any, + 'google.protobuf.Duration': Duration, + 'google.protobuf.FieldMask': FieldMask, + 'google.protobuf.Timestamp': Timestamp, +} diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py new file mode 100644 index 00000000..60b0c47d --- /dev/null +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -0,0 +1,509 @@ +#! /usr/bin/env python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test for google.protobuf.internal.well_known_types.""" + +__author__ = 'jieluo@google.com (Jie Luo)' + +from datetime import datetime + +from google.protobuf import duration_pb2 +from google.protobuf import field_mask_pb2 +from google.protobuf import timestamp_pb2 +import unittest +from google.protobuf import unittest_pb2 +from google.protobuf.internal import test_util +from google.protobuf.internal import well_known_types +from google.protobuf import descriptor + + +class TimeUtilTestBase(unittest.TestCase): + + def CheckTimestampConversion(self, message, text): + self.assertEqual(text, message.ToJsonString()) + parsed_message = timestamp_pb2.Timestamp() + parsed_message.FromJsonString(text) + self.assertEqual(message, parsed_message) + + def CheckDurationConversion(self, message, text): + self.assertEqual(text, message.ToJsonString()) + parsed_message = duration_pb2.Duration() + parsed_message.FromJsonString(text) + self.assertEqual(message, parsed_message) + + +class TimeUtilTest(TimeUtilTestBase): + + def testTimestampSerializeAndParse(self): + message = timestamp_pb2.Timestamp() + # Generated output should contain 3, 6, or 9 fractional digits. + message.seconds = 0 + message.nanos = 0 + self.CheckTimestampConversion(message, '1970-01-01T00:00:00Z') + message.nanos = 10000000 + self.CheckTimestampConversion(message, '1970-01-01T00:00:00.010Z') + message.nanos = 10000 + self.CheckTimestampConversion(message, '1970-01-01T00:00:00.000010Z') + message.nanos = 10 + self.CheckTimestampConversion(message, '1970-01-01T00:00:00.000000010Z') + # Test min timestamps. + message.seconds = -62135596800 + message.nanos = 0 + self.CheckTimestampConversion(message, '0001-01-01T00:00:00Z') + # Test max timestamps. + message.seconds = 253402300799 + message.nanos = 999999999 + self.CheckTimestampConversion(message, '9999-12-31T23:59:59.999999999Z') + # Test negative timestamps. + message.seconds = -1 + self.CheckTimestampConversion(message, '1969-12-31T23:59:59.999999999Z') + + # Parsing accepts an fractional digits as long as they fit into nano + # precision. + message.FromJsonString('1970-01-01T00:00:00.1Z') + self.assertEqual(0, message.seconds) + self.assertEqual(100000000, message.nanos) + # Parsing accpets offsets. + message.FromJsonString('1970-01-01T00:00:00-08:00') + self.assertEqual(8 * 3600, message.seconds) + self.assertEqual(0, message.nanos) + + def testDurationSerializeAndParse(self): + message = duration_pb2.Duration() + # Generated output should contain 3, 6, or 9 fractional digits. + message.seconds = 0 + message.nanos = 0 + self.CheckDurationConversion(message, '0s') + message.nanos = 10000000 + self.CheckDurationConversion(message, '0.010s') + message.nanos = 10000 + self.CheckDurationConversion(message, '0.000010s') + message.nanos = 10 + self.CheckDurationConversion(message, '0.000000010s') + + # Test min and max + message.seconds = 315576000000 + message.nanos = 999999999 + self.CheckDurationConversion(message, '315576000000.999999999s') + message.seconds = -315576000000 + message.nanos = -999999999 + self.CheckDurationConversion(message, '-315576000000.999999999s') + + # Parsing accepts an fractional digits as long as they fit into nano + # precision. + message.FromJsonString('0.1s') + self.assertEqual(100000000, message.nanos) + message.FromJsonString('0.0000001s') + self.assertEqual(100, message.nanos) + + def testTimestampIntegerConversion(self): + message = timestamp_pb2.Timestamp() + message.FromNanoseconds(1) + self.assertEqual('1970-01-01T00:00:00.000000001Z', + message.ToJsonString()) + self.assertEqual(1, message.ToNanoseconds()) + + message.FromNanoseconds(-1) + self.assertEqual('1969-12-31T23:59:59.999999999Z', + message.ToJsonString()) + self.assertEqual(-1, message.ToNanoseconds()) + + message.FromMicroseconds(1) + self.assertEqual('1970-01-01T00:00:00.000001Z', + message.ToJsonString()) + self.assertEqual(1, message.ToMicroseconds()) + + message.FromMicroseconds(-1) + self.assertEqual('1969-12-31T23:59:59.999999Z', + message.ToJsonString()) + self.assertEqual(-1, message.ToMicroseconds()) + + message.FromMilliseconds(1) + self.assertEqual('1970-01-01T00:00:00.001Z', + message.ToJsonString()) + self.assertEqual(1, message.ToMilliseconds()) + + message.FromMilliseconds(-1) + self.assertEqual('1969-12-31T23:59:59.999Z', + message.ToJsonString()) + self.assertEqual(-1, message.ToMilliseconds()) + + message.FromSeconds(1) + self.assertEqual('1970-01-01T00:00:01Z', + message.ToJsonString()) + self.assertEqual(1, message.ToSeconds()) + + message.FromSeconds(-1) + self.assertEqual('1969-12-31T23:59:59Z', + message.ToJsonString()) + self.assertEqual(-1, message.ToSeconds()) + + message.FromNanoseconds(1999) + self.assertEqual(1, message.ToMicroseconds()) + # For negative values, Timestamp will be rounded down. + # For example, "1969-12-31T23:59:59.5Z" (i.e., -0.5s) rounded to seconds + # will be "1969-12-31T23:59:59Z" (i.e., -1s) rather than + # "1970-01-01T00:00:00Z" (i.e., 0s). + message.FromNanoseconds(-1999) + self.assertEqual(-2, message.ToMicroseconds()) + + def testDurationIntegerConversion(self): + message = duration_pb2.Duration() + message.FromNanoseconds(1) + self.assertEqual('0.000000001s', + message.ToJsonString()) + self.assertEqual(1, message.ToNanoseconds()) + + message.FromNanoseconds(-1) + self.assertEqual('-0.000000001s', + message.ToJsonString()) + self.assertEqual(-1, message.ToNanoseconds()) + + message.FromMicroseconds(1) + self.assertEqual('0.000001s', + message.ToJsonString()) + self.assertEqual(1, message.ToMicroseconds()) + + message.FromMicroseconds(-1) + self.assertEqual('-0.000001s', + message.ToJsonString()) + self.assertEqual(-1, message.ToMicroseconds()) + + message.FromMilliseconds(1) + self.assertEqual('0.001s', + message.ToJsonString()) + self.assertEqual(1, message.ToMilliseconds()) + + message.FromMilliseconds(-1) + self.assertEqual('-0.001s', + message.ToJsonString()) + self.assertEqual(-1, message.ToMilliseconds()) + + message.FromSeconds(1) + self.assertEqual('1s', message.ToJsonString()) + self.assertEqual(1, message.ToSeconds()) + + message.FromSeconds(-1) + self.assertEqual('-1s', + message.ToJsonString()) + self.assertEqual(-1, message.ToSeconds()) + + # Test truncation behavior. + message.FromNanoseconds(1999) + self.assertEqual(1, message.ToMicroseconds()) + + # For negative values, Duration will be rounded towards 0. + message.FromNanoseconds(-1999) + self.assertEqual(-1, message.ToMicroseconds()) + + def testDatetimeConverison(self): + message = timestamp_pb2.Timestamp() + dt = datetime(1970, 1, 1) + message.FromDatetime(dt) + self.assertEqual(dt, message.ToDatetime()) + + message.FromMilliseconds(1999) + self.assertEqual(datetime(1970, 1, 1, 0, 0, 1, 999000), + message.ToDatetime()) + + def testTimedeltaConversion(self): + message = duration_pb2.Duration() + message.FromNanoseconds(1999999999) + td = message.ToTimedelta() + self.assertEqual(1, td.seconds) + self.assertEqual(999999, td.microseconds) + + message.FromNanoseconds(-1999999999) + td = message.ToTimedelta() + self.assertEqual(-1, td.days) + self.assertEqual(86398, td.seconds) + self.assertEqual(1, td.microseconds) + + message.FromMicroseconds(-1) + td = message.ToTimedelta() + self.assertEqual(-1, td.days) + self.assertEqual(86399, td.seconds) + self.assertEqual(999999, td.microseconds) + converted_message = duration_pb2.Duration() + converted_message.FromTimedelta(td) + self.assertEqual(message, converted_message) + + def testInvalidTimestamp(self): + message = timestamp_pb2.Timestamp() + self.assertRaisesRegexp( + ValueError, + 'time data \'10000-01-01T00:00:00\' does not match' + ' format \'%Y-%m-%dT%H:%M:%S\'', + message.FromJsonString, '10000-01-01T00:00:00.00Z') + self.assertRaisesRegexp( + well_known_types.ParseError, + 'nanos 0123456789012 more than 9 fractional digits.', + message.FromJsonString, + '1970-01-01T00:00:00.0123456789012Z') + self.assertRaisesRegexp( + well_known_types.ParseError, + (r'Invalid timezone offset value: \+08.'), + message.FromJsonString, + '1972-01-01T01:00:00.01+08',) + self.assertRaisesRegexp( + ValueError, + 'year is out of range', + message.FromJsonString, + '0000-01-01T00:00:00Z') + message.seconds = 253402300800 + self.assertRaisesRegexp( + OverflowError, + 'date value out of range', + message.ToJsonString) + + def testInvalidDuration(self): + message = duration_pb2.Duration() + self.assertRaisesRegexp( + well_known_types.ParseError, + 'Duration must end with letter "s": 1.', + message.FromJsonString, '1') + self.assertRaisesRegexp( + well_known_types.ParseError, + 'Couldn\'t parse duration: 1...2s.', + message.FromJsonString, '1...2s') + + +class FieldMaskTest(unittest.TestCase): + + def testStringFormat(self): + mask = field_mask_pb2.FieldMask() + self.assertEqual('', mask.ToJsonString()) + mask.paths.append('foo') + self.assertEqual('foo', mask.ToJsonString()) + mask.paths.append('bar') + self.assertEqual('foo,bar', mask.ToJsonString()) + + mask.FromJsonString('') + self.assertEqual('', mask.ToJsonString()) + mask.FromJsonString('foo') + self.assertEqual(['foo'], mask.paths) + mask.FromJsonString('foo,bar') + self.assertEqual(['foo', 'bar'], mask.paths) + + def testDescriptorToFieldMask(self): + mask = field_mask_pb2.FieldMask() + msg_descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR + mask.AllFieldsFromDescriptor(msg_descriptor) + self.assertEqual(75, len(mask.paths)) + self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) + for field in msg_descriptor.fields: + self.assertTrue(field.name in mask.paths) + mask.paths.append('optional_nested_message.bb') + self.assertTrue(mask.IsValidForDescriptor(msg_descriptor)) + mask.paths.append('repeated_nested_message.bb') + self.assertFalse(mask.IsValidForDescriptor(msg_descriptor)) + + def testCanonicalFrom(self): + mask = field_mask_pb2.FieldMask() + out_mask = field_mask_pb2.FieldMask() + # Paths will be sorted. + mask.FromJsonString('baz.quz,bar,foo') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('bar,baz.quz,foo', out_mask.ToJsonString()) + # Duplicated paths will be removed. + mask.FromJsonString('foo,bar,foo') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('bar,foo', out_mask.ToJsonString()) + # Sub-paths of other paths will be removed. + mask.FromJsonString('foo.b1,bar.b1,foo.b2,bar') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('bar,foo.b1,foo.b2', out_mask.ToJsonString()) + + # Test more deeply nested cases. + mask.FromJsonString( + 'foo.bar.baz1,foo.bar.baz2.quz,foo.bar.baz2') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('foo.bar.baz1,foo.bar.baz2', + out_mask.ToJsonString()) + mask.FromJsonString( + 'foo.bar.baz1,foo.bar.baz2,foo.bar.baz2.quz') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('foo.bar.baz1,foo.bar.baz2', + out_mask.ToJsonString()) + mask.FromJsonString( + 'foo.bar.baz1,foo.bar.baz2,foo.bar.baz2.quz,foo.bar') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('foo.bar', out_mask.ToJsonString()) + mask.FromJsonString( + 'foo.bar.baz1,foo.bar.baz2,foo.bar.baz2.quz,foo') + out_mask.CanonicalFormFromMask(mask) + self.assertEqual('foo', out_mask.ToJsonString()) + + def testUnion(self): + mask1 = field_mask_pb2.FieldMask() + mask2 = field_mask_pb2.FieldMask() + out_mask = field_mask_pb2.FieldMask() + mask1.FromJsonString('foo,baz') + mask2.FromJsonString('bar,quz') + out_mask.Union(mask1, mask2) + self.assertEqual('bar,baz,foo,quz', out_mask.ToJsonString()) + # Overlap with duplicated paths. + mask1.FromJsonString('foo,baz.bb') + mask2.FromJsonString('baz.bb,quz') + out_mask.Union(mask1, mask2) + self.assertEqual('baz.bb,foo,quz', out_mask.ToJsonString()) + # Overlap with paths covering some other paths. + mask1.FromJsonString('foo.bar.baz,quz') + mask2.FromJsonString('foo.bar,bar') + out_mask.Union(mask1, mask2) + self.assertEqual('bar,foo.bar,quz', out_mask.ToJsonString()) + + def testIntersect(self): + mask1 = field_mask_pb2.FieldMask() + mask2 = field_mask_pb2.FieldMask() + out_mask = field_mask_pb2.FieldMask() + # Test cases without overlapping. + mask1.FromJsonString('foo,baz') + mask2.FromJsonString('bar,quz') + out_mask.Intersect(mask1, mask2) + self.assertEqual('', out_mask.ToJsonString()) + # Overlap with duplicated paths. + mask1.FromJsonString('foo,baz.bb') + mask2.FromJsonString('baz.bb,quz') + out_mask.Intersect(mask1, mask2) + self.assertEqual('baz.bb', out_mask.ToJsonString()) + # Overlap with paths covering some other paths. + mask1.FromJsonString('foo.bar.baz,quz') + mask2.FromJsonString('foo.bar,bar') + out_mask.Intersect(mask1, mask2) + self.assertEqual('foo.bar.baz', out_mask.ToJsonString()) + mask1.FromJsonString('foo.bar,bar') + mask2.FromJsonString('foo.bar.baz,quz') + out_mask.Intersect(mask1, mask2) + self.assertEqual('foo.bar.baz', out_mask.ToJsonString()) + + def testMergeMessage(self): + # Test merge one field. + src = unittest_pb2.TestAllTypes() + test_util.SetAllFields(src) + for field in src.DESCRIPTOR.fields: + if field.containing_oneof: + continue + field_name = field.name + dst = unittest_pb2.TestAllTypes() + # Only set one path to mask. + mask = field_mask_pb2.FieldMask() + mask.paths.append(field_name) + mask.MergeMessage(src, dst) + # The expected result message. + msg = unittest_pb2.TestAllTypes() + if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: + repeated_src = getattr(src, field_name) + repeated_msg = getattr(msg, field_name) + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + for item in repeated_src: + repeated_msg.add().CopyFrom(item) + else: + repeated_msg.extend(repeated_src) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + getattr(msg, field_name).CopyFrom(getattr(src, field_name)) + else: + setattr(msg, field_name, getattr(src, field_name)) + # Only field specified in mask is merged. + self.assertEqual(msg, dst) + + # Test merge nested fields. + nested_src = unittest_pb2.NestedTestAllTypes() + nested_dst = unittest_pb2.NestedTestAllTypes() + nested_src.child.payload.optional_int32 = 1234 + nested_src.child.child.payload.optional_int32 = 5678 + mask = field_mask_pb2.FieldMask() + mask.FromJsonString('child.payload') + mask.MergeMessage(nested_src, nested_dst) + self.assertEqual(1234, nested_dst.child.payload.optional_int32) + self.assertEqual(0, nested_dst.child.child.payload.optional_int32) + + mask.FromJsonString('child.child.payload') + mask.MergeMessage(nested_src, nested_dst) + self.assertEqual(1234, nested_dst.child.payload.optional_int32) + self.assertEqual(5678, nested_dst.child.child.payload.optional_int32) + + nested_dst.Clear() + mask.FromJsonString('child.child.payload') + mask.MergeMessage(nested_src, nested_dst) + self.assertEqual(0, nested_dst.child.payload.optional_int32) + self.assertEqual(5678, nested_dst.child.child.payload.optional_int32) + + nested_dst.Clear() + mask.FromJsonString('child') + mask.MergeMessage(nested_src, nested_dst) + self.assertEqual(1234, nested_dst.child.payload.optional_int32) + self.assertEqual(5678, nested_dst.child.child.payload.optional_int32) + + # Test MergeOptions. + nested_dst.Clear() + nested_dst.child.payload.optional_int64 = 4321 + # Message fields will be merged by default. + mask.FromJsonString('child.payload') + mask.MergeMessage(nested_src, nested_dst) + self.assertEqual(1234, nested_dst.child.payload.optional_int32) + self.assertEqual(4321, nested_dst.child.payload.optional_int64) + # Change the behavior to replace message fields. + mask.FromJsonString('child.payload') + mask.MergeMessage(nested_src, nested_dst, True, False) + self.assertEqual(1234, nested_dst.child.payload.optional_int32) + self.assertEqual(0, nested_dst.child.payload.optional_int64) + + # By default, fields missing in source are not cleared in destination. + nested_dst.payload.optional_int32 = 1234 + self.assertTrue(nested_dst.HasField('payload')) + mask.FromJsonString('payload') + mask.MergeMessage(nested_src, nested_dst) + self.assertTrue(nested_dst.HasField('payload')) + # But they are cleared when replacing message fields. + nested_dst.Clear() + nested_dst.payload.optional_int32 = 1234 + mask.FromJsonString('payload') + mask.MergeMessage(nested_src, nested_dst, True, False) + self.assertFalse(nested_dst.HasField('payload')) + + nested_src.payload.repeated_int32.append(1234) + nested_dst.payload.repeated_int32.append(5678) + # Repeated fields will be appended by default. + mask.FromJsonString('payload.repeated_int32') + mask.MergeMessage(nested_src, nested_dst) + self.assertEqual(2, len(nested_dst.payload.repeated_int32)) + self.assertEqual(5678, nested_dst.payload.repeated_int32[0]) + self.assertEqual(1234, nested_dst.payload.repeated_int32[1]) + # Change the behavior to replace repeated fields. + mask.FromJsonString('payload.repeated_int32') + mask.MergeMessage(nested_src, nested_dst, False, True) + self.assertEqual(1, len(nested_dst.payload.repeated_int32)) + self.assertEqual(1234, nested_dst.payload.repeated_int32[0]) + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index d95557d7..cb76e116 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py @@ -28,22 +28,29 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Contains routines for printing protocol messages in JSON format.""" +"""Contains routines for printing protocol messages in JSON format. + +Simple usage example: + + # Create a proto object and serialize it to a json format string. + message = my_proto_pb2.MyMessage(foo='bar') + json_string = json_format.MessageToJson(message) + + # Parse a json format string to proto object. + message = json_format.Parse(json_string, my_proto_pb2.MyMessage()) +""" __author__ = 'jieluo@google.com (Jie Luo)' import base64 -from datetime import datetime import json import math -import re +from six import text_type import sys from google.protobuf import descriptor _TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' -_NUMBER = re.compile(u'[0-9+-][0-9e.+-]*') -_INTEGER = re.compile(u'[0-9+-]') _INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32, descriptor.FieldDescriptor.CPPTYPE_UINT32, descriptor.FieldDescriptor.CPPTYPE_INT64, @@ -52,17 +59,20 @@ _INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64, descriptor.FieldDescriptor.CPPTYPE_UINT64]) _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) -if str is bytes: - _UNICODETYPE = unicode -else: - _UNICODETYPE = str +_INFINITY = 'Infinity' +_NEG_INFINITY = '-Infinity' +_NAN = 'NaN' + +class Error(Exception): + """Top-level module error for json_format.""" -class SerializeToJsonError(Exception): + +class SerializeToJsonError(Error): """Thrown if serialization to JSON fails.""" -class ParseError(Exception): +class ParseError(Error): """Thrown in case of parsing error.""" @@ -86,12 +96,8 @@ def MessageToJson(message, including_default_value_fields=False): def _MessageToJsonObject(message, including_default_value_fields): """Converts message to an object according to Proto3 JSON Specification.""" message_descriptor = message.DESCRIPTOR - if _IsTimestampMessage(message_descriptor): - return _TimestampMessageToJsonObject(message) - if _IsDurationMessage(message_descriptor): - return _DurationMessageToJsonObject(message) - if _IsFieldMaskMessage(message_descriptor): - return _FieldMaskMessageToJsonObject(message) + if hasattr(message, 'ToJsonString'): + return message.ToJsonString() if _IsWrapperMessage(message_descriptor): return _WrapperMessageToJsonObject(message) return _RegularMessageToJsonObject(message, including_default_value_fields) @@ -107,12 +113,14 @@ def _RegularMessageToJsonObject(message, including_default_value_fields): """Converts normal message according to Proto3 JSON Specification.""" js = {} fields = message.ListFields() + include_default = including_default_value_fields try: for field, value in fields: name = field.camelcase_name if _IsMapEntry(field): # Convert a map field. + v_field = field.message_type.fields_by_name['value'] js_map = {} for key in value: if isinstance(key, bool): @@ -122,20 +130,15 @@ def _RegularMessageToJsonObject(message, including_default_value_fields): recorded_key = 'false' else: recorded_key = key - js_map[recorded_key] = _ConvertFieldToJsonObject( - field.message_type.fields_by_name['value'], - value[key], including_default_value_fields) + js_map[recorded_key] = _FieldToJsonObject( + v_field, value[key], including_default_value_fields) js[name] = js_map elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: # Convert a repeated field. - repeated = [] - for element in value: - repeated.append(_ConvertFieldToJsonObject( - field, element, including_default_value_fields)) - js[name] = repeated + js[name] = [_FieldToJsonObject(field, k, include_default) + for k in value] else: - js[name] = _ConvertFieldToJsonObject( - field, value, including_default_value_fields) + js[name] = _FieldToJsonObject(field, value, include_default) # Serialize default value if including_default_value_fields is True. if including_default_value_fields: @@ -155,16 +158,16 @@ def _RegularMessageToJsonObject(message, including_default_value_fields): elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: js[name] = [] else: - js[name] = _ConvertFieldToJsonObject(field, field.default_value) + js[name] = _FieldToJsonObject(field, field.default_value) except ValueError as e: raise SerializeToJsonError( - 'Failed to serialize {0} field: {1}'.format(field.name, e)) + 'Failed to serialize {0} field: {1}.'.format(field.name, e)) return js -def _ConvertFieldToJsonObject( +def _FieldToJsonObject( field, value, including_default_value_fields=False): """Converts field value according to Proto3 JSON Specification.""" if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: @@ -183,101 +186,26 @@ def _ConvertFieldToJsonObject( else: return value elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: - if value: - return True - else: - return False + return bool(value) elif field.cpp_type in _INT64_TYPES: return str(value) elif field.cpp_type in _FLOAT_TYPES: if math.isinf(value): if value < 0.0: - return '-Infinity' + return _NEG_INFINITY else: - return 'Infinity' + return _INFINITY if math.isnan(value): - return 'NaN' + return _NAN return value -def _IsTimestampMessage(message_descriptor): - return (message_descriptor.name == 'Timestamp' and - message_descriptor.file.name == 'google/protobuf/timestamp.proto') - - -def _TimestampMessageToJsonObject(message): - """Converts Timestamp message according to Proto3 JSON Specification.""" - nanos = message.nanos % 1e9 - dt = datetime.utcfromtimestamp( - message.seconds + (message.nanos - nanos) / 1e9) - result = dt.isoformat() - if (nanos % 1e9) == 0: - # If there are 0 fractional digits, the fractional - # point '.' should be omitted when serializing. - return result + 'Z' - if (nanos % 1e6) == 0: - # Serialize 3 fractional digits. - return result + '.%03dZ' % (nanos / 1e6) - if (nanos % 1e3) == 0: - # Serialize 6 fractional digits. - return result + '.%06dZ' % (nanos / 1e3) - # Serialize 9 fractional digits. - return result + '.%09dZ' % nanos - - -def _IsDurationMessage(message_descriptor): - return (message_descriptor.name == 'Duration' and - message_descriptor.file.name == 'google/protobuf/duration.proto') - - -def _DurationMessageToJsonObject(message): - """Converts Duration message according to Proto3 JSON Specification.""" - if message.seconds < 0 or message.nanos < 0: - result = '-' - seconds = - message.seconds + int((0 - message.nanos) / 1e9) - nanos = (0 - message.nanos) % 1e9 - else: - result = '' - seconds = message.seconds + int(message.nanos / 1e9) - nanos = message.nanos % 1e9 - result += '%d' % seconds - if (nanos % 1e9) == 0: - # If there are 0 fractional digits, the fractional - # point '.' should be omitted when serializing. - return result + 's' - if (nanos % 1e6) == 0: - # Serialize 3 fractional digits. - return result + '.%03ds' % (nanos / 1e6) - if (nanos % 1e3) == 0: - # Serialize 6 fractional digits. - return result + '.%06ds' % (nanos / 1e3) - # Serialize 9 fractional digits. - return result + '.%09ds' % nanos - - -def _IsFieldMaskMessage(message_descriptor): - return (message_descriptor.name == 'FieldMask' and - message_descriptor.file.name == 'google/protobuf/field_mask.proto') - - -def _FieldMaskMessageToJsonObject(message): - """Converts FieldMask message according to Proto3 JSON Specification.""" - result = '' - first = True - for path in message.paths: - if not first: - result += ',' - result += path - first = False - return result - - def _IsWrapperMessage(message_descriptor): return message_descriptor.file.name == 'google/protobuf/wrappers.proto' def _WrapperMessageToJsonObject(message): - return _ConvertFieldToJsonObject( + return _FieldToJsonObject( message.DESCRIPTOR.fields_by_name['value'], message.value) @@ -285,7 +213,7 @@ def _DuplicateChecker(js): result = {} for name, value in js: if name in result: - raise ParseError('Failed to load JSON: duplicate key ' + name) + raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name)) result[name] = value return result @@ -303,7 +231,7 @@ def Parse(text, message): Raises:: ParseError: On JSON parsing problems. """ - if not isinstance(text, _UNICODETYPE): text = text.decode('utf-8') + if not isinstance(text, text_type): text = text.decode('utf-8') try: if sys.version_info < (2, 7): # object_pair_hook is not supported before python2.7 @@ -311,7 +239,7 @@ def Parse(text, message): else: js = json.loads(text, object_pairs_hook=_DuplicateChecker) except ValueError as e: - raise ParseError('Failed to load JSON: ' + str(e)) + raise ParseError('Failed to load JSON: {0}.'.format(str(e))) _ConvertFieldValuePair(js, message) return message @@ -362,7 +290,7 @@ def _ConvertFieldValuePair(js, message): message.ClearField(field.name) if not isinstance(value, list): raise ParseError('repeated field {0} must be in [] which is ' - '{1}'.format(name, value)) + '{1}.'.format(name, value)) for item in value: if item is None: continue @@ -383,9 +311,9 @@ def _ConvertFieldValuePair(js, message): else: raise ParseError(str(e)) except ValueError as e: - raise ParseError('Failed to parse {0} field: {1}'.format(name, e)) + raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) except TypeError as e: - raise ParseError('Failed to parse {0} field: {1}'.format(name, e)) + raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) def _ConvertMessage(value, message): @@ -399,88 +327,13 @@ def _ConvertMessage(value, message): ParseError: In case of convert problems. """ message_descriptor = message.DESCRIPTOR - if _IsTimestampMessage(message_descriptor): - _ConvertTimestampMessage(value, message) - elif _IsDurationMessage(message_descriptor): - _ConvertDurationMessage(value, message) - elif _IsFieldMaskMessage(message_descriptor): - _ConvertFieldMaskMessage(value, message) + if hasattr(message, 'FromJsonString'): + message.FromJsonString(value) elif _IsWrapperMessage(message_descriptor): _ConvertWrapperMessage(value, message) else: _ConvertFieldValuePair(value, message) - -def _ConvertTimestampMessage(value, message): - """Convert a JSON representation into Timestamp message.""" - timezone_offset = value.find('Z') - if timezone_offset == -1: - timezone_offset = value.find('+') - if timezone_offset == -1: - timezone_offset = value.rfind('-') - if timezone_offset == -1: - raise ParseError( - 'Failed to parse timestamp: missing valid timezone offset.') - time_value = value[0:timezone_offset] - # Parse datetime and nanos - point_position = time_value.find('.') - if point_position == -1: - second_value = time_value - nano_value = '' - else: - second_value = time_value[:point_position] - nano_value = time_value[point_position + 1:] - date_object = datetime.strptime(second_value, _TIMESTAMPFOMAT) - td = date_object - datetime(1970, 1, 1) - seconds = td.seconds + td.days * 24 * 3600 - if len(nano_value) > 9: - raise ParseError( - 'Failed to parse Timestamp: nanos {0} more than ' - '9 fractional digits.'.format(nano_value)) - if nano_value: - nanos = round(float('0.' + nano_value) * 1e9) - else: - nanos = 0 - # Parse timezone offsets - if value[timezone_offset] == 'Z': - if len(value) != timezone_offset + 1: - raise ParseError( - 'Failed to parse timestamp: invalid trailing data {0}.'.format(value)) - else: - timezone = value[timezone_offset:] - pos = timezone.find(':') - if pos == -1: - raise ParseError( - 'Invalid timezone offset value: ' + timezone) - if timezone[0] == '+': - seconds += (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60 - else: - seconds -= (int(timezone[1:pos])*60+int(timezone[pos+1:]))*60 - # Set seconds and nanos - message.seconds = int(seconds) - message.nanos = int(nanos) - - -def _ConvertDurationMessage(value, message): - """Convert a JSON representation into Duration message.""" - if value[-1] != 's': - raise ParseError( - 'Duration must end with letter "s": ' + value) - try: - duration = float(value[:-1]) - except ValueError: - raise ParseError( - 'Couldn\'t parse duration: ' + value) - message.seconds = int(duration) - message.nanos = int(round((duration - message.seconds) * 1e9)) - - -def _ConvertFieldMaskMessage(value, message): - """Convert a JSON representation into FieldMask message.""" - for path in value.split(','): - message.paths.append(path) - - def _ConvertWrapperMessage(value, message): """Convert a JSON representation into Wrapper message.""" field = message.DESCRIPTOR.fields_by_name['value'] @@ -512,13 +365,13 @@ def _ConvertMapFieldValue(value, message, field): value[key], value_field) -def _ConvertScalarFieldValue(value, field, require_quote=False): +def _ConvertScalarFieldValue(value, field, require_str=False): """Convert a single scalar field value. Args: value: A scalar value to convert the scalar field value. field: The descriptor of the field to convert. - require_quote: If True, '"' is required for the field value. + require_str: If True, the field value must be a str. Returns: The converted scalar field value @@ -531,7 +384,7 @@ def _ConvertScalarFieldValue(value, field, require_quote=False): elif field.cpp_type in _FLOAT_TYPES: return _ConvertFloat(value) elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: - return _ConvertBool(value, require_quote) + return _ConvertBool(value, require_str) elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: if field.type == descriptor.FieldDescriptor.TYPE_BYTES: return base64.b64decode(value) @@ -561,10 +414,10 @@ def _ConvertInteger(value): ParseError: If an integer couldn't be consumed. """ if isinstance(value, float): - raise ParseError('Couldn\'t parse integer: {0}'.format(value)) + raise ParseError('Couldn\'t parse integer: {0}.'.format(value)) - if isinstance(value, _UNICODETYPE) and not _INTEGER.match(value): - raise ParseError('Couldn\'t parse integer: "{0}"'.format(value)) + if isinstance(value, text_type) and value.find(' ') != -1: + raise ParseError('Couldn\'t parse integer: "{0}".'.format(value)) return int(value) @@ -572,28 +425,28 @@ def _ConvertInteger(value): def _ConvertFloat(value): """Convert an floating point number.""" if value == 'nan': - raise ParseError('Couldn\'t parse float "nan", use "NaN" instead') + raise ParseError('Couldn\'t parse float "nan", use "NaN" instead.') try: # Assume Python compatible syntax. return float(value) except ValueError: # Check alternative spellings. - if value == '-Infinity': + if value == _NEG_INFINITY: return float('-inf') - elif value == 'Infinity': + elif value == _INFINITY: return float('inf') - elif value == 'NaN': + elif value == _NAN: return float('nan') else: - raise ParseError('Couldn\'t parse float: {0}'.format(value)) + raise ParseError('Couldn\'t parse float: {0}.'.format(value)) -def _ConvertBool(value, require_quote): +def _ConvertBool(value, require_str): """Convert a boolean value. Args: value: A scalar value to convert. - require_quote: If True, '"' is required for the boolean value. + require_str: If True, value must be a str. Returns: The bool parsed. @@ -601,13 +454,13 @@ def _ConvertBool(value, require_quote): Raises: ParseError: If a boolean value couldn't be consumed. """ - if require_quote: + if require_str: if value == 'true': return True elif value == 'false': return False else: - raise ParseError('Expect "true" or "false", not {0}.'.format(value)) + raise ParseError('Expected "true" or "false", not {0}.'.format(value)) if not isinstance(value, bool): raise ParseError('Expected true or false without quotes.') diff --git a/python/google/protobuf/message_factory.py b/python/google/protobuf/message_factory.py index 9cd9c2a8..1b059d13 100644 --- a/python/google/protobuf/message_factory.py +++ b/python/google/protobuf/message_factory.py @@ -39,7 +39,6 @@ my_proto_instance = message_classes['some.proto.package.MessageName']() __author__ = 'matthewtoia@google.com (Matt Toia)' -from google.protobuf import descriptor_database from google.protobuf import descriptor_pool from google.protobuf import message from google.protobuf import reflection @@ -50,8 +49,7 @@ class MessageFactory(object): def __init__(self, pool=None): """Initializes a new factory.""" - self.pool = (pool or descriptor_pool.DescriptorPool( - descriptor_database.DescriptorDatabase())) + self.pool = pool or descriptor_pool.DescriptorPool() # local cache of all classes built from protobuf descriptors self._classes = {} diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 61a3d237..a875a7be 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -202,6 +202,14 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { const Descriptor *message_type = options.GetDescriptor(); PyObject* message_class(cdescriptor_pool::GetMessageClass( pool, message_type)); + if (message_class == NULL) { + // The Options message was not found in the current DescriptorPool. + // In this case, there cannot be extensions to these options, and we can + // try to use the basic pool instead. + PyErr_Clear(); + message_class = cdescriptor_pool::GetMessageClass( + GetDefaultDescriptorPool(), message_type); + } if (message_class == NULL) { PyErr_Format(PyExc_TypeError, "Could not retrieve class for Options: %s", message_type->full_name().c_str()); @@ -211,6 +219,12 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { if (value == NULL) { return NULL; } + if (!PyObject_TypeCheck(value.get(), &CMessage_Type)) { + PyErr_Format(PyExc_TypeError, "Invalid class for %s: %s", + message_type->full_name().c_str(), + Py_TYPE(value.get())->tp_name); + return NULL; + } CMessage* cmsg = reinterpret_cast(value.get()); const Reflection* reflection = options.GetReflection(); @@ -327,7 +341,8 @@ PyObject* NewInternedDescriptor(PyTypeObject* type, PyDescriptorPool* pool = GetDescriptorPool_FromPool( GetFileDescriptor(descriptor)->pool()); if (pool == NULL) { - Py_DECREF(py_descriptor); + // Don't DECREF, the object is not fully initialized. + PyObject_Del(py_descriptor); return NULL; } Py_INCREF(pool); @@ -1213,6 +1228,13 @@ static void Dealloc(PyFileDescriptor* self) { descriptor::Dealloc(&self->base); } +static PyObject* GetPool(PyFileDescriptor *self, void *closure) { + PyObject* pool = reinterpret_cast( + GetDescriptorPool_FromPool(_GetDescriptor(self)->pool())); + Py_XINCREF(pool); + return pool; +} + static PyObject* GetName(PyFileDescriptor *self, void *closure) { return PyString_FromCppString(_GetDescriptor(self)->name()); } @@ -1292,6 +1314,7 @@ static PyObject* CopyToProto(PyFileDescriptor *self, PyObject *target) { } static PyGetSetDef Getters[] = { + { "pool", (getter)GetPool, NULL, "pool"}, { "name", (getter)GetName, NULL, "name"}, { "package", (getter)GetPackage, NULL, "package"}, { "serialized_pb", (getter)GetSerializedPb}, @@ -1354,8 +1377,8 @@ PyTypeObject PyFileDescriptor_Type = { 0, // tp_descr_set 0, // tp_dictoffset 0, // tp_init - PyType_GenericAlloc, // tp_alloc - PyType_GenericNew, // tp_new + 0, // tp_alloc + 0, // tp_new PyObject_Del, // tp_free }; diff --git a/python/google/protobuf/pyext/descriptor_database.cc b/python/google/protobuf/pyext/descriptor_database.cc new file mode 100644 index 00000000..514722b4 --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_database.cc @@ -0,0 +1,145 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines a C++ DescriptorDatabase, which wraps a Python Database +// and delegate all its operations to Python methods. + +#include + +#include +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace python { + +PyDescriptorDatabase::PyDescriptorDatabase(PyObject* py_database) + : py_database_(py_database) { + Py_INCREF(py_database_); +} + +PyDescriptorDatabase::~PyDescriptorDatabase() { Py_DECREF(py_database_); } + +// Convert a Python object to a FileDescriptorProto pointer. +// Handles all kinds of Python errors, which are simply logged. +static bool GetFileDescriptorProto(PyObject* py_descriptor, + FileDescriptorProto* output) { + if (py_descriptor == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + // Expected error: item was simply not found. + PyErr_Clear(); + } else { + GOOGLE_LOG(ERROR) << "DescriptorDatabase method raised an error"; + PyErr_Print(); + } + return false; + } + const Descriptor* filedescriptor_descriptor = + FileDescriptorProto::default_instance().GetDescriptor(); + CMessage* message = reinterpret_cast(py_descriptor); + if (PyObject_TypeCheck(py_descriptor, &CMessage_Type) && + message->message->GetDescriptor() == filedescriptor_descriptor) { + // Fast path: Just use the pointer. + FileDescriptorProto* file_proto = + static_cast(message->message); + *output = *file_proto; + return true; + } else { + // Slow path: serialize the message. This allows to use databases which + // use a different implementation of FileDescriptorProto. + ScopedPyObjectPtr serialized_pb( + PyObject_CallMethod(py_descriptor, "SerializeToString", NULL)); + if (serialized_pb == NULL) { + GOOGLE_LOG(ERROR) + << "DescriptorDatabase method did not return a FileDescriptorProto"; + PyErr_Print(); + return false; + } + char* str; + Py_ssize_t len; + if (PyBytes_AsStringAndSize(serialized_pb.get(), &str, &len) < 0) { + GOOGLE_LOG(ERROR) + << "DescriptorDatabase method did not return a FileDescriptorProto"; + PyErr_Print(); + return false; + } + FileDescriptorProto file_proto; + if (!file_proto.ParseFromArray(str, len)) { + GOOGLE_LOG(ERROR) + << "DescriptorDatabase method did not return a FileDescriptorProto"; + return false; + } + *output = file_proto; + return true; + } +} + +// Find a file by file name. +bool PyDescriptorDatabase::FindFileByName(const string& filename, + FileDescriptorProto* output) { + ScopedPyObjectPtr py_descriptor(PyObject_CallMethod( + py_database_, "FindFileByName", "s#", filename.c_str(), filename.size())); + return GetFileDescriptorProto(py_descriptor.get(), output); +} + +// Find the file that declares the given fully-qualified symbol name. +bool PyDescriptorDatabase::FindFileContainingSymbol( + const string& symbol_name, FileDescriptorProto* output) { + ScopedPyObjectPtr py_descriptor( + PyObject_CallMethod(py_database_, "FindFileContainingSymbol", "s#", + symbol_name.c_str(), symbol_name.size())); + return GetFileDescriptorProto(py_descriptor.get(), output); +} + +// Find the file which defines an extension extending the given message type +// with the given field number. +// Python DescriptorDatabases are not required to implement this method. +bool PyDescriptorDatabase::FindFileContainingExtension( + const string& containing_type, int field_number, + FileDescriptorProto* output) { + ScopedPyObjectPtr py_method( + PyObject_GetAttrString(py_database_, "FindFileContainingExtension")); + if (py_method == NULL) { + // This method is not implemented, returns without error. + PyErr_Clear(); + return false; + } + ScopedPyObjectPtr py_descriptor( + PyObject_CallFunction(py_method.get(), "s#i", containing_type.c_str(), + containing_type.size(), field_number)); + return GetFileDescriptorProto(py_descriptor.get(), output); +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/descriptor_database.h b/python/google/protobuf/pyext/descriptor_database.h new file mode 100644 index 00000000..fc71c4bc --- /dev/null +++ b/python/google/protobuf/pyext/descriptor_database.h @@ -0,0 +1,75 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ + +#include + +#include + +namespace google { +namespace protobuf { +namespace python { + +class PyDescriptorDatabase : public DescriptorDatabase { + public: + explicit PyDescriptorDatabase(PyObject* py_database); + ~PyDescriptorDatabase(); + + // Implement the abstract interface. All these functions fill the output + // with a copy of FileDescriptorProto. + + // Find a file by file name. + bool FindFileByName(const string& filename, + FileDescriptorProto* output); + + // Find the file that declares the given fully-qualified symbol name. + bool FindFileContainingSymbol(const string& symbol_name, + FileDescriptorProto* output); + + // Find the file which defines an extension extending the given message type + // with the given field number. + // Containing_type must be a fully-qualified type name. + // Python objects are not required to implement this method. + bool FindFileContainingExtension(const string& containing_type, + int field_number, + FileDescriptorProto* output); + + private: + // The python object that implements the database. The reference is owned. + PyObject* py_database_; +}; + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_DATABASE_H__ diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index 0f7487fa..0bc76bc9 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -34,8 +34,9 @@ #include #include -#include #include +#include +#include #include #include @@ -60,38 +61,93 @@ static hash_map descriptor_pool_map; namespace cdescriptor_pool { -static PyDescriptorPool* NewDescriptorPool() { - PyDescriptorPool* cdescriptor_pool = PyObject_New( +// Create a Python DescriptorPool object, but does not fill the "pool" +// attribute. +static PyDescriptorPool* _CreateDescriptorPool() { + PyDescriptorPool* cpool = PyObject_New( PyDescriptorPool, &PyDescriptorPool_Type); - if (cdescriptor_pool == NULL) { + if (cpool == NULL) { return NULL; } - // Build a DescriptorPool for messages only declared in Python libraries. - // generated_pool() contains all messages linked in C++ libraries, and is used - // as underlay. - cdescriptor_pool->pool = new DescriptorPool(DescriptorPool::generated_pool()); + cpool->underlay = NULL; + cpool->database = NULL; DynamicMessageFactory* message_factory = new DynamicMessageFactory(); // This option might be the default some day. message_factory->SetDelegateToGeneratedFactory(true); - cdescriptor_pool->message_factory = message_factory; + cpool->message_factory = message_factory; // TODO(amauryfa): Rewrite the SymbolDatabase in C so that it uses the same // storage. - cdescriptor_pool->classes_by_descriptor = + cpool->classes_by_descriptor = new PyDescriptorPool::ClassesByMessageMap(); - cdescriptor_pool->descriptor_options = + cpool->descriptor_options = new hash_map(); + return cpool; +} + +// Create a Python DescriptorPool, using the given pool as an underlay: +// new messages will be added to a custom pool, not to the underlay. +// +// Ownership of the underlay is not transferred, its pointer should +// stay alive. +static PyDescriptorPool* PyDescriptorPool_NewWithUnderlay( + const DescriptorPool* underlay) { + PyDescriptorPool* cpool = _CreateDescriptorPool(); + if (cpool == NULL) { + return NULL; + } + cpool->pool = new DescriptorPool(underlay); + cpool->underlay = underlay; + if (!descriptor_pool_map.insert( - std::make_pair(cdescriptor_pool->pool, cdescriptor_pool)).second) { + std::make_pair(cpool->pool, cpool)).second) { // Should never happen -- would indicate an internal error / bug. PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); return NULL; } - return cdescriptor_pool; + return cpool; +} + +static PyDescriptorPool* PyDescriptorPool_NewWithDatabase( + DescriptorDatabase* database) { + PyDescriptorPool* cpool = _CreateDescriptorPool(); + if (cpool == NULL) { + return NULL; + } + if (database != NULL) { + cpool->pool = new DescriptorPool(database); + cpool->database = database; + } else { + cpool->pool = new DescriptorPool(); + } + + if (!descriptor_pool_map.insert(std::make_pair(cpool->pool, cpool)).second) { + // Should never happen -- would indicate an internal error / bug. + PyErr_SetString(PyExc_ValueError, "DescriptorPool already registered"); + return NULL; + } + + return cpool; +} + +// The public DescriptorPool constructor. +static PyObject* New(PyTypeObject* type, + PyObject* args, PyObject* kwargs) { + static char* kwlist[] = {"descriptor_db", 0}; + PyObject* py_database = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &py_database)) { + return NULL; + } + DescriptorDatabase* database = NULL; + if (py_database && py_database != Py_None) { + database = new PyDescriptorDatabase(py_database); + } + return reinterpret_cast( + PyDescriptorPool_NewWithDatabase(database)); } static void Dealloc(PyDescriptorPool* self) { @@ -108,8 +164,9 @@ static void Dealloc(PyDescriptorPool* self) { Py_DECREF(it->second); } delete self->descriptor_options; - delete self->pool; delete self->message_factory; + delete self->database; + delete self->pool; Py_TYPE(self)->tp_free(reinterpret_cast(self)); } @@ -354,6 +411,14 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) { char* message_type; Py_ssize_t message_len; + if (self->database != NULL) { + PyErr_SetString( + PyExc_ValueError, + "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. " + "Add your file to the underlying database."); + return NULL; + } + if (PyBytes_AsStringAndSize(serialized_pb, &message_type, &message_len) < 0) { return NULL; } @@ -366,8 +431,10 @@ PyObject* AddSerializedFile(PyDescriptorPool* self, PyObject* serialized_pb) { // If the file was already part of a C++ library, all its descriptors are in // the underlying pool. No need to do anything else. - const FileDescriptor* generated_file = - DescriptorPool::generated_pool()->FindFileByName(file_proto.name()); + const FileDescriptor* generated_file = NULL; + if (self->underlay) { + generated_file = self->underlay->FindFileByName(file_proto.name()); + } if (generated_file != NULL) { return PyFileDescriptor_FromDescriptorWithSerializedPb( generated_file, serialized_pb); @@ -470,7 +537,7 @@ PyTypeObject PyDescriptorPool_Type = { 0, // tp_dictoffset 0, // tp_init 0, // tp_alloc - 0, // tp_new + cdescriptor_pool::New, // tp_new PyObject_Del, // tp_free }; @@ -482,7 +549,11 @@ bool InitDescriptorPool() { if (PyType_Ready(&PyDescriptorPool_Type) < 0) return false; - python_generated_pool = cdescriptor_pool::NewDescriptorPool(); + // The Pool of messages declared in Python libraries. + // generated_pool() contains all messages already linked in C++ libraries, and + // is used as underlay. + python_generated_pool = cdescriptor_pool::PyDescriptorPool_NewWithUnderlay( + DescriptorPool::generated_pool()); if (python_generated_pool == NULL) { return false; } @@ -494,6 +565,10 @@ bool InitDescriptorPool() { return true; } +// The default DescriptorPool used everywhere in this module. +// Today it's the python_generated_pool. +// TODO(amauryfa): Remove all usages of this function: the pool should be +// derived from the context. PyDescriptorPool* GetDefaultDescriptorPool() { return python_generated_pool; } diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index eda73d38..16bc910c 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -55,8 +55,17 @@ namespace python { typedef struct PyDescriptorPool { PyObject_HEAD + // The C++ pool containing Descriptors. DescriptorPool* pool; + // The C++ pool acting as an underlay. Can be NULL. + // This pointer is not owned and must stay alive. + const DescriptorPool* underlay; + + // The C++ descriptor database used to fetch unknown protos. Can be NULL. + // This pointer is owned. + const DescriptorDatabase* database; + // DynamicMessageFactory used to create C++ instances of messages. // This object cache the descriptors that were used, so the DescriptorPool // needs to get rid of it before it can delete itself. @@ -138,6 +147,7 @@ PyObject* FindOneofByName(PyDescriptorPool* self, PyObject* arg); // Retrieve the global descriptor pool owned by the _message module. // This is the one used by pb2.py generated modules. // Returns a *borrowed* reference. +// "Default" pool used to register messages from _pb2.py modules. PyDescriptorPool* GetDefaultDescriptorPool(); // Retrieve the python descriptor pool owning a C++ descriptor pool. diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index b361b342..555bd293 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -94,13 +94,13 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { if (descriptor == NULL) { return NULL; } - if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) { + if (!CheckFieldBelongsToMessage(descriptor, self->message)) { return NULL; } if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return cmessage::InternalGetScalar(self->parent->message, descriptor); + return cmessage::InternalGetScalar(self->message, descriptor); } PyObject* value = PyDict_GetItem(self->values, key); @@ -109,6 +109,14 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { return value; } + if (self->parent == NULL) { + // We are in "detached" state. Don't allow further modifications. + // TODO(amauryfa): Support adding non-scalars to a detached extension dict. + // This probably requires to store the type of the main message. + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } + if (descriptor->label() != FieldDescriptor::LABEL_REPEATED && descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { PyObject* sub_message = cmessage::InternalGetSubMessage( @@ -154,7 +162,7 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { if (descriptor == NULL) { return -1; } - if (!CheckFieldBelongsToMessage(descriptor, self->parent->message)) { + if (!CheckFieldBelongsToMessage(descriptor, self->message)) { return -1; } @@ -164,9 +172,11 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) { "type"); return -1; } - cmessage::AssureWritable(self->parent); - if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { - return -1; + if (self->parent) { + cmessage::AssureWritable(self->parent); + if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) { + return -1; + } } // TODO(tibell): We shouldn't write scalars to the cache. PyDict_SetItem(self->values, key, value); @@ -180,15 +190,17 @@ PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) { return NULL; } PyObject* value = PyDict_GetItem(self->values, extension); - if (value != NULL) { - if (ReleaseExtension(self, value, descriptor) < 0) { + if (self->parent) { + if (value != NULL) { + if (ReleaseExtension(self, value, descriptor) < 0) { + return NULL; + } + } + if (ScopedPyObjectPtr(cmessage::ClearFieldByDescriptor( + self->parent, descriptor)) == NULL) { return NULL; } } - if (ScopedPyObjectPtr(cmessage::ClearFieldByDescriptor( - self->parent, descriptor)) == NULL) { - return NULL; - } if (PyDict_DelItem(self->values, extension) < 0) { PyErr_Clear(); } @@ -201,8 +213,15 @@ PyObject* HasExtension(ExtensionDict* self, PyObject* extension) { if (descriptor == NULL) { return NULL; } - PyObject* result = cmessage::HasFieldByDescriptor(self->parent, descriptor); - return result; + if (self->parent) { + return cmessage::HasFieldByDescriptor(self->parent, descriptor); + } else { + int exists = PyDict_Contains(self->values, extension); + if (exists < 0) { + return NULL; + } + return PyBool_FromLong(exists); + } } PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) { diff --git a/python/google/protobuf/pyext/extension_dict.h b/python/google/protobuf/pyext/extension_dict.h index 7a66cb23..d92cf956 100644 --- a/python/google/protobuf/pyext/extension_dict.h +++ b/python/google/protobuf/pyext/extension_dict.h @@ -117,11 +117,6 @@ int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value); PyObject* ClearExtension(ExtensionDict* self, PyObject* extension); -// Checks if the dict has an extension. -// -// Returns a new python boolean reference. -PyObject* HasExtension(ExtensionDict* self, PyObject* extension); - // Gets an extension from the dict given the extension name as opposed to // descriptor. // diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc new file mode 100644 index 00000000..c39f7b83 --- /dev/null +++ b/python/google/protobuf/pyext/map_container.cc @@ -0,0 +1,912 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: haberman@google.com (Josh Haberman) + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t +#endif + +namespace google { +namespace protobuf { +namespace python { + +// Functions that need access to map reflection functionality. +// They need to be contained in this class because it is friended. +class MapReflectionFriend { + public: + // Methods that are in common between the map types. + static PyObject* Contains(PyObject* _self, PyObject* key); + static Py_ssize_t Length(PyObject* _self); + static PyObject* GetIterator(PyObject *_self); + static PyObject* IterNext(PyObject* _self); + + // Methods that differ between the map types. + static PyObject* ScalarMapGetItem(PyObject* _self, PyObject* key); + static PyObject* MessageMapGetItem(PyObject* _self, PyObject* key); + static int ScalarMapSetItem(PyObject* _self, PyObject* key, PyObject* v); + static int MessageMapSetItem(PyObject* _self, PyObject* key, PyObject* v); +}; + +struct MapIterator { + PyObject_HEAD; + + scoped_ptr< ::google::protobuf::MapIterator> iter; + + // A pointer back to the container, so we can notice changes to the version. + // We own a ref on this. + MapContainer* container; + + // We need to keep a ref on the Message* too, because + // MapIterator::~MapIterator() accesses it. Normally this would be ok because + // the ref on container (above) would guarantee outlive semantics. However in + // the case of ClearField(), InitializeAndCopyToParentContainer() resets the + // message pointer (and the owner) to a different message, a copy of the + // original. But our iterator still points to the original, which could now + // get deleted before us. + // + // To prevent this, we ensure that the Message will always stay alive as long + // as this iterator does. This is solely for the benefit of the MapIterator + // destructor -- we should never actually access the iterator in this state + // except to delete it. + shared_ptr owner; + + // The version of the map when we took the iterator to it. + // + // We store this so that if the map is modified during iteration we can throw + // an error. + uint64 version; + + // True if the container is empty. We signal this separately to avoid calling + // any of the iteration methods, which are non-const. + bool empty; +}; + +Message* MapContainer::GetMutableMessage() { + cmessage::AssureWritable(parent); + return const_cast(message); +} + +// Consumes a reference on the Python string object. +static bool PyStringToSTL(PyObject* py_string, string* stl_string) { + char *value; + Py_ssize_t value_len; + + if (!py_string) { + return false; + } + if (PyBytes_AsStringAndSize(py_string, &value, &value_len) < 0) { + Py_DECREF(py_string); + return false; + } else { + stl_string->assign(value, value_len); + Py_DECREF(py_string); + return true; + } +} + +static bool PythonToMapKey(PyObject* obj, + const FieldDescriptor* field_descriptor, + MapKey* key) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(obj, value, false); + key->SetInt32Value(value); + break; + } + case FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(obj, value, false); + key->SetInt64Value(value); + break; + } + case FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(obj, value, false); + key->SetUInt32Value(value); + break; + } + case FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(obj, value, false); + key->SetUInt64Value(value); + break; + } + case FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(obj, value, false); + key->SetBoolValue(value); + break; + } + case FieldDescriptor::CPPTYPE_STRING: { + string str; + if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) { + return false; + } + key->SetStringValue(str); + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Type %d cannot be a map key", + field_descriptor->cpp_type()); + return false; + } + return true; +} + +static PyObject* MapKeyToPython(const FieldDescriptor* field_descriptor, + const MapKey& key) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return PyInt_FromLong(key.GetInt32Value()); + case FieldDescriptor::CPPTYPE_INT64: + return PyLong_FromLongLong(key.GetInt64Value()); + case FieldDescriptor::CPPTYPE_UINT32: + return PyInt_FromSize_t(key.GetUInt32Value()); + case FieldDescriptor::CPPTYPE_UINT64: + return PyLong_FromUnsignedLongLong(key.GetUInt64Value()); + case FieldDescriptor::CPPTYPE_BOOL: + return PyBool_FromLong(key.GetBoolValue()); + case FieldDescriptor::CPPTYPE_STRING: + return ToStringObject(field_descriptor, key.GetStringValue()); + default: + PyErr_Format( + PyExc_SystemError, "Couldn't convert type %d to value", + field_descriptor->cpp_type()); + return NULL; + } +} + +// This is only used for ScalarMap, so we don't need to handle the +// CPPTYPE_MESSAGE case. +PyObject* MapValueRefToPython(const FieldDescriptor* field_descriptor, + MapValueRef* value) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return PyInt_FromLong(value->GetInt32Value()); + case FieldDescriptor::CPPTYPE_INT64: + return PyLong_FromLongLong(value->GetInt64Value()); + case FieldDescriptor::CPPTYPE_UINT32: + return PyInt_FromSize_t(value->GetUInt32Value()); + case FieldDescriptor::CPPTYPE_UINT64: + return PyLong_FromUnsignedLongLong(value->GetUInt64Value()); + case FieldDescriptor::CPPTYPE_FLOAT: + return PyFloat_FromDouble(value->GetFloatValue()); + case FieldDescriptor::CPPTYPE_DOUBLE: + return PyFloat_FromDouble(value->GetDoubleValue()); + case FieldDescriptor::CPPTYPE_BOOL: + return PyBool_FromLong(value->GetBoolValue()); + case FieldDescriptor::CPPTYPE_STRING: + return ToStringObject(field_descriptor, value->GetStringValue()); + case FieldDescriptor::CPPTYPE_ENUM: + return PyInt_FromLong(value->GetEnumValue()); + default: + PyErr_Format( + PyExc_SystemError, "Couldn't convert type %d to value", + field_descriptor->cpp_type()); + return NULL; + } +} + +// This is only used for ScalarMap, so we don't need to handle the +// CPPTYPE_MESSAGE case. +static bool PythonToMapValueRef(PyObject* obj, + const FieldDescriptor* field_descriptor, + bool allow_unknown_enum_values, + MapValueRef* value_ref) { + switch (field_descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: { + GOOGLE_CHECK_GET_INT32(obj, value, false); + value_ref->SetInt32Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_INT64: { + GOOGLE_CHECK_GET_INT64(obj, value, false); + value_ref->SetInt64Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_UINT32: { + GOOGLE_CHECK_GET_UINT32(obj, value, false); + value_ref->SetUInt32Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_UINT64: { + GOOGLE_CHECK_GET_UINT64(obj, value, false); + value_ref->SetUInt64Value(value); + return true; + } + case FieldDescriptor::CPPTYPE_FLOAT: { + GOOGLE_CHECK_GET_FLOAT(obj, value, false); + value_ref->SetFloatValue(value); + return true; + } + case FieldDescriptor::CPPTYPE_DOUBLE: { + GOOGLE_CHECK_GET_DOUBLE(obj, value, false); + value_ref->SetDoubleValue(value); + return true; + } + case FieldDescriptor::CPPTYPE_BOOL: { + GOOGLE_CHECK_GET_BOOL(obj, value, false); + value_ref->SetBoolValue(value); + return true;; + } + case FieldDescriptor::CPPTYPE_STRING: { + string str; + if (!PyStringToSTL(CheckString(obj, field_descriptor), &str)) { + return false; + } + value_ref->SetStringValue(str); + return true; + } + case FieldDescriptor::CPPTYPE_ENUM: { + GOOGLE_CHECK_GET_INT32(obj, value, false); + if (allow_unknown_enum_values) { + value_ref->SetEnumValue(value); + return true; + } else { + const EnumDescriptor* enum_descriptor = field_descriptor->enum_type(); + const EnumValueDescriptor* enum_value = + enum_descriptor->FindValueByNumber(value); + if (enum_value != NULL) { + value_ref->SetEnumValue(value); + return true; + } else { + PyErr_Format(PyExc_ValueError, "Unknown enum value: %d", value); + return false; + } + } + break; + } + default: + PyErr_Format( + PyExc_SystemError, "Setting value to a field of unknown type %d", + field_descriptor->cpp_type()); + return false; + } +} + +// Map methods common to ScalarMap and MessageMap ////////////////////////////// + +static MapContainer* GetMap(PyObject* obj) { + return reinterpret_cast(obj); +} + +Py_ssize_t MapReflectionFriend::Length(PyObject* _self) { + MapContainer* self = GetMap(_self); + const google::protobuf::Message* message = self->message; + return message->GetReflection()->MapSize(*message, + self->parent_field_descriptor); +} + +PyObject* Clear(PyObject* _self) { + MapContainer* self = GetMap(_self); + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + + reflection->ClearField(message, self->parent_field_descriptor); + + Py_RETURN_NONE; +} + +PyObject* MapReflectionFriend::Contains(PyObject* _self, PyObject* key) { + MapContainer* self = GetMap(_self); + + const Message* message = self->message; + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return NULL; + } + + if (reflection->ContainsMapKey(*message, self->parent_field_descriptor, + map_key)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +// Initializes the underlying Message object of "to" so it becomes a new parent +// repeated scalar, and copies all the values from "from" to it. A child scalar +// container can be released by passing it as both from and to (e.g. making it +// the recipient of the new parent message and copying the values from itself). +static int InitializeAndCopyToParentContainer(MapContainer* from, + MapContainer* to) { + // For now we require from == to, re-evaluate if we want to support deep copy + // as in repeated_scalar_container.cc. + GOOGLE_DCHECK(from == to); + Message* new_message = from->message->New(); + + if (MapReflectionFriend::Length(reinterpret_cast(from)) > 0) { + // A somewhat roundabout way of copying just one field from old_message to + // new_message. This is the best we can do with what Reflection gives us. + Message* mutable_old = from->GetMutableMessage(); + vector fields; + fields.push_back(from->parent_field_descriptor); + + // Move the map field into the new message. + mutable_old->GetReflection()->SwapFields(mutable_old, new_message, fields); + + // If/when we support from != to, this will be required also to copy the + // map field back into the existing message: + // mutable_old->MergeFrom(*new_message); + } + + // If from == to this could delete old_message. + to->owner.reset(new_message); + + to->parent = NULL; + to->parent_field_descriptor = from->parent_field_descriptor; + to->message = new_message; + + // Invalidate iterators, since they point to the old copy of the field. + to->version++; + + return 0; +} + +int MapContainer::Release() { + return InitializeAndCopyToParentContainer(this, this); +} + + +// ScalarMap /////////////////////////////////////////////////////////////////// + +PyObject *NewScalarMapContainer( + CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { + if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { + return NULL; + } + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0)); + if (obj.get() == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate new container."); + } + + MapContainer* self = GetMap(obj.get()); + + self->message = parent->message; + self->parent = parent; + self->parent_field_descriptor = parent_field_descriptor; + self->owner = parent->owner; + self->version = 0; + + self->key_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("key"); + self->value_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("value"); + + if (self->key_field_descriptor == NULL || + self->value_field_descriptor == NULL) { + return PyErr_Format(PyExc_KeyError, + "Map entry descriptor did not have key/value fields"); + } + + return obj.release(); +} + +PyObject* MapReflectionFriend::ScalarMapGetItem(PyObject* _self, + PyObject* key) { + MapContainer* self = GetMap(_self); + + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return NULL; + } + + if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value)) { + self->version++; + } + + return MapValueRefToPython(self->value_field_descriptor, &value); +} + +int MapReflectionFriend::ScalarMapSetItem(PyObject* _self, PyObject* key, + PyObject* v) { + MapContainer* self = GetMap(_self); + + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return -1; + } + + self->version++; + + if (v) { + // Set item to v. + reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value); + + return PythonToMapValueRef(v, self->value_field_descriptor, + reflection->SupportsUnknownEnumValues(), &value) + ? 0 + : -1; + } else { + // Delete key from map. + if (reflection->DeleteMapValue(message, self->parent_field_descriptor, + map_key)) { + return 0; + } else { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } + } +} + +static PyObject* ScalarMapGet(PyObject* self, PyObject* args) { + PyObject* key; + PyObject* default_value = NULL; + if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { + return NULL; + } + + ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key)); + if (is_present.get() == NULL) { + return NULL; + } + + if (PyObject_IsTrue(is_present.get())) { + return MapReflectionFriend::ScalarMapGetItem(self, key); + } else { + if (default_value != NULL) { + Py_INCREF(default_value); + return default_value; + } else { + Py_RETURN_NONE; + } + } +} + +static void ScalarMapDealloc(PyObject* _self) { + MapContainer* self = GetMap(_self); + self->owner.reset(); + Py_TYPE(_self)->tp_free(_self); +} + +static PyMappingMethods ScalarMapMappingMethods = { + MapReflectionFriend::Length, // mp_length + MapReflectionFriend::ScalarMapGetItem, // mp_subscript + MapReflectionFriend::ScalarMapSetItem, // mp_ass_subscript +}; + +static PyMethodDef ScalarMapMethods[] = { + { "__contains__", MapReflectionFriend::Contains, METH_O, + "Tests whether a key is a member of the map." }, + { "clear", (PyCFunction)Clear, METH_NOARGS, + "Removes all elements from the map." }, + { "get", ScalarMapGet, METH_VARARGS, + "Gets the value for the given key if present, or otherwise a default" }, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +PyTypeObject ScalarMapContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".ScalarMapContainer", // tp_name + sizeof(MapContainer), // tp_basicsize + 0, // tp_itemsize + ScalarMapDealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &ScalarMapMappingMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map container", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + MapReflectionFriend::GetIterator, // tp_iter + 0, // tp_iternext + ScalarMapMethods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + + +// MessageMap ////////////////////////////////////////////////////////////////// + +static MessageMapContainer* GetMessageMap(PyObject* obj) { + return reinterpret_cast(obj); +} + +static PyObject* GetCMessage(MessageMapContainer* self, Message* message) { + // Get or create the CMessage object corresponding to this message. + ScopedPyObjectPtr key(PyLong_FromVoidPtr(message)); + PyObject* ret = PyDict_GetItem(self->message_dict, key.get()); + + if (ret == NULL) { + CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, + message->GetDescriptor()); + ret = reinterpret_cast(cmsg); + + if (cmsg == NULL) { + return NULL; + } + cmsg->owner = self->owner; + cmsg->message = message; + cmsg->parent = self->parent; + + if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) { + Py_DECREF(ret); + return NULL; + } + } else { + Py_INCREF(ret); + } + + return ret; +} + +PyObject* NewMessageMapContainer( + CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor, + PyObject* concrete_class) { + if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { + return NULL; + } + + PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0); + if (obj == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate new container."); + } + + MessageMapContainer* self = GetMessageMap(obj); + + self->message = parent->message; + self->parent = parent; + self->parent_field_descriptor = parent_field_descriptor; + self->owner = parent->owner; + self->version = 0; + + self->key_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("key"); + self->value_field_descriptor = + parent_field_descriptor->message_type()->FindFieldByName("value"); + + self->message_dict = PyDict_New(); + if (self->message_dict == NULL) { + return PyErr_Format(PyExc_RuntimeError, + "Could not allocate message dict."); + } + + Py_INCREF(concrete_class); + self->subclass_init = concrete_class; + + if (self->key_field_descriptor == NULL || + self->value_field_descriptor == NULL) { + Py_DECREF(obj); + return PyErr_Format(PyExc_KeyError, + "Map entry descriptor did not have key/value fields"); + } + + return obj; +} + +int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key, + PyObject* v) { + if (v) { + PyErr_Format(PyExc_ValueError, + "Direct assignment of submessage not allowed"); + return -1; + } + + // Now we know that this is a delete, not a set. + + MessageMapContainer* self = GetMessageMap(_self); + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + self->version++; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return -1; + } + + // Delete key from map. + if (reflection->DeleteMapValue(message, self->parent_field_descriptor, + map_key)) { + return 0; + } else { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } +} + +PyObject* MapReflectionFriend::MessageMapGetItem(PyObject* _self, + PyObject* key) { + MessageMapContainer* self = GetMessageMap(_self); + + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + MapKey map_key; + MapValueRef value; + + if (!PythonToMapKey(key, self->key_field_descriptor, &map_key)) { + return NULL; + } + + if (reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value)) { + self->version++; + } + + return GetCMessage(self, value.MutableMessageValue()); +} + +PyObject* MessageMapGet(PyObject* self, PyObject* args) { + PyObject* key; + PyObject* default_value = NULL; + if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { + return NULL; + } + + ScopedPyObjectPtr is_present(MapReflectionFriend::Contains(self, key)); + if (is_present.get() == NULL) { + return NULL; + } + + if (PyObject_IsTrue(is_present.get())) { + return MapReflectionFriend::MessageMapGetItem(self, key); + } else { + if (default_value != NULL) { + Py_INCREF(default_value); + return default_value; + } else { + Py_RETURN_NONE; + } + } +} + +static void MessageMapDealloc(PyObject* _self) { + MessageMapContainer* self = GetMessageMap(_self); + self->owner.reset(); + Py_DECREF(self->message_dict); + Py_TYPE(_self)->tp_free(_self); +} + +static PyMappingMethods MessageMapMappingMethods = { + MapReflectionFriend::Length, // mp_length + MapReflectionFriend::MessageMapGetItem, // mp_subscript + MapReflectionFriend::MessageMapSetItem, // mp_ass_subscript +}; + +static PyMethodDef MessageMapMethods[] = { + { "__contains__", (PyCFunction)MapReflectionFriend::Contains, METH_O, + "Tests whether the map contains this element."}, + { "clear", (PyCFunction)Clear, METH_NOARGS, + "Removes all elements from the map."}, + { "get", MessageMapGet, METH_VARARGS, + "Gets the value for the given key if present, or otherwise a default" }, + { "get_or_create", MapReflectionFriend::MessageMapGetItem, METH_O, + "Alias for getitem, useful to make explicit that the map is mutated." }, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +PyTypeObject MessageMapContainer_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".MessageMapContainer", // tp_name + sizeof(MessageMapContainer), // tp_basicsize + 0, // tp_itemsize + MessageMapDealloc, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + &MessageMapMappingMethods, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A map container for message", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + MapReflectionFriend::GetIterator, // tp_iter + 0, // tp_iternext + MessageMapMethods, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +// MapIterator ///////////////////////////////////////////////////////////////// + +static MapIterator* GetIter(PyObject* obj) { + return reinterpret_cast(obj); +} + +PyObject* MapReflectionFriend::GetIterator(PyObject *_self) { + MapContainer* self = GetMap(_self); + + ScopedPyObjectPtr obj(PyType_GenericAlloc(&MapIterator_Type, 0)); + if (obj == NULL) { + return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); + } + + MapIterator* iter = GetIter(obj.get()); + + Py_INCREF(self); + iter->container = self; + iter->version = self->version; + iter->owner = self->owner; + + if (MapReflectionFriend::Length(_self) > 0) { + Message* message = self->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + + iter->iter.reset(new ::google::protobuf::MapIterator( + reflection->MapBegin(message, self->parent_field_descriptor))); + } + + return obj.release(); +} + +PyObject* MapReflectionFriend::IterNext(PyObject* _self) { + MapIterator* self = GetIter(_self); + + // This won't catch mutations to the map performed by MergeFrom(); no easy way + // to address that. + if (self->version != self->container->version) { + return PyErr_Format(PyExc_RuntimeError, + "Map modified during iteration."); + } + + if (self->iter.get() == NULL) { + return NULL; + } + + Message* message = self->container->GetMutableMessage(); + const Reflection* reflection = message->GetReflection(); + + if (*self->iter == + reflection->MapEnd(message, self->container->parent_field_descriptor)) { + return NULL; + } + + PyObject* ret = MapKeyToPython(self->container->key_field_descriptor, + self->iter->GetKey()); + + ++(*self->iter); + + return ret; +} + +static void DeallocMapIterator(PyObject* _self) { + MapIterator* self = GetIter(_self); + self->iter.reset(); + self->owner.reset(); + Py_XDECREF(self->container); + Py_TYPE(_self)->tp_free(_self); +} + +PyTypeObject MapIterator_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + FULL_MODULE_NAME ".MapIterator", // tp_name + sizeof(MapIterator), // tp_basicsize + 0, // tp_itemsize + DeallocMapIterator, // tp_dealloc + 0, // tp_print + 0, // tp_getattr + 0, // tp_setattr + 0, // tp_compare + 0, // tp_repr + 0, // tp_as_number + 0, // tp_as_sequence + 0, // tp_as_mapping + 0, // tp_hash + 0, // tp_call + 0, // tp_str + 0, // tp_getattro + 0, // tp_setattro + 0, // tp_as_buffer + Py_TPFLAGS_DEFAULT, // tp_flags + "A scalar map iterator", // tp_doc + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + PyObject_SelfIter, // tp_iter + MapReflectionFriend::IterNext, // tp_iternext + 0, // tp_methods + 0, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset + 0, // tp_init +}; + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/pyext/map_container.h b/python/google/protobuf/pyext/map_container.h new file mode 100644 index 00000000..2de61187 --- /dev/null +++ b/python/google/protobuf/pyext/map_container.h @@ -0,0 +1,133 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ +#define GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ + +#include + +#include +#ifndef _SHARED_PTR_H +#include +#endif + +#include +#include + +namespace google { +namespace protobuf { + +class Message; + +#ifdef _SHARED_PTR_H +using std::shared_ptr; +#else +using internal::shared_ptr; +#endif + +namespace python { + +struct CMessage; + +// This struct is used directly for ScalarMap, and is the base class of +// MessageMapContainer, which is used for MessageMap. +struct MapContainer { + PyObject_HEAD; + + // This is the top-level C++ Message object that owns the whole + // proto tree. Every Python MapContainer holds a + // reference to it in order to keep it alive as long as there's a + // Python object that references any part of the tree. + shared_ptr owner; + + // Pointer to the C++ Message that contains this container. The + // MapContainer does not own this pointer. + const Message* message; + + // Use to get a mutable message when necessary. + Message* GetMutableMessage(); + + // Weak reference to a parent CMessage object (i.e. may be NULL.) + // + // Used to make sure all ancestors are also mutable when first + // modifying the container. + CMessage* parent; + + // Pointer to the parent's descriptor that describes this + // field. Used together with the parent's message when making a + // default message instance mutable. + // The pointer is owned by the global DescriptorPool. + const FieldDescriptor* parent_field_descriptor; + const FieldDescriptor* key_field_descriptor; + const FieldDescriptor* value_field_descriptor; + + // We bump this whenever we perform a mutation, to invalidate existing + // iterators. + uint64 version; + + // Releases the messages in the container to a new message. + // + // Returns 0 on success, -1 on failure. + int Release(); + + // Set the owner field of self and any children of self. + void SetOwner(const shared_ptr& new_owner) { + owner = new_owner; + } +}; + +struct MessageMapContainer : public MapContainer { + // A callable that is used to create new child messages. + PyObject* subclass_init; + + // A dict mapping Message* -> CMessage. + PyObject* message_dict; +}; + +extern PyTypeObject ScalarMapContainer_Type; +extern PyTypeObject MessageMapContainer_Type; +extern PyTypeObject MapIterator_Type; // Both map types use the same iterator. + +// Builds a MapContainer object, from a parent message and a +// field descriptor. +extern PyObject* NewScalarMapContainer( + CMessage* parent, const FieldDescriptor* parent_field_descriptor); + +// Builds a MessageMap object, from a parent message and a +// field descriptor. +extern PyObject* NewMessageMapContainer( + CMessage* parent, const FieldDescriptor* parent_field_descriptor, + PyObject* concrete_class); + +} // namespace python +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MAP_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 72f51ec1..863cde01 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -33,6 +33,7 @@ #include +#include #include #ifndef _SHARED_PTR_H #include @@ -61,8 +62,7 @@ #include #include #include -#include -#include +#include #include #include @@ -96,6 +96,7 @@ static PyObject* k_extensions_by_number; PyObject* EnumTypeWrapper_class; static PyObject* PythonMessage_class; static PyObject* kEmptyWeakref; +static PyObject* WKT_classes = NULL; // Defines the Metaclass of all Message classes. // It allows us to cache some C++ pointers in the class object itself, they are @@ -274,8 +275,32 @@ static PyObject* New(PyTypeObject* type, // Build the arguments to the base metaclass. // We change the __bases__ classes. - ScopedPyObjectPtr new_args(Py_BuildValue( - "s(OO)O", name, &CMessage_Type, PythonMessage_class, dict)); + ScopedPyObjectPtr new_args; + const Descriptor* message_descriptor = + PyMessageDescriptor_AsDescriptor(py_descriptor); + if (message_descriptor == NULL) { + return NULL; + } + + if (WKT_classes == NULL) { + ScopedPyObjectPtr well_known_types(PyImport_ImportModule( + "google.protobuf.internal.well_known_types")); + GOOGLE_DCHECK(well_known_types != NULL); + + WKT_classes = PyObject_GetAttrString(well_known_types.get(), "WKTBASES"); + GOOGLE_DCHECK(WKT_classes != NULL); + } + + PyObject* well_known_class = PyDict_GetItemString( + WKT_classes, message_descriptor->full_name().c_str()); + if (well_known_class == NULL) { + new_args.reset(Py_BuildValue("s(OO)O", name, &CMessage_Type, + PythonMessage_class, dict)); + } else { + new_args.reset(Py_BuildValue("s(OOO)O", name, &CMessage_Type, + PythonMessage_class, well_known_class, dict)); + } + if (new_args == NULL) { return NULL; } @@ -448,21 +473,9 @@ static int VisitCompositeField(const FieldDescriptor* descriptor, if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) { if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { if (descriptor->is_map()) { - const Descriptor* entry_type = descriptor->message_type(); - const FieldDescriptor* value_type = - entry_type->FindFieldByName("value"); - if (value_type->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - MessageMapContainer* container = - reinterpret_cast(child); - if (visitor.VisitMessageMapContainer(container) == -1) { - return -1; - } - } else { - ScalarMapContainer* container = - reinterpret_cast(child); - if (visitor.VisitScalarMapContainer(container) == -1) { - return -1; - } + MapContainer* container = reinterpret_cast(child); + if (visitor.VisitMapContainer(container) == -1) { + return -1; } } else { RepeatedCompositeContainer* container = @@ -579,12 +592,14 @@ bool CheckAndGetInteger( if (PyObject_RichCompareBool(min, arg, Py_LE) != 1 || PyObject_RichCompareBool(max, arg, Py_GE) != 1) { #endif - PyObject *s = PyObject_Str(arg); - if (s) { - PyErr_Format(PyExc_ValueError, - "Value out of range: %s", - PyString_AsString(s)); - Py_DECREF(s); + if (!PyErr_Occurred()) { + PyObject *s = PyObject_Str(arg); + if (s) { + PyErr_Format(PyExc_ValueError, + "Value out of range: %s", + PyString_AsString(s)); + Py_DECREF(s); + } } return false; } @@ -642,38 +657,51 @@ bool CheckAndGetBool(PyObject* arg, bool* value) { return true; } -bool CheckAndSetString( - PyObject* arg, Message* message, - const FieldDescriptor* descriptor, - const Reflection* reflection, - bool append, - int index) { +// Checks whether the given object (which must be "bytes" or "unicode") contains +// valid UTF-8. +bool IsValidUTF8(PyObject* obj) { + if (PyBytes_Check(obj)) { + PyObject* unicode = PyUnicode_FromEncodedObject(obj, "utf-8", NULL); + + // Clear the error indicator; we report our own error when desired. + PyErr_Clear(); + + if (unicode) { + Py_DECREF(unicode); + return true; + } else { + return false; + } + } else { + // Unicode object, known to be valid UTF-8. + return true; + } +} + +bool AllowInvalidUTF8(const FieldDescriptor* field) { return false; } + +PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor) { GOOGLE_DCHECK(descriptor->type() == FieldDescriptor::TYPE_STRING || descriptor->type() == FieldDescriptor::TYPE_BYTES); if (descriptor->type() == FieldDescriptor::TYPE_STRING) { if (!PyBytes_Check(arg) && !PyUnicode_Check(arg)) { FormatTypeError(arg, "bytes, unicode"); - return false; + return NULL; } - if (PyBytes_Check(arg)) { - PyObject* unicode = PyUnicode_FromEncodedObject(arg, "utf-8", NULL); - if (unicode == NULL) { - PyObject* repr = PyObject_Repr(arg); - PyErr_Format(PyExc_ValueError, - "%s has type str, but isn't valid UTF-8 " - "encoding. Non-UTF-8 strings must be converted to " - "unicode objects before being added.", - PyString_AsString(repr)); - Py_DECREF(repr); - return false; - } else { - Py_DECREF(unicode); - } + if (!IsValidUTF8(arg) && !AllowInvalidUTF8(descriptor)) { + PyObject* repr = PyObject_Repr(arg); + PyErr_Format(PyExc_ValueError, + "%s has type str, but isn't valid UTF-8 " + "encoding. Non-UTF-8 strings must be converted to " + "unicode objects before being added.", + PyString_AsString(repr)); + Py_DECREF(repr); + return NULL; } } else if (!PyBytes_Check(arg)) { FormatTypeError(arg, "bytes"); - return false; + return NULL; } PyObject* encoded_string = NULL; @@ -691,14 +719,24 @@ bool CheckAndSetString( Py_INCREF(encoded_string); } - if (encoded_string == NULL) { + return encoded_string; +} + +bool CheckAndSetString( + PyObject* arg, Message* message, + const FieldDescriptor* descriptor, + const Reflection* reflection, + bool append, + int index) { + ScopedPyObjectPtr encoded_string(CheckString(arg, descriptor)); + + if (encoded_string.get() == NULL) { return false; } char* value; Py_ssize_t value_len; - if (PyBytes_AsStringAndSize(encoded_string, &value, &value_len) < 0) { - Py_DECREF(encoded_string); + if (PyBytes_AsStringAndSize(encoded_string.get(), &value, &value_len) < 0) { return false; } @@ -710,7 +748,6 @@ bool CheckAndSetString( } else { reflection->SetRepeatedString(message, descriptor, index, value_string); } - Py_DECREF(encoded_string); return true; } @@ -823,12 +860,7 @@ struct FixupMessageReference : public ChildVisitor { return 0; } - int VisitScalarMapContainer(ScalarMapContainer* container) { - container->message = message_; - return 0; - } - - int VisitMessageMapContainer(MessageMapContainer* container) { + int VisitMapContainer(MapContainer* container) { container->message = message_; return 0; } @@ -870,9 +902,8 @@ int AssureWritable(CMessage* self) { // When a CMessage is made writable its Message pointer is updated // to point to a new mutable Message. When that happens we need to // update any references to the old, read-only CMessage. There are - // five places such references occur: RepeatedScalarContainer, - // RepeatedCompositeContainer, ScalarMapContainer, MessageMapContainer, - // and ExtensionDict. + // four places such references occur: RepeatedScalarContainer, + // RepeatedCompositeContainer, MapContainer, and ExtensionDict. if (self->extensions != NULL) self->extensions->message = self->message; if (ForEachCompositeField(self, FixupMessageReference(self->message)) == -1) @@ -1054,7 +1085,8 @@ int InitAttributes(CMessage* self, PyObject* kwargs) { } const FieldDescriptor* descriptor = GetFieldDescriptor(self, name); if (descriptor == NULL) { - PyErr_Format(PyExc_ValueError, "Protocol message has no \"%s\" field.", + PyErr_Format(PyExc_ValueError, "Protocol message %s has no \"%s\" field.", + self->message->GetDescriptor()->name().c_str(), PyString_AsString(name)); return -1; } @@ -1203,18 +1235,6 @@ CMessage* NewEmptyMessage(PyObject* type, const Descriptor *descriptor) { self->composite_fields = NULL; - // If there are extension_ranges, the message is "extendable". Allocate a - // dictionary to store the extension fields. - if (descriptor->extension_range_count() > 0) { - // TODO(amauryfa): Delay the construction of this dict until extensions are - // really used on the object. - ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); - if (extension_dict == NULL) { - return NULL; - } - self->extensions = extension_dict; - } - return self; } @@ -1285,12 +1305,7 @@ struct ClearWeakReferences : public ChildVisitor { return 0; } - int VisitScalarMapContainer(ScalarMapContainer* container) { - container->parent = NULL; - return 0; - } - - int VisitMessageMapContainer(MessageMapContainer* container) { + int VisitMapContainer(MapContainer* container) { container->parent = NULL; return 0; } @@ -1305,6 +1320,9 @@ struct ClearWeakReferences : public ChildVisitor { static void Dealloc(CMessage* self) { // Null out all weak references from children to this message. GOOGLE_CHECK_EQ(0, ForEachCompositeField(self, ClearWeakReferences())); + if (self->extensions) { + self->extensions->parent = NULL; + } Py_CLEAR(self->extensions); Py_CLEAR(self->composite_fields); @@ -1466,20 +1484,27 @@ PyObject* HasField(CMessage* self, PyObject* arg) { Py_RETURN_FALSE; } -PyObject* ClearExtension(CMessage* self, PyObject* arg) { +PyObject* ClearExtension(CMessage* self, PyObject* extension) { if (self->extensions != NULL) { - return extension_dict::ClearExtension(self->extensions, arg); + return extension_dict::ClearExtension(self->extensions, extension); + } else { + const FieldDescriptor* descriptor = GetExtensionDescriptor(extension); + if (descriptor == NULL) { + return NULL; + } + if (ScopedPyObjectPtr(ClearFieldByDescriptor(self, descriptor)) == NULL) { + return NULL; + } } - PyErr_SetString(PyExc_TypeError, "Message is not extendable"); - return NULL; + Py_RETURN_NONE; } -PyObject* HasExtension(CMessage* self, PyObject* arg) { - if (self->extensions != NULL) { - return extension_dict::HasExtension(self->extensions, arg); +PyObject* HasExtension(CMessage* self, PyObject* extension) { + const FieldDescriptor* descriptor = GetExtensionDescriptor(extension); + if (descriptor == NULL) { + return NULL; } - PyErr_SetString(PyExc_TypeError, "Message is not extendable"); - return NULL; + return HasFieldByDescriptor(self, descriptor); } // --------------------------------------------------------------------- @@ -1529,13 +1554,8 @@ struct SetOwnerVisitor : public ChildVisitor { return 0; } - int VisitScalarMapContainer(ScalarMapContainer* container) { - scalar_map_container::SetOwner(container, new_owner_); - return 0; - } - - int VisitMessageMapContainer(MessageMapContainer* container) { - message_map_container::SetOwner(container, new_owner_); + int VisitMapContainer(MapContainer* container) { + container->SetOwner(new_owner_); return 0; } @@ -1608,14 +1628,8 @@ struct ReleaseChild : public ChildVisitor { reinterpret_cast(container)); } - int VisitScalarMapContainer(ScalarMapContainer* container) { - return scalar_map_container::Release( - reinterpret_cast(container)); - } - - int VisitMessageMapContainer(MessageMapContainer* container) { - return message_map_container::Release( - reinterpret_cast(container)); + int VisitMapContainer(MapContainer* container) { + return reinterpret_cast(container)->Release(); } int VisitCMessage(CMessage* cmessage, @@ -1707,17 +1721,7 @@ PyObject* Clear(CMessage* self) { AssureWritable(self); if (ForEachCompositeField(self, ReleaseChild(self)) == -1) return NULL; - - // The old ExtensionDict still aliases this CMessage, but all its - // fields have been released. - if (self->extensions != NULL) { - Py_CLEAR(self->extensions); - ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); - if (extension_dict == NULL) { - return NULL; - } - self->extensions = extension_dict; - } + Py_CLEAR(self->extensions); if (self->composite_fields) { PyDict_Clear(self->composite_fields); } @@ -1997,7 +2001,6 @@ static PyObject* RegisterExtension(PyObject* cls, if (descriptor->is_extension() && descriptor->containing_type()->options().message_set_wire_format() && descriptor->type() == FieldDescriptor::TYPE_MESSAGE && - descriptor->message_type() == descriptor->extension_scope() && descriptor->label() == FieldDescriptor::LABEL_OPTIONAL) { ScopedPyObjectPtr message_name(PyString_FromStringAndSize( descriptor->message_type()->full_name().c_str(), @@ -2042,6 +2045,8 @@ static PyObject* WhichOneof(CMessage* self, PyObject* arg) { } } +static PyObject* GetExtensionDict(CMessage* self, void *closure); + static PyObject* ListFields(CMessage* self) { vector fields; self->message->GetReflection()->ListFields(*self->message, &fields); @@ -2079,12 +2084,13 @@ static PyObject* ListFields(CMessage* self) { PyErr_Clear(); continue; } - PyObject* extensions = reinterpret_cast(self->extensions); + ScopedPyObjectPtr extensions(GetExtensionDict(self, NULL)); if (extensions == NULL) { return NULL; } // 'extension' reference later stolen by PyTuple_SET_ITEM. - PyObject* extension = PyObject_GetItem(extensions, extension_field.get()); + PyObject* extension = PyObject_GetItem( + extensions.get(), extension_field.get()); if (extension == NULL) { return NULL; } @@ -2493,9 +2499,31 @@ PyObject* _CheckCalledFromGeneratedFile(PyObject* unused, Py_RETURN_NONE; } -static PyMemberDef Members[] = { - {"Extensions", T_OBJECT_EX, offsetof(CMessage, extensions), 0, - "Extension dict"}, +static PyObject* GetExtensionDict(CMessage* self, void *closure) { + if (self->extensions) { + Py_INCREF(self->extensions); + return reinterpret_cast(self->extensions); + } + + // If there are extension_ranges, the message is "extendable". Allocate a + // dictionary to store the extension fields. + const Descriptor* descriptor = GetMessageDescriptor(Py_TYPE(self)); + if (descriptor->extension_range_count() > 0) { + ExtensionDict* extension_dict = extension_dict::NewExtensionDict(self); + if (extension_dict == NULL) { + return NULL; + } + self->extensions = extension_dict; + Py_INCREF(self->extensions); + return reinterpret_cast(self->extensions); + } + + PyErr_SetNone(PyExc_AttributeError); + return NULL; +} + +static PyGetSetDef Getters[] = { + {"Extensions", (getter)GetExtensionDict, NULL, "Extension dict"}, {NULL} }; @@ -2592,10 +2620,10 @@ PyObject* GetAttr(CMessage* self, PyObject* name) { if (value_class == NULL) { return NULL; } - py_container = message_map_container::NewContainer(self, field_descriptor, - value_class); + py_container = + NewMessageMapContainer(self, field_descriptor, value_class); } else { - py_container = scalar_map_container::NewContainer(self, field_descriptor); + py_container = NewScalarMapContainer(self, field_descriptor); } if (py_container == NULL) { return NULL; @@ -2672,7 +2700,10 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) { } } - PyErr_Format(PyExc_AttributeError, "Assignment not allowed"); + PyErr_Format(PyExc_AttributeError, + "Assignment not allowed " + "(no field \"%s\"in protocol message object).", + PyString_AsString(name)); return -1; } @@ -2707,8 +2738,8 @@ PyTypeObject CMessage_Type = { 0, // tp_iter 0, // tp_iternext cmessage::Methods, // tp_methods - cmessage::Members, // tp_members - 0, // tp_getset + 0, // tp_members + cmessage::Getters, // tp_getset 0, // tp_base 0, // tp_dict 0, // tp_descr_get @@ -2910,12 +2941,12 @@ bool InitProto2MessageModule(PyObject *m) { reinterpret_cast(&ScalarMapContainer_Type)); #endif - if (PyType_Ready(&ScalarMapIterator_Type) < 0) { + if (PyType_Ready(&MapIterator_Type) < 0) { return false; } - PyModule_AddObject(m, "ScalarMapIterator", - reinterpret_cast(&ScalarMapIterator_Type)); + PyModule_AddObject(m, "MapIterator", + reinterpret_cast(&MapIterator_Type)); #if PY_MAJOR_VERSION >= 3 @@ -2934,13 +2965,6 @@ bool InitProto2MessageModule(PyObject *m) { PyModule_AddObject(m, "MessageMapContainer", reinterpret_cast(&MessageMapContainer_Type)); #endif - - if (PyType_Ready(&MessageMapIterator_Type) < 0) { - return false; - } - - PyModule_AddObject(m, "MessageMapIterator", - reinterpret_cast(&MessageMapIterator_Type)); } if (PyType_Ready(&ExtensionDict_Type) < 0) { @@ -2957,6 +2981,9 @@ bool InitProto2MessageModule(PyObject *m) { PyModule_AddObject(m, "default_pool", reinterpret_cast(GetDefaultDescriptorPool())); + PyModule_AddObject(m, "DescriptorPool", reinterpret_cast( + &PyDescriptorPool_Type)); + // This implementation provides full Descriptor types, we advertise it so that // descriptor.py can use them in replacement of the Python classes. PyModule_AddIntConstant(m, "_USE_C_DESCRIPTORS", 1); diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 94de4551..cc0012e9 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -307,6 +307,7 @@ bool CheckAndGetInteger( bool CheckAndGetDouble(PyObject* arg, double* value); bool CheckAndGetFloat(PyObject* arg, float* value); bool CheckAndGetBool(PyObject* arg, bool* value); +PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor); bool CheckAndSetString( PyObject* arg, Message* message, const FieldDescriptor* descriptor, diff --git a/python/google/protobuf/pyext/message_map_container.cc b/python/google/protobuf/pyext/message_map_container.cc deleted file mode 100644 index 8902fa00..00000000 --- a/python/google/protobuf/pyext/message_map_container.cc +++ /dev/null @@ -1,569 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: haberman@google.com (Josh Haberman) - -#include - -#include -#include -#include -#include -#include - -namespace google { -namespace protobuf { -namespace python { - -struct MessageMapIterator { - PyObject_HEAD; - - // This dict contains the full contents of what we want to iterate over. - // There's no way to avoid building this, because the list representation - // (which is canonical) can contain duplicate keys. So at the very least we - // need a set that lets us skip duplicate keys. And at the point that we're - // doing that, we might as well just build the actual dict we're iterating - // over and use dict's built-in iterator. - PyObject* dict; - - // An iterator on dict. - PyObject* iter; - - // A pointer back to the container, so we can notice changes to the version. - MessageMapContainer* container; - - // The version of the map when we took the iterator to it. - // - // We store this so that if the map is modified during iteration we can throw - // an error. - uint64 version; -}; - -static MessageMapIterator* GetIter(PyObject* obj) { - return reinterpret_cast(obj); -} - -namespace message_map_container { - -static MessageMapContainer* GetMap(PyObject* obj) { - return reinterpret_cast(obj); -} - -// The private constructor of MessageMapContainer objects. -PyObject* NewContainer(CMessage* parent, - const google::protobuf::FieldDescriptor* parent_field_descriptor, - PyObject* concrete_class) { - if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { - return NULL; - } - -#if PY_MAJOR_VERSION >= 3 - PyObject* obj = PyType_GenericAlloc( - reinterpret_cast(MessageMapContainer_Type), 0); -#else - PyObject* obj = PyType_GenericAlloc(&MessageMapContainer_Type, 0); -#endif - if (obj == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate new container."); - } - - MessageMapContainer* self = GetMap(obj); - - self->message = parent->message; - self->parent = parent; - self->parent_field_descriptor = parent_field_descriptor; - self->owner = parent->owner; - self->version = 0; - - self->key_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("key"); - self->value_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("value"); - - self->message_dict = PyDict_New(); - if (self->message_dict == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate message dict."); - } - - Py_INCREF(concrete_class); - self->subclass_init = concrete_class; - - if (self->key_field_descriptor == NULL || - self->value_field_descriptor == NULL) { - Py_DECREF(obj); - return PyErr_Format(PyExc_KeyError, - "Map entry descriptor did not have key/value fields"); - } - - return obj; -} - -// Initializes the underlying Message object of "to" so it becomes a new parent -// repeated scalar, and copies all the values from "from" to it. A child scalar -// container can be released by passing it as both from and to (e.g. making it -// the recipient of the new parent message and copying the values from itself). -static int InitializeAndCopyToParentContainer( - MessageMapContainer* from, - MessageMapContainer* to) { - // For now we require from == to, re-evaluate if we want to support deep copy - // as in repeated_composite_container.cc. - GOOGLE_DCHECK(from == to); - Message* old_message = from->message; - Message* new_message = old_message->New(); - to->parent = NULL; - to->parent_field_descriptor = from->parent_field_descriptor; - to->message = new_message; - to->owner.reset(new_message); - - vector fields; - fields.push_back(from->parent_field_descriptor); - old_message->GetReflection()->SwapFields(old_message, new_message, fields); - return 0; -} - -static PyObject* GetCMessage(MessageMapContainer* self, Message* entry) { - // Get or create the CMessage object corresponding to this message. - Message* message = entry->GetReflection()->MutableMessage( - entry, self->value_field_descriptor); - ScopedPyObjectPtr key(PyLong_FromVoidPtr(message)); - PyObject* ret = PyDict_GetItem(self->message_dict, key.get()); - - if (ret == NULL) { - CMessage* cmsg = cmessage::NewEmptyMessage(self->subclass_init, - message->GetDescriptor()); - ret = reinterpret_cast(cmsg); - - if (cmsg == NULL) { - return NULL; - } - cmsg->owner = self->owner; - cmsg->message = message; - cmsg->parent = self->parent; - - if (PyDict_SetItem(self->message_dict, key.get(), ret) < 0) { - Py_DECREF(ret); - return NULL; - } - } else { - Py_INCREF(ret); - } - - return ret; -} - -int Release(MessageMapContainer* self) { - InitializeAndCopyToParentContainer(self, self); - return 0; -} - -void SetOwner(MessageMapContainer* self, - const shared_ptr& new_owner) { - self->owner = new_owner; -} - -Py_ssize_t Length(PyObject* _self) { - MessageMapContainer* self = GetMap(_self); - google::protobuf::Message* message = self->message; - return message->GetReflection()->FieldSize(*message, - self->parent_field_descriptor); -} - -int MapKeyMatches(MessageMapContainer* self, const Message* entry, - PyObject* key) { - // TODO(haberman): do we need more strict type checking? - ScopedPyObjectPtr entry_key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - int ret = PyObject_RichCompareBool(key, entry_key.get(), Py_EQ); - return ret; -} - -int SetItem(PyObject *_self, PyObject *key, PyObject *v) { - if (v) { - PyErr_Format(PyExc_ValueError, - "Direct assignment of submessage not allowed"); - return -1; - } - - // Now we know that this is a delete, not a set. - - MessageMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - bool found = false; - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return -1; - if (matches) { - found = true; - if (i != (int)size - 1) { - reflection->SwapElements(message, self->parent_field_descriptor, i, - size - 1); - } - reflection->RemoveLast(message, self->parent_field_descriptor); - - // Can't exit now, the repeated field representation of maps allows - // duplicate keys, and we have to be sure to remove all of them. - } - } - - if (!found) { - PyErr_Format(PyExc_KeyError, "Key not present in map"); - return -1; - } - - self->version++; - - return 0; -} - -PyObject* GetIterator(PyObject *_self) { - MessageMapContainer* self = GetMap(_self); - - ScopedPyObjectPtr obj(PyType_GenericAlloc(&MessageMapIterator_Type, 0)); - if (obj == NULL) { - return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); - } - - MessageMapIterator* iter = GetIter(obj.get()); - - Py_INCREF(self); - iter->container = self; - iter->version = self->version; - iter->dict = PyDict_New(); - if (iter->dict == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate dict for iterator."); - } - - // Build the entire map into a dict right now. Start from the beginning so - // that later entries win in the case of duplicates. - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - ScopedPyObjectPtr key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - if (PyDict_SetItem(iter->dict, key.get(), GetCMessage(self, entry)) < 0) { - return PyErr_Format(PyExc_RuntimeError, - "SetItem failed in iterator construction."); - } - } - - iter->iter = PyObject_GetIter(iter->dict); - - return obj.release(); -} - -PyObject* GetItem(PyObject* _self, PyObject* key) { - MessageMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return NULL; - if (matches) { - return GetCMessage(self, entry); - } - } - - // Key is not already present; insert a new entry. - Message* entry = - reflection->AddMessage(message, self->parent_field_descriptor); - - self->version++; - - if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, - key) < 0) { - reflection->RemoveLast(message, self->parent_field_descriptor); - return NULL; - } - - return GetCMessage(self, entry); -} - -PyObject* Contains(PyObject* _self, PyObject* key) { - MessageMapContainer* self = GetMap(_self); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - int size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = 0; i < size; i++) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return NULL; - if (matches) { - Py_RETURN_TRUE; - } - } - - Py_RETURN_FALSE; -} - -PyObject* Clear(PyObject* _self) { - MessageMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - self->version++; - reflection->ClearField(message, self->parent_field_descriptor); - - Py_RETURN_NONE; -} - -PyObject* Get(PyObject* self, PyObject* args) { - PyObject* key; - PyObject* default_value = NULL; - if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { - return NULL; - } - - ScopedPyObjectPtr is_present(Contains(self, key)); - if (is_present.get() == NULL) { - return NULL; - } - - if (PyObject_IsTrue(is_present.get())) { - return GetItem(self, key); - } else { - if (default_value != NULL) { - Py_INCREF(default_value); - return default_value; - } else { - Py_RETURN_NONE; - } - } -} - -static void Dealloc(PyObject* _self) { - MessageMapContainer* self = GetMap(_self); - self->owner.reset(); - Py_DECREF(self->message_dict); - Py_TYPE(_self)->tp_free(_self); -} - -static PyMethodDef Methods[] = { - { "__contains__", (PyCFunction)Contains, METH_O, - "Tests whether the map contains this element."}, - { "clear", (PyCFunction)Clear, METH_NOARGS, - "Removes all elements from the map."}, - { "get", Get, METH_VARARGS, - "Gets the value for the given key if present, or otherwise a default" }, - { "get_or_create", GetItem, METH_O, - "Alias for getitem, useful to make explicit that the map is mutated." }, - /* - { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, - "Makes a deep copy of the class." }, - { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, - "Outputs picklable representation of the repeated field." }, - */ - {NULL, NULL}, -}; - -} // namespace message_map_container - -namespace message_map_iterator { - -static void Dealloc(PyObject* _self) { - MessageMapIterator* self = GetIter(_self); - Py_DECREF(self->dict); - Py_DECREF(self->iter); - Py_DECREF(self->container); - Py_TYPE(_self)->tp_free(_self); -} - -PyObject* IterNext(PyObject* _self) { - MessageMapIterator* self = GetIter(_self); - - // This won't catch mutations to the map performed by MergeFrom(); no easy way - // to address that. - if (self->version != self->container->version) { - return PyErr_Format(PyExc_RuntimeError, - "Map modified during iteration."); - } - - return PyIter_Next(self->iter); -} - -} // namespace message_map_iterator - -#if PY_MAJOR_VERSION >= 3 - static PyType_Slot MessageMapContainer_Type_slots[] = { - {Py_tp_dealloc, (void *)message_map_container::Dealloc}, - {Py_mp_length, (void *)message_map_container::Length}, - {Py_mp_subscript, (void *)message_map_container::GetItem}, - {Py_mp_ass_subscript, (void *)message_map_container::SetItem}, - {Py_tp_methods, (void *)message_map_container::Methods}, - {Py_tp_iter, (void *)message_map_container::GetIterator}, - {0, 0} - }; - - PyType_Spec MessageMapContainer_Type_spec = { - FULL_MODULE_NAME ".MessageMapContainer", - sizeof(MessageMapContainer), - 0, - Py_TPFLAGS_DEFAULT, - MessageMapContainer_Type_slots - }; - - PyObject *MessageMapContainer_Type; - -#else - static PyMappingMethods MpMethods = { - message_map_container::Length, // mp_length - message_map_container::GetItem, // mp_subscript - message_map_container::SetItem, // mp_ass_subscript - }; - - PyTypeObject MessageMapContainer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".MessageMapContainer", // tp_name - sizeof(MessageMapContainer), // tp_basicsize - 0, // tp_itemsize - message_map_container::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - &MpMethods, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A map container for message", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - message_map_container::GetIterator, // tp_iter - 0, // tp_iternext - message_map_container::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - }; -#endif - -PyTypeObject MessageMapIterator_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".MessageMapIterator", // tp_name - sizeof(MessageMapIterator), // tp_basicsize - 0, // tp_itemsize - message_map_iterator::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map iterator", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - PyObject_SelfIter, // tp_iter - message_map_iterator::IterNext, // tp_iternext - 0, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init -}; - -} // namespace python -} // namespace protobuf -} // namespace google diff --git a/python/google/protobuf/pyext/message_map_container.h b/python/google/protobuf/pyext/message_map_container.h deleted file mode 100644 index 4f6cb26a..00000000 --- a/python/google/protobuf/pyext/message_map_container.h +++ /dev/null @@ -1,126 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ -#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ - -#include - -#include -#ifndef _SHARED_PTR_H -#include -#endif - -#include - -namespace google { -namespace protobuf { - -class Message; - -#ifdef _SHARED_PTR_H -using std::shared_ptr; -#else -using internal::shared_ptr; -#endif - -namespace python { - -struct CMessage; - -struct MessageMapContainer { - PyObject_HEAD; - - // This is the top-level C++ Message object that owns the whole - // proto tree. Every Python MessageMapContainer holds a - // reference to it in order to keep it alive as long as there's a - // Python object that references any part of the tree. - shared_ptr owner; - - // Pointer to the C++ Message that contains this container. The - // MessageMapContainer does not own this pointer. - Message* message; - - // Weak reference to a parent CMessage object (i.e. may be NULL.) - // - // Used to make sure all ancestors are also mutable when first - // modifying the container. - CMessage* parent; - - // Pointer to the parent's descriptor that describes this - // field. Used together with the parent's message when making a - // default message instance mutable. - // The pointer is owned by the global DescriptorPool. - const FieldDescriptor* parent_field_descriptor; - const FieldDescriptor* key_field_descriptor; - const FieldDescriptor* value_field_descriptor; - - // A callable that is used to create new child messages. - PyObject* subclass_init; - - // A dict mapping Message* -> CMessage. - PyObject* message_dict; - - // We bump this whenever we perform a mutation, to invalidate existing - // iterators. - uint64 version; -}; - -#if PY_MAJOR_VERSION >= 3 - extern PyObject *MessageMapContainer_Type; - extern PyType_Spec MessageMapContainer_Type_spec; -#else - extern PyTypeObject MessageMapContainer_Type; -#endif -extern PyTypeObject MessageMapIterator_Type; - -namespace message_map_container { - -// Builds a MessageMapContainer object, from a parent message and a -// field descriptor. -extern PyObject* NewContainer(CMessage* parent, - const FieldDescriptor* parent_field_descriptor, - PyObject* concrete_class); - -// Releases the messages in the container to a new message. -// -// Returns 0 on success, -1 on failure. -int Release(MessageMapContainer* self); - -// Set the owner field of self and any children of self. -void SetOwner(MessageMapContainer* self, - const shared_ptr& new_owner); - -} // namespace message_map_container -} // namespace python -} // namespace protobuf - -} // namespace google -#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_MAP_CONTAINER_H__ diff --git a/python/google/protobuf/pyext/scalar_map_container.cc b/python/google/protobuf/pyext/scalar_map_container.cc deleted file mode 100644 index 0b0d5a3d..00000000 --- a/python/google/protobuf/pyext/scalar_map_container.cc +++ /dev/null @@ -1,542 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: haberman@google.com (Josh Haberman) - -#include - -#include -#include -#include -#include -#include - -namespace google { -namespace protobuf { -namespace python { - -struct ScalarMapIterator { - PyObject_HEAD; - - // This dict contains the full contents of what we want to iterate over. - // There's no way to avoid building this, because the list representation - // (which is canonical) can contain duplicate keys. So at the very least we - // need a set that lets us skip duplicate keys. And at the point that we're - // doing that, we might as well just build the actual dict we're iterating - // over and use dict's built-in iterator. - PyObject* dict; - - // An iterator on dict. - PyObject* iter; - - // A pointer back to the container, so we can notice changes to the version. - ScalarMapContainer* container; - - // The version of the map when we took the iterator to it. - // - // We store this so that if the map is modified during iteration we can throw - // an error. - uint64 version; -}; - -static ScalarMapIterator* GetIter(PyObject* obj) { - return reinterpret_cast(obj); -} - -namespace scalar_map_container { - -static ScalarMapContainer* GetMap(PyObject* obj) { - return reinterpret_cast(obj); -} - -// The private constructor of ScalarMapContainer objects. -PyObject *NewContainer( - CMessage* parent, const google::protobuf::FieldDescriptor* parent_field_descriptor) { - if (!CheckFieldBelongsToMessage(parent_field_descriptor, parent->message)) { - return NULL; - } - -#if PY_MAJOR_VERSION >= 3 - ScopedPyObjectPtr obj(PyType_GenericAlloc( - reinterpret_cast(ScalarMapContainer_Type), 0)); -#else - ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapContainer_Type, 0)); -#endif - if (obj.get() == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate new container."); - } - - ScalarMapContainer* self = GetMap(obj.get()); - - self->message = parent->message; - self->parent = parent; - self->parent_field_descriptor = parent_field_descriptor; - self->owner = parent->owner; - self->version = 0; - - self->key_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("key"); - self->value_field_descriptor = - parent_field_descriptor->message_type()->FindFieldByName("value"); - - if (self->key_field_descriptor == NULL || - self->value_field_descriptor == NULL) { - return PyErr_Format(PyExc_KeyError, - "Map entry descriptor did not have key/value fields"); - } - - return obj.release(); -} - -// Initializes the underlying Message object of "to" so it becomes a new parent -// repeated scalar, and copies all the values from "from" to it. A child scalar -// container can be released by passing it as both from and to (e.g. making it -// the recipient of the new parent message and copying the values from itself). -static int InitializeAndCopyToParentContainer( - ScalarMapContainer* from, - ScalarMapContainer* to) { - // For now we require from == to, re-evaluate if we want to support deep copy - // as in repeated_scalar_container.cc. - GOOGLE_DCHECK(from == to); - Message* old_message = from->message; - Message* new_message = old_message->New(); - to->parent = NULL; - to->parent_field_descriptor = from->parent_field_descriptor; - to->message = new_message; - to->owner.reset(new_message); - - vector fields; - fields.push_back(from->parent_field_descriptor); - old_message->GetReflection()->SwapFields(old_message, new_message, fields); - return 0; -} - -int Release(ScalarMapContainer* self) { - return InitializeAndCopyToParentContainer(self, self); -} - -void SetOwner(ScalarMapContainer* self, - const shared_ptr& new_owner) { - self->owner = new_owner; -} - -Py_ssize_t Length(PyObject* _self) { - ScalarMapContainer* self = GetMap(_self); - google::protobuf::Message* message = self->message; - return message->GetReflection()->FieldSize(*message, - self->parent_field_descriptor); -} - -int MapKeyMatches(ScalarMapContainer* self, const Message* entry, - PyObject* key) { - // TODO(haberman): do we need more strict type checking? - ScopedPyObjectPtr entry_key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - int ret = PyObject_RichCompareBool(key, entry_key.get(), Py_EQ); - return ret; -} - -PyObject* GetItem(PyObject* _self, PyObject* key) { - ScalarMapContainer* self = GetMap(_self); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - const Message& entry = reflection->GetRepeatedMessage( - *message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, &entry, key); - if (matches < 0) return NULL; - if (matches) { - return cmessage::InternalGetScalar(&entry, self->value_field_descriptor); - } - } - - // Need to add a new entry. - Message* entry = - reflection->AddMessage(message, self->parent_field_descriptor); - PyObject* ret = NULL; - - if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, - key) >= 0) { - ret = cmessage::InternalGetScalar(entry, self->value_field_descriptor); - } - - self->version++; - - // If there was a type error above, it set the Python exception. - return ret; -} - -int SetItem(PyObject *_self, PyObject *key, PyObject *v) { - ScalarMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - self->version++; - - if (v) { - // Set item. - // - // Right now the Reflection API doesn't support map lookup, so we implement - // it via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return -1; - if (matches) { - return cmessage::InternalSetNonOneofScalar( - entry, self->value_field_descriptor, v); - } - } - - // Key is not already present; insert a new entry. - Message* entry = - reflection->AddMessage(message, self->parent_field_descriptor); - - if (cmessage::InternalSetNonOneofScalar(entry, self->key_field_descriptor, - key) < 0 || - cmessage::InternalSetNonOneofScalar(entry, self->value_field_descriptor, - v) < 0) { - reflection->RemoveLast(message, self->parent_field_descriptor); - return -1; - } - - return 0; - } else { - bool found = false; - for (int i = size - 1; i >= 0; i--) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, entry, key); - if (matches < 0) return -1; - if (matches) { - found = true; - if (i != (int)size - 1) { - reflection->SwapElements(message, self->parent_field_descriptor, i, - size - 1); - } - reflection->RemoveLast(message, self->parent_field_descriptor); - - // Can't exit now, the repeated field representation of maps allows - // duplicate keys, and we have to be sure to remove all of them. - } - } - - if (found) { - return 0; - } else { - PyErr_Format(PyExc_KeyError, "Key not present in map"); - return -1; - } - } -} - -PyObject* GetIterator(PyObject *_self) { - ScalarMapContainer* self = GetMap(_self); - - ScopedPyObjectPtr obj(PyType_GenericAlloc(&ScalarMapIterator_Type, 0)); - if (obj == NULL) { - return PyErr_Format(PyExc_KeyError, "Could not allocate iterator"); - } - - ScalarMapIterator* iter = GetIter(obj.get()); - - Py_INCREF(self); - iter->container = self; - iter->version = self->version; - iter->dict = PyDict_New(); - if (iter->dict == NULL) { - return PyErr_Format(PyExc_RuntimeError, - "Could not allocate dict for iterator."); - } - - // Build the entire map into a dict right now. Start from the beginning so - // that later entries win in the case of duplicates. - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. We need to search from the end because the underlying - // representation can have duplicates if a user calls MergeFrom(); the last - // one needs to win. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = - reflection->FieldSize(*message, self->parent_field_descriptor); - for (size_t i = 0; i < size; i++) { - Message* entry = reflection->MutableRepeatedMessage( - message, self->parent_field_descriptor, i); - ScopedPyObjectPtr key( - cmessage::InternalGetScalar(entry, self->key_field_descriptor)); - ScopedPyObjectPtr val( - cmessage::InternalGetScalar(entry, self->value_field_descriptor)); - if (PyDict_SetItem(iter->dict, key.get(), val.get()) < 0) { - return PyErr_Format(PyExc_RuntimeError, - "SetItem failed in iterator construction."); - } - } - - - iter->iter = PyObject_GetIter(iter->dict); - - - return obj.release(); -} - -PyObject* Clear(PyObject* _self) { - ScalarMapContainer* self = GetMap(_self); - cmessage::AssureWritable(self->parent); - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - reflection->ClearField(message, self->parent_field_descriptor); - - Py_RETURN_NONE; -} - -PyObject* Contains(PyObject* _self, PyObject* key) { - ScalarMapContainer* self = GetMap(_self); - - Message* message = self->message; - const Reflection* reflection = message->GetReflection(); - - // Right now the Reflection API doesn't support map lookup, so we implement it - // via linear search. - // - // TODO(haberman): add lookup API to Reflection API. - size_t size = reflection->FieldSize(*message, self->parent_field_descriptor); - for (int i = size - 1; i >= 0; i--) { - const Message& entry = reflection->GetRepeatedMessage( - *message, self->parent_field_descriptor, i); - int matches = MapKeyMatches(self, &entry, key); - if (matches < 0) return NULL; - if (matches) { - Py_RETURN_TRUE; - } - } - - Py_RETURN_FALSE; -} - -PyObject* Get(PyObject* self, PyObject* args) { - PyObject* key; - PyObject* default_value = NULL; - if (PyArg_ParseTuple(args, "O|O", &key, &default_value) < 0) { - return NULL; - } - - ScopedPyObjectPtr is_present(Contains(self, key)); - if (is_present.get() == NULL) { - return NULL; - } - - if (PyObject_IsTrue(is_present.get())) { - return GetItem(self, key); - } else { - if (default_value != NULL) { - Py_INCREF(default_value); - return default_value; - } else { - Py_RETURN_NONE; - } - } -} - -static void Dealloc(PyObject* _self) { - ScalarMapContainer* self = GetMap(_self); - self->owner.reset(); - Py_TYPE(_self)->tp_free(_self); -} - -static PyMethodDef Methods[] = { - { "__contains__", Contains, METH_O, - "Tests whether a key is a member of the map." }, - { "clear", (PyCFunction)Clear, METH_NOARGS, - "Removes all elements from the map." }, - { "get", Get, METH_VARARGS, - "Gets the value for the given key if present, or otherwise a default" }, - /* - { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, - "Makes a deep copy of the class." }, - { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, - "Outputs picklable representation of the repeated field." }, - */ - {NULL, NULL}, -}; - -} // namespace scalar_map_container - -namespace scalar_map_iterator { - -static void Dealloc(PyObject* _self) { - ScalarMapIterator* self = GetIter(_self); - Py_DECREF(self->dict); - Py_DECREF(self->iter); - Py_DECREF(self->container); - Py_TYPE(_self)->tp_free(_self); -} - -PyObject* IterNext(PyObject* _self) { - ScalarMapIterator* self = GetIter(_self); - - // This won't catch mutations to the map performed by MergeFrom(); no easy way - // to address that. - if (self->version != self->container->version) { - return PyErr_Format(PyExc_RuntimeError, - "Map modified during iteration."); - } - - return PyIter_Next(self->iter); -} - -} // namespace scalar_map_iterator - - -#if PY_MAJOR_VERSION >= 3 - static PyType_Slot ScalarMapContainer_Type_slots[] = { - {Py_tp_dealloc, (void *)scalar_map_container::Dealloc}, - {Py_mp_length, (void *)scalar_map_container::Length}, - {Py_mp_subscript, (void *)scalar_map_container::GetItem}, - {Py_mp_ass_subscript, (void *)scalar_map_container::SetItem}, - {Py_tp_methods, (void *)scalar_map_container::Methods}, - {Py_tp_iter, (void *)scalar_map_container::GetIterator}, - {0, 0}, - }; - - PyType_Spec ScalarMapContainer_Type_spec = { - FULL_MODULE_NAME ".ScalarMapContainer", - sizeof(ScalarMapContainer), - 0, - Py_TPFLAGS_DEFAULT, - ScalarMapContainer_Type_slots - }; - PyObject *ScalarMapContainer_Type; -#else - static PyMappingMethods MpMethods = { - scalar_map_container::Length, // mp_length - scalar_map_container::GetItem, // mp_subscript - scalar_map_container::SetItem, // mp_ass_subscript - }; - - PyTypeObject ScalarMapContainer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".ScalarMapContainer", // tp_name - sizeof(ScalarMapContainer), // tp_basicsize - 0, // tp_itemsize - scalar_map_container::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - &MpMethods, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map container", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - scalar_map_container::GetIterator, // tp_iter - 0, // tp_iternext - scalar_map_container::Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init - }; -#endif - -PyTypeObject ScalarMapIterator_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - FULL_MODULE_NAME ".ScalarMapIterator", // tp_name - sizeof(ScalarMapIterator), // tp_basicsize - 0, // tp_itemsize - scalar_map_iterator::Dealloc, // tp_dealloc - 0, // tp_print - 0, // tp_getattr - 0, // tp_setattr - 0, // tp_compare - 0, // tp_repr - 0, // tp_as_number - 0, // tp_as_sequence - 0, // tp_as_mapping - 0, // tp_hash - 0, // tp_call - 0, // tp_str - 0, // tp_getattro - 0, // tp_setattro - 0, // tp_as_buffer - Py_TPFLAGS_DEFAULT, // tp_flags - "A scalar map iterator", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - PyObject_SelfIter, // tp_iter - scalar_map_iterator::IterNext, // tp_iternext - 0, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset - 0, // tp_init -}; - -} // namespace python -} // namespace protobuf -} // namespace google diff --git a/python/google/protobuf/pyext/scalar_map_container.h b/python/google/protobuf/pyext/scalar_map_container.h deleted file mode 100644 index 4d663b88..00000000 --- a/python/google/protobuf/pyext/scalar_map_container.h +++ /dev/null @@ -1,119 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ -#define GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ - -#include - -#include -#ifndef _SHARED_PTR_H -#include -#endif - -#include - -namespace google { -namespace protobuf { - -class Message; - -#ifdef _SHARED_PTR_H -using std::shared_ptr; -#else -using internal::shared_ptr; -#endif - -namespace python { - -struct CMessage; - -struct ScalarMapContainer { - PyObject_HEAD; - - // This is the top-level C++ Message object that owns the whole - // proto tree. Every Python ScalarMapContainer holds a - // reference to it in order to keep it alive as long as there's a - // Python object that references any part of the tree. - shared_ptr owner; - - // Pointer to the C++ Message that contains this container. The - // ScalarMapContainer does not own this pointer. - Message* message; - - // Weak reference to a parent CMessage object (i.e. may be NULL.) - // - // Used to make sure all ancestors are also mutable when first - // modifying the container. - CMessage* parent; - - // Pointer to the parent's descriptor that describes this - // field. Used together with the parent's message when making a - // default message instance mutable. - // The pointer is owned by the global DescriptorPool. - const FieldDescriptor* parent_field_descriptor; - const FieldDescriptor* key_field_descriptor; - const FieldDescriptor* value_field_descriptor; - - // We bump this whenever we perform a mutation, to invalidate existing - // iterators. - uint64 version; -}; - -#if PY_MAJOR_VERSION >= 3 - extern PyObject *ScalarMapContainer_Type; - extern PyType_Spec ScalarMapContainer_Type_spec; -#else - extern PyTypeObject ScalarMapContainer_Type; -#endif -extern PyTypeObject ScalarMapIterator_Type; - -namespace scalar_map_container { - -// Builds a ScalarMapContainer object, from a parent message and a -// field descriptor. -extern PyObject *NewContainer( - CMessage* parent, const FieldDescriptor* parent_field_descriptor); - -// Releases the messages in the container to a new message. -// -// Returns 0 on success, -1 on failure. -int Release(ScalarMapContainer* self); - -// Set the owner field of self and any children of self. -void SetOwner(ScalarMapContainer* self, - const shared_ptr& new_owner); - -} // namespace scalar_map_container -} // namespace python -} // namespace protobuf - -} // namespace google -#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCALAR_MAP_CONTAINER_H__ diff --git a/python/google/protobuf/symbol_database.py b/python/google/protobuf/symbol_database.py index b81ef4d7..87760f26 100644 --- a/python/google/protobuf/symbol_database.py +++ b/python/google/protobuf/symbol_database.py @@ -60,7 +60,6 @@ Example usage: """ -from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool @@ -73,37 +72,12 @@ class SymbolDatabase(object): buffer types used within a program. """ - # pylint: disable=protected-access - if _descriptor._USE_C_DESCRIPTORS: - - def __new__(cls): - raise TypeError("Instances of SymbolDatabase cannot be created") - - @classmethod - def _CreateDefaultDatabase(cls): - self = object.__new__(cls) # Bypass the __new__ above. - # Don't call __init__() and initialize here. - self._symbols = {} - self._symbols_by_file = {} - # As of today all descriptors are registered and retrieved from - # _message.default_pool (see FileDescriptor.__new__), so it's not - # necessary to use another pool. - self.pool = _descriptor._message.default_pool - return self - # pylint: enable=protected-access - - else: - - @classmethod - def _CreateDefaultDatabase(cls): - return cls() - - def __init__(self): + def __init__(self, pool=None): """Constructor.""" self._symbols = {} self._symbols_by_file = {} - self.pool = descriptor_pool.DescriptorPool() + self.pool = pool or descriptor_pool.Default() def RegisterMessage(self, message): """Registers the given message type in the local database. @@ -203,7 +177,7 @@ class SymbolDatabase(object): result.update(self._symbols_by_file[f]) return result -_DEFAULT = SymbolDatabase._CreateDefaultDatabase() +_DEFAULT = SymbolDatabase(pool=descriptor_pool.Default()) def Default(): diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index e4fadf09..8d256076 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -66,6 +66,7 @@ _FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE) _FLOAT_NAN = re.compile('nanf?', re.IGNORECASE) _FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) +_QUOTES = frozenset(("'", '"')) class Error(Exception): @@ -73,7 +74,8 @@ class Error(Exception): class ParseError(Error): - """Thrown in case of ASCII parsing error.""" + """Thrown in case of text parsing error.""" + class TextWriter(object): def __init__(self, as_utf8): @@ -102,7 +104,8 @@ def MessageToString(message, as_utf8=False, as_one_line=False, Floating point values can be formatted compactly with 15 digits of precision (which is the most that IEEE 754 "double" can guarantee) - using float_format='.15g'. + using float_format='.15g'. To ensure that converting to text and back to a + proto will result in an identical value, float_format='.17g' should be used. Args: message: The protocol buffers message. @@ -130,11 +133,13 @@ def MessageToString(message, as_utf8=False, as_one_line=False, return result.rstrip() return result + def _IsMapEntry(field): return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and field.message_type.has_options and field.message_type.GetOptions().map_entry) + def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None): @@ -166,17 +171,18 @@ def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False, use_index_order=use_index_order, float_format=float_format) + def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, float_format=None): """Print a single field name/value pair. For repeated fields, the value - should be a single element.""" + should be a single element. + """ out.write(' ' * indent) if field.is_extension: out.write('[') if (field.containing_type.GetOptions().message_set_wire_format and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and - field.message_type == field.extension_scope and field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL): out.write(field.message_type.full_name) else: @@ -262,95 +268,113 @@ def PrintFieldValue(field, value, out, indent=0, as_utf8=False, out.write(str(value)) -def Parse(text, message): - """Parses an ASCII representation of a protocol message into a message. +def Parse(text, message, allow_unknown_extension=False): + """Parses an text representation of a protocol message into a message. Args: - text: Message ASCII representation. + text: Message text representation. message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing Returns: The same message passed as argument. Raises: - ParseError: On ASCII parsing problems. + ParseError: On text parsing problems. """ - if not isinstance(text, str): text = text.decode('utf-8') - return ParseLines(text.split('\n'), message) + if not isinstance(text, str): + text = text.decode('utf-8') + return ParseLines(text.split('\n'), message, allow_unknown_extension) -def Merge(text, message): - """Parses an ASCII representation of a protocol message into a message. +def Merge(text, message, allow_unknown_extension=False): + """Parses an text representation of a protocol message into a message. Like Parse(), but allows repeated values for a non-repeated field, and uses the last one. Args: - text: Message ASCII representation. + text: Message text representation. message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing Returns: The same message passed as argument. Raises: - ParseError: On ASCII parsing problems. + ParseError: On text parsing problems. """ - return MergeLines(text.split('\n'), message) + return MergeLines(text.split('\n'), message, allow_unknown_extension) -def ParseLines(lines, message): - """Parses an ASCII representation of a protocol message into a message. +def ParseLines(lines, message, allow_unknown_extension=False): + """Parses an text representation of a protocol message into a message. Args: - lines: An iterable of lines of a message's ASCII representation. + lines: An iterable of lines of a message's text representation. message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing Returns: The same message passed as argument. Raises: - ParseError: On ASCII parsing problems. + ParseError: On text parsing problems. """ - _ParseOrMerge(lines, message, False) + _ParseOrMerge(lines, message, False, allow_unknown_extension) return message -def MergeLines(lines, message): - """Parses an ASCII representation of a protocol message into a message. +def MergeLines(lines, message, allow_unknown_extension=False): + """Parses an text representation of a protocol message into a message. Args: - lines: An iterable of lines of a message's ASCII representation. + lines: An iterable of lines of a message's text representation. message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing Returns: The same message passed as argument. Raises: - ParseError: On ASCII parsing problems. + ParseError: On text parsing problems. """ - _ParseOrMerge(lines, message, True) + _ParseOrMerge(lines, message, True, allow_unknown_extension) return message -def _ParseOrMerge(lines, message, allow_multiple_scalars): - """Converts an ASCII representation of a protocol message into a message. +def _ParseOrMerge(lines, + message, + allow_multiple_scalars, + allow_unknown_extension=False): + """Converts an text representation of a protocol message into a message. Args: - lines: Lines of a message's ASCII representation. + lines: Lines of a message's text representation. message: A protocol buffer message to merge into. allow_multiple_scalars: Determines if repeated values for a non-repeated field are permitted, e.g., the string "foo: 1 foo: 2" for a required/optional field named "foo". + allow_unknown_extension: if True, skip over missing extensions and keep + parsing Raises: - ParseError: On ASCII parsing problems. + ParseError: On text parsing problems. """ tokenizer = _Tokenizer(lines) while not tokenizer.AtEnd(): - _MergeField(tokenizer, message, allow_multiple_scalars) + _MergeField(tokenizer, message, allow_multiple_scalars, + allow_unknown_extension) -def _MergeField(tokenizer, message, allow_multiple_scalars): +def _MergeField(tokenizer, + message, + allow_multiple_scalars, + allow_unknown_extension=False): """Merges a single protocol message field into a message. Args: @@ -359,9 +383,11 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): allow_multiple_scalars: Determines if repeated values for a non-repeated field are permitted, e.g., the string "foo: 1 foo: 2" for a required/optional field named "foo". + allow_unknown_extension: if True, skip over missing extensions and keep + parsing Raises: - ParseError: In case of ASCII parsing problems. + ParseError: In case of text parsing problems. """ message_descriptor = message.DESCRIPTOR if (hasattr(message_descriptor, 'syntax') and @@ -383,13 +409,18 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): field = message.Extensions._FindExtensionByName(name) # pylint: enable=protected-access if not field: - raise tokenizer.ParseErrorPreviousToken( - 'Extension "%s" not registered.' % name) + if allow_unknown_extension: + field = None + else: + raise tokenizer.ParseErrorPreviousToken( + 'Extension "%s" not registered.' % name) elif message_descriptor != field.containing_type: raise tokenizer.ParseErrorPreviousToken( 'Extension "%s" does not extend message type "%s".' % ( name, message_descriptor.full_name)) + tokenizer.Consume(']') + else: name = tokenizer.ConsumeIdentifier() field = message_descriptor.fields_by_name.get(name, None) @@ -411,7 +442,7 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): 'Message type "%s" has no field named "%s".' % ( message_descriptor.full_name, name)) - if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + if field and field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: is_map_entry = _IsMapEntry(field) tokenizer.TryConsume(':') @@ -438,7 +469,8 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): while not tokenizer.TryConsume(end_token): if tokenizer.AtEnd(): raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token)) - _MergeField(tokenizer, sub_message, allow_multiple_scalars) + _MergeField(tokenizer, sub_message, allow_multiple_scalars, + allow_unknown_extension) if is_map_entry: value_cpptype = field.message_type.fields_by_name['value'].cpp_type @@ -447,8 +479,63 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): value.MergeFrom(sub_message.value) else: getattr(message, field.name)[sub_message.key] = sub_message.value + elif field: + tokenizer.Consume(':') + if (field.label == descriptor.FieldDescriptor.LABEL_REPEATED and + tokenizer.TryConsume('[')): + # Short repeated format, e.g. "foo: [1, 2, 3]" + while True: + _MergeScalarField(tokenizer, message, field, allow_multiple_scalars) + if tokenizer.TryConsume(']'): + break + tokenizer.Consume(',') + else: + _MergeScalarField(tokenizer, message, field, allow_multiple_scalars) + else: # Proto field is unknown. + assert allow_unknown_extension + _SkipFieldContents(tokenizer) + + # For historical reasons, fields may optionally be separated by commas or + # semicolons. + if not tokenizer.TryConsume(','): + tokenizer.TryConsume(';') + + +def _SkipFieldContents(tokenizer): + """Skips over contents (value or message) of a field. + + Args: + tokenizer: A tokenizer to parse the field name and values. + """ + # Try to guess the type of this field. + # If this field is not a message, there should be a ":" between the + # field name and the field value and also the field value should not + # start with "{" or "<" which indicates the beginning of a message body. + # If there is no ":" or there is a "{" or "<" after ":", this field has + # to be a message or the input is ill-formed. + if tokenizer.TryConsume(':') and not tokenizer.LookingAt( + '{') and not tokenizer.LookingAt('<'): + _SkipFieldValue(tokenizer) + else: + _SkipFieldMessage(tokenizer) + + +def _SkipField(tokenizer): + """Skips over a complete field (name and value/message). + + Args: + tokenizer: A tokenizer to parse the field name and values. + """ + if tokenizer.TryConsume('['): + # Consume extension name. + tokenizer.ConsumeIdentifier() + while tokenizer.TryConsume('.'): + tokenizer.ConsumeIdentifier() + tokenizer.Consume(']') else: - _MergeScalarField(tokenizer, message, field, allow_multiple_scalars) + tokenizer.ConsumeIdentifier() + + _SkipFieldContents(tokenizer) # For historical reasons, fields may optionally be separated by commas or # semicolons. @@ -456,6 +543,48 @@ def _MergeField(tokenizer, message, allow_multiple_scalars): tokenizer.TryConsume(';') +def _SkipFieldMessage(tokenizer): + """Skips over a field message. + + Args: + tokenizer: A tokenizer to parse the field name and values. + """ + + if tokenizer.TryConsume('<'): + delimiter = '>' + else: + tokenizer.Consume('{') + delimiter = '}' + + while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'): + _SkipField(tokenizer) + + tokenizer.Consume(delimiter) + + +def _SkipFieldValue(tokenizer): + """Skips over a field value. + + Args: + tokenizer: A tokenizer to parse the field name and values. + + Raises: + ParseError: In case an invalid field value is found. + """ + # String tokens can come in multiple adjacent string literals. + # If we can consume one, consume as many as we can. + if tokenizer.TryConsumeString(): + while tokenizer.TryConsumeString(): + pass + return + + if (not tokenizer.TryConsumeIdentifier() and + not tokenizer.TryConsumeInt64() and + not tokenizer.TryConsumeUint64() and + not tokenizer.TryConsumeFloat()): + raise ParseError('Invalid field value: ' + tokenizer.token) + + def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars): """Merges a single protocol message scalar field into a message. @@ -468,10 +597,9 @@ def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars): required/optional field named "foo". Raises: - ParseError: In case of ASCII parsing problems. + ParseError: In case of text parsing problems. RuntimeError: On runtime errors. """ - tokenizer.Consume(':') value = None if field.type in (descriptor.FieldDescriptor.TYPE_INT32, @@ -525,7 +653,7 @@ def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars): class _Tokenizer(object): - """Protocol buffer ASCII representation tokenizer. + """Protocol buffer text representation tokenizer. This class handles the lower level string parsing by splitting it into meaningful tokens. @@ -534,11 +662,13 @@ class _Tokenizer(object): """ _WHITESPACE = re.compile('(\\s|(#.*$))+', re.MULTILINE) - _TOKEN = re.compile( - '[a-zA-Z_][0-9a-zA-Z_+-]*|' # an identifier - '[0-9+-][0-9a-zA-Z_.+-]*|' # a number - '\"([^\"\n\\\\]|\\\\.)*(\"|\\\\?$)|' # a double-quoted string - '\'([^\'\n\\\\]|\\\\.)*(\'|\\\\?$)') # a single-quoted string + _TOKEN = re.compile('|'.join([ + r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier + r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number + ] + [ # quoted str for each quote mark + r'{qt}([^{qt}\n\\]|\\.)*({qt}|\\?$)'.format(qt=mark) for mark in _QUOTES + ])) + _IDENTIFIER = re.compile(r'\w+') def __init__(self, lines): @@ -555,6 +685,9 @@ class _Tokenizer(object): self._SkipWhitespace() self.NextToken() + def LookingAt(self, token): + return self.token == token + def AtEnd(self): """Checks the end of the text was reached. @@ -610,6 +743,13 @@ class _Tokenizer(object): if not self.TryConsume(token): raise self._ParseError('Expected "%s".' % token) + def TryConsumeIdentifier(self): + try: + self.ConsumeIdentifier() + return True + except ParseError: + return False + def ConsumeIdentifier(self): """Consumes protocol message field identifier. @@ -657,6 +797,13 @@ class _Tokenizer(object): self.NextToken() return result + def TryConsumeInt64(self): + try: + self.ConsumeInt64() + return True + except ParseError: + return False + def ConsumeInt64(self): """Consumes a signed 64bit integer number. @@ -673,6 +820,13 @@ class _Tokenizer(object): self.NextToken() return result + def TryConsumeUint64(self): + try: + self.ConsumeUint64() + return True + except ParseError: + return False + def ConsumeUint64(self): """Consumes an unsigned 64bit integer number. @@ -689,6 +843,13 @@ class _Tokenizer(object): self.NextToken() return result + def TryConsumeFloat(self): + try: + self.ConsumeFloat() + return True + except ParseError: + return False + def ConsumeFloat(self): """Consumes an floating point number. @@ -721,6 +882,13 @@ class _Tokenizer(object): self.NextToken() return result + def TryConsumeString(self): + try: + self.ConsumeString() + return True + except ParseError: + return False + def ConsumeString(self): """Consumes a string value. @@ -746,7 +914,7 @@ class _Tokenizer(object): ParseError: If a byte array value couldn't be consumed. """ the_list = [self._ConsumeSingleByteString()] - while self.token and self.token[0] in ('\'', '"'): + while self.token and self.token[0] in _QUOTES: the_list.append(self._ConsumeSingleByteString()) return b''.join(the_list) @@ -757,11 +925,13 @@ class _Tokenizer(object): tokens which are automatically concatenated, like in C or Python. This method only consumes one token. + Returns: + The token parsed. Raises: ParseError: When the wrong format data is found. """ text = self.token - if len(text) < 1 or text[0] not in ('\'', '"'): + if len(text) < 1 or text[0] not in _QUOTES: raise self._ParseError('Expected string but found: %r' % (text,)) if len(text) < 2 or text[-1] != text[0]: diff --git a/python/setup.py b/python/setup.py index 18865e03..22f6e816 100755 --- a/python/setup.py +++ b/python/setup.py @@ -89,6 +89,7 @@ def GenerateUnittestProtos(): generate_proto("../src/google/protobuf/unittest_no_generic_services.proto", False) generate_proto("../src/google/protobuf/unittest_proto3_arena.proto", False) generate_proto("../src/google/protobuf/util/json_format_proto3.proto", False) + generate_proto("google/protobuf/internal/any_test.proto", False) generate_proto("google/protobuf/internal/descriptor_pool_test1.proto", False) generate_proto("google/protobuf/internal/descriptor_pool_test2.proto", False) generate_proto("google/protobuf/internal/factory_test1.proto", False) diff --git a/src/Makefile.am b/src/Makefile.am index b7d64093..d684c937 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -148,15 +148,16 @@ nobase_include_HEADERS = \ google/protobuf/compiler/plugin.h \ google/protobuf/compiler/plugin.pb.h \ google/protobuf/compiler/cpp/cpp_generator.h \ + google/protobuf/compiler/csharp/csharp_generator.h \ + google/protobuf/compiler/csharp/csharp_names.h \ google/protobuf/compiler/java/java_generator.h \ google/protobuf/compiler/java/java_names.h \ google/protobuf/compiler/javanano/javanano_generator.h \ + google/protobuf/compiler/js/js_generator.h \ google/protobuf/compiler/objectivec/objectivec_generator.h \ google/protobuf/compiler/objectivec/objectivec_helpers.h \ google/protobuf/compiler/python/python_generator.h \ google/protobuf/compiler/ruby/ruby_generator.h \ - google/protobuf/compiler/csharp/csharp_generator.h \ - google/protobuf/compiler/csharp/csharp_names.h \ google/protobuf/util/type_resolver.h \ google/protobuf/util/field_comparator.h \ google/protobuf/util/field_mask_util.h \ @@ -276,6 +277,8 @@ libprotobuf_la_SOURCES = \ google/protobuf/util/internal/protostream_objectsource.h \ google/protobuf/util/internal/protostream_objectwriter.cc \ google/protobuf/util/internal/protostream_objectwriter.h \ + google/protobuf/util/internal/proto_writer.cc \ + google/protobuf/util/internal/proto_writer.h \ google/protobuf/util/internal/snake2camel_objectwriter.h \ google/protobuf/util/internal/structured_objectwriter.h \ google/protobuf/util/internal/testdata \ @@ -386,6 +389,7 @@ libprotoc_la_SOURCES = \ google/protobuf/compiler/java/java_string_field_lite.h \ google/protobuf/compiler/java/java_doc_comment.cc \ google/protobuf/compiler/java/java_doc_comment.h \ + google/protobuf/compiler/js/js_generator.cc \ google/protobuf/compiler/javanano/javanano_enum.cc \ google/protobuf/compiler/javanano/javanano_enum.h \ google/protobuf/compiler/javanano/javanano_enum_field.cc \ diff --git a/src/google/protobuf/any.cc b/src/google/protobuf/any.cc index 7351d377..f3ca06bf 100644 --- a/src/google/protobuf/any.cc +++ b/src/google/protobuf/any.cc @@ -65,9 +65,16 @@ bool AnyMetadata::UnpackTo(Message* message) const { } bool AnyMetadata::InternalIs(const Descriptor* descriptor) const { - return type_url_->GetNoArena( - &::google::protobuf::internal::GetEmptyString()) == - GetTypeUrl(descriptor); + const string type_url = type_url_->GetNoArena( + &::google::protobuf::internal::GetEmptyString()); + const string full_name = descriptor->full_name(); + if (type_url.length() < full_name.length()) { + return false; + } + return (0 == type_url.compare( + type_url.length() - full_name.length(), + full_name.length(), + full_name)); } bool ParseAnyTypeUrl(const string& type_url, string* full_type_name) { diff --git a/src/google/protobuf/any.h b/src/google/protobuf/any.h index f760ad5d..c8dbef13 100644 --- a/src/google/protobuf/any.h +++ b/src/google/protobuf/any.h @@ -75,7 +75,7 @@ extern const char kTypeGoogleProdComPrefix[]; // "type.googleprod.com/". // Get the proto type name from Any::type_url value. For example, passing // "type.googleapis.com/rpc.QueryOrigin" will return "rpc.QueryOrigin" in // *full_type_name. Returns false if type_url does not start with -// "type.googleapis.com". +// "type.googleapis.com" or "type.googleprod.com". bool ParseAnyTypeUrl(const string& type_url, string* full_type_name); // See if message is of type google.protobuf.Any, if so, return the descriptors diff --git a/src/google/protobuf/any.pb.cc b/src/google/protobuf/any.pb.cc index 321fd315..0bf523b3 100644 --- a/src/google/protobuf/any.pb.cc +++ b/src/google/protobuf/any.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/any.proto b/src/google/protobuf/any.proto index 423699be..e8a18bc3 100644 --- a/src/google/protobuf/any.proto +++ b/src/google/protobuf/any.proto @@ -27,21 +27,22 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "AnyProto"; -option java_package = "com.google.protobuf"; option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option java_generate_equals_and_hash = true; option objc_class_prefix = "GPB"; - // `Any` contains an arbitrary serialized message along with a URL // that describes the type of the serialized message. // +// // JSON // ==== // The JSON representation of an `Any` value uses the regular @@ -62,8 +63,8 @@ option objc_class_prefix = "GPB"; // // If the embedded message type is well-known and has a custom JSON // representation, that representation will be embedded adding a field -// `value` which holds the custom JSON in addition to the the `@type` -// field. Example (for message [google.protobuf.Duration][google.protobuf.Duration]): +// `value` which holds the custom JSON in addition to the `@type` +// field. Example (for message [google.protobuf.Duration][]): // // { // "@type": "type.googleapis.com/google.protobuf.Duration", @@ -80,7 +81,7 @@ message Any { // * If no schema is provided, `https` is assumed. // * The last segment of the URL's path must represent the fully // qualified name of the type (as in `path/google.protobuf.Duration`). - // * An HTTP GET on the URL must yield a [google.protobuf.Type][google.protobuf.Type] + // * An HTTP GET on the URL must yield a [google.protobuf.Type][] // value in binary format, or produce an error. // * Applications are allowed to cache lookup results based on the // URL, or have them precompiled into a binary to avoid any diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index ed90702c..e589a89d 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/api.proto b/src/google/protobuf/api.proto index 597a6497..dbe87b8f 100644 --- a/src/google/protobuf/api.proto +++ b/src/google/protobuf/api.proto @@ -27,6 +27,7 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; @@ -34,11 +35,11 @@ package google.protobuf; import "google/protobuf/source_context.proto"; import "google/protobuf/type.proto"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option java_package = "com.google.protobuf"; option java_outer_classname = "ApiProto"; option java_multiple_files = true; option java_generate_equals_and_hash = true; -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option objc_class_prefix = "GPB"; // Api is a light-weight descriptor for a protocol buffer service. @@ -75,6 +76,7 @@ message Api { // be omitted. Zero major versions must only be used for // experimental, none-GA apis. // + // string version = 4; // Source context for the protocol buffer service represented by this @@ -141,7 +143,6 @@ message Method { // // package google.storage.v2; // service Storage { -// // (-- see AccessControl.GetAcl --) // rpc GetAcl(GetAclRequest) returns (Acl); // // // Get a data record. diff --git a/src/google/protobuf/arena.h b/src/google/protobuf/arena.h index 40c1e7c2..6a35183e 100644 --- a/src/google/protobuf/arena.h +++ b/src/google/protobuf/arena.h @@ -591,14 +591,12 @@ class LIBPROTOBUF_EXPORT Arena { // This is inside Arena because only Arena has the friend relationships // necessary to see the underlying generated code traits. template - struct is_destructor_skippable : - public google::protobuf::internal::integral_constant(static_cast(0))) == - sizeof(char) || - google::protobuf::internal::has_trivial_destructor::value> { - }; - + struct is_destructor_skippable + : public google::protobuf::internal::integral_constant< + bool, + sizeof(InternalIsDestructorSkippableHelper::DestructorSkippable< + const T>(static_cast(0))) == sizeof(char) || + google::protobuf::internal::has_trivial_destructor::value> {}; // CreateMessage requires that T supports arenas, but this private method // works whether or not T supports arenas. These are not exposed to user code diff --git a/src/google/protobuf/arenastring.h b/src/google/protobuf/arenastring.h index 1dacdc68..ef57033b 100755 --- a/src/google/protobuf/arenastring.h +++ b/src/google/protobuf/arenastring.h @@ -36,7 +36,6 @@ #include #include #include - #include #include diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 26a4f0b0..deb3d0f1 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -271,15 +271,35 @@ class CommandLineInterface::ErrorPrinter : public MultiFileErrorCollector, // implements MultiFileErrorCollector ------------------------------ void AddError(const string& filename, int line, int column, const string& message) { + AddErrorOrWarning(filename, line, column, message, "error", std::cerr); + } + + void AddWarning(const string& filename, int line, int column, + const string& message) { + AddErrorOrWarning(filename, line, column, message, "warning", std::clog); + } + // implements io::ErrorCollector ----------------------------------- + void AddError(int line, int column, const string& message) { + AddError("input", line, column, message); + } + + void AddWarning(int line, int column, const string& message) { + AddErrorOrWarning("input", line, column, message, "warning", std::clog); + } + + private: + void AddErrorOrWarning( + const string& filename, int line, int column, + const string& message, const string& type, ostream& out) { // Print full path when running under MSVS string dfile; if (format_ == CommandLineInterface::ERROR_FORMAT_MSVS && tree_ != NULL && tree_->VirtualFileToDiskFile(filename, &dfile)) { - std::cerr << dfile; + out << dfile; } else { - std::cerr << filename; + out << filename; } // Users typically expect 1-based line/column numbers, so we add 1 @@ -288,24 +308,22 @@ class CommandLineInterface::ErrorPrinter : public MultiFileErrorCollector, // Allow for both GCC- and Visual-Studio-compatible output. switch (format_) { case CommandLineInterface::ERROR_FORMAT_GCC: - std::cerr << ":" << (line + 1) << ":" << (column + 1); + out << ":" << (line + 1) << ":" << (column + 1); break; case CommandLineInterface::ERROR_FORMAT_MSVS: - std::cerr << "(" << (line + 1) - << ") : error in column=" << (column + 1); + out << "(" << (line + 1) << ") : " + << type << " in column=" << (column + 1); break; } } - std::cerr << ": " << message << std::endl; - } - - // implements io::ErrorCollector ----------------------------------- - void AddError(int line, int column, const string& message) { - AddError("input", line, column, message); + if (type == "warning") { + out << ": warning: " << message << std::endl; + } else { + out << ": " << message << std::endl; + } } - private: const ErrorFormat format_; DiskSourceTree *tree_; }; @@ -1353,7 +1371,7 @@ void CommandLineInterface::PrintHelpText() { " defined in the given proto files. Groups share\n" " the same field number space with the parent \n" " message. Extension ranges are counted as \n" -" occupied fields numbers." +" occupied fields numbers.\n" << std::endl; if (!plugin_prefix_.empty()) { std::cerr << @@ -1442,6 +1460,7 @@ bool CommandLineInterface::GenerateDependencyManifestFile( set already_seen; for (int i = 0; i < parsed_files.size(); i++) { GetTransitiveDependencies(parsed_files[i], + false, false, &already_seen, file_set.mutable_file()); @@ -1522,6 +1541,7 @@ bool CommandLineInterface::GeneratePluginOutput( for (int i = 0; i < parsed_files.size(); i++) { request.add_file_to_generate(parsed_files[i]->name()); GetTransitiveDependencies(parsed_files[i], + true, // Include json_name for plugins. true, // Include source code info. &already_seen, request.mutable_proto_file()); } @@ -1654,6 +1674,7 @@ bool CommandLineInterface::WriteDescriptorSet( set already_seen; for (int i = 0; i < parsed_files.size(); i++) { GetTransitiveDependencies(parsed_files[i], + true, // Include json_name source_info_in_descriptor_set_, &already_seen, file_set.mutable_file()); } @@ -1665,6 +1686,7 @@ bool CommandLineInterface::WriteDescriptorSet( } FileDescriptorProto* file_proto = file_set.add_file(); parsed_files[i]->CopyTo(file_proto); + parsed_files[i]->CopyJsonNameTo(file_proto); if (source_info_in_descriptor_set_) { parsed_files[i]->CopySourceCodeInfoTo(file_proto); } @@ -1699,7 +1721,9 @@ bool CommandLineInterface::WriteDescriptorSet( } void CommandLineInterface::GetTransitiveDependencies( - const FileDescriptor* file, bool include_source_code_info, + const FileDescriptor* file, + bool include_json_name, + bool include_source_code_info, set* already_seen, RepeatedPtrField* output) { if (!already_seen->insert(file).second) { @@ -1709,13 +1733,18 @@ void CommandLineInterface::GetTransitiveDependencies( // Add all dependencies. for (int i = 0; i < file->dependency_count(); i++) { - GetTransitiveDependencies(file->dependency(i), include_source_code_info, + GetTransitiveDependencies(file->dependency(i), + include_json_name, + include_source_code_info, already_seen, output); } // Add this file. FileDescriptorProto* new_descriptor = output->Add(); file->CopyTo(new_descriptor); + if (include_json_name) { + file->CopyJsonNameTo(new_descriptor); + } if (include_source_code_info) { file->CopySourceCodeInfoTo(new_descriptor); } diff --git a/src/google/protobuf/compiler/command_line_interface.h b/src/google/protobuf/compiler/command_line_interface.h index 7e611c44..f196ffc5 100644 --- a/src/google/protobuf/compiler/command_line_interface.h +++ b/src/google/protobuf/compiler/command_line_interface.h @@ -262,8 +262,11 @@ class LIBPROTOC_EXPORT CommandLineInterface { // in order. Any files in *already_seen will not be added, and each file // added will be inserted into *already_seen. If include_source_code_info is // true then include the source code information in the FileDescriptorProtos. + // If include_json_name is true, populate the json_name field of + // FieldDescriptorProto for all fields. static void GetTransitiveDependencies( const FileDescriptor* file, + bool include_json_name, bool include_source_code_info, set* already_seen, RepeatedPtrField* output); diff --git a/src/google/protobuf/compiler/command_line_interface_unittest.cc b/src/google/protobuf/compiler/command_line_interface_unittest.cc index fa792dae..4ce81a75 100644 --- a/src/google/protobuf/compiler/command_line_interface_unittest.cc +++ b/src/google/protobuf/compiler/command_line_interface_unittest.cc @@ -886,6 +886,10 @@ TEST_F(CommandLineInterfaceTest, WriteDescriptorSet) { EXPECT_EQ("bar.proto", descriptor_set.file(0).name()); // Descriptor set should not have source code info. EXPECT_FALSE(descriptor_set.file(0).has_source_code_info()); + // Descriptor set should have json_name. + EXPECT_EQ("Bar", descriptor_set.file(0).message_type(0).name()); + EXPECT_EQ("foo", descriptor_set.file(0).message_type(0).field(0).name()); + EXPECT_TRUE(descriptor_set.file(0).message_type(0).field(0).has_json_name()); } TEST_F(CommandLineInterfaceTest, WriteDescriptorSetWithDuplicates) { @@ -919,6 +923,10 @@ TEST_F(CommandLineInterfaceTest, WriteDescriptorSetWithDuplicates) { EXPECT_EQ("baz.proto", descriptor_set.file(2).name()); // Descriptor set should not have source code info. EXPECT_FALSE(descriptor_set.file(0).has_source_code_info()); + // Descriptor set should have json_name. + EXPECT_EQ("Bar", descriptor_set.file(0).message_type(0).name()); + EXPECT_EQ("foo", descriptor_set.file(0).message_type(0).field(0).name()); + EXPECT_TRUE(descriptor_set.file(0).message_type(0).field(0).has_json_name()); } TEST_F(CommandLineInterfaceTest, WriteDescriptorSetWithSourceInfo) { @@ -1413,6 +1421,18 @@ TEST_F(CommandLineInterfaceTest, PluginReceivesSourceCodeInfo) { "Saw message type MockCodeGenerator_HasSourceCodeInfo: 1."); } +TEST_F(CommandLineInterfaceTest, PluginReceivesJsonName) { + CreateTempFile("foo.proto", + "syntax = \"proto2\";\n" + "message MockCodeGenerator_HasJsonName {\n" + " optional int32 value = 1;\n" + "}\n"); + + Run("protocol_compiler --plug_out=$tmpdir --proto_path=$tmpdir foo.proto"); + + ExpectErrorSubstring("Saw json_name: 1"); +} + TEST_F(CommandLineInterfaceTest, GeneratorPluginNotFound) { // Test what happens if the plugin isn't found. diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index 9aa41534..37e4bae4 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -252,6 +252,7 @@ void FileGenerator::GenerateSource(io::Printer* printer) { "#include \n" // for swap() "\n" "#include \n" + "#include \n" "#include \n" "#include \n" "#include \n", diff --git a/src/google/protobuf/compiler/importer.cc b/src/google/protobuf/compiler/importer.cc index 8333684e..0d9093c0 100644 --- a/src/google/protobuf/compiler/importer.cc +++ b/src/google/protobuf/compiler/importer.cc @@ -185,6 +185,19 @@ void SourceTreeDescriptorDatabase::ValidationErrorCollector::AddError( owner_->error_collector_->AddError(filename, line, column, message); } +void SourceTreeDescriptorDatabase::ValidationErrorCollector::AddWarning( + const string& filename, + const string& element_name, + const Message* descriptor, + ErrorLocation location, + const string& message) { + if (owner_->error_collector_ == NULL) return; + + int line, column; + owner_->source_locations_.Find(descriptor, location, &line, &column); + owner_->error_collector_->AddWarning(filename, line, column, message); +} + // =================================================================== Importer::Importer(SourceTree* source_tree, diff --git a/src/google/protobuf/compiler/importer.h b/src/google/protobuf/compiler/importer.h index f010fd08..cc8fcc39 100644 --- a/src/google/protobuf/compiler/importer.h +++ b/src/google/protobuf/compiler/importer.h @@ -121,6 +121,12 @@ class LIBPROTOBUF_EXPORT SourceTreeDescriptorDatabase : public DescriptorDatabas ErrorLocation location, const string& message); + virtual void AddWarning(const string& filename, + const string& element_name, + const Message* descriptor, + ErrorLocation location, + const string& message); + private: SourceTreeDescriptorDatabase* owner_; }; @@ -188,6 +194,9 @@ class LIBPROTOBUF_EXPORT MultiFileErrorCollector { virtual void AddError(const string& filename, int line, int column, const string& message) = 0; + virtual void AddWarning(const string& filename, int line, int column, + const string& message) {} + private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MultiFileErrorCollector); }; diff --git a/src/google/protobuf/compiler/importer_unittest.cc b/src/google/protobuf/compiler/importer_unittest.cc index 33c9328f..be19aa2e 100644 --- a/src/google/protobuf/compiler/importer_unittest.cc +++ b/src/google/protobuf/compiler/importer_unittest.cc @@ -71,6 +71,7 @@ class MockErrorCollector : public MultiFileErrorCollector { ~MockErrorCollector() {} string text_; + string warning_text_; // implements ErrorCollector --------------------------------------- void AddError(const string& filename, int line, int column, @@ -78,6 +79,12 @@ class MockErrorCollector : public MultiFileErrorCollector { strings::SubstituteAndAppend(&text_, "$0:$1:$2: $3\n", filename, line, column, message); } + + void AddWarning(const string& filename, int line, int column, + const string& message) { + strings::SubstituteAndAppend(&warning_text_, "$0:$1:$2: $3\n", + filename, line, column, message); + } }; // ------------------------------------------------------------------- @@ -123,6 +130,7 @@ class ImporterTest : public testing::Test { // Return the collected error text string error() const { return error_collector_.text_; } + string warning() const { return error_collector_.warning_text_; } MockErrorCollector error_collector_; MockSourceTree source_tree_; diff --git a/src/google/protobuf/compiler/java/java_enum.cc b/src/google/protobuf/compiler/java/java_enum.cc index 8a09f3a8..5fc9b002 100644 --- a/src/google/protobuf/compiler/java/java_enum.cc +++ b/src/google/protobuf/compiler/java/java_enum.cc @@ -85,17 +85,10 @@ EnumGenerator::~EnumGenerator() {} void EnumGenerator::Generate(io::Printer* printer) { WriteEnumDocComment(printer, descriptor_); - if (HasDescriptorMethods(descriptor_)) { - printer->Print( - "public enum $classname$\n" - " implements com.google.protobuf.ProtocolMessageEnum {\n", - "classname", descriptor_->name()); - } else { - printer->Print( - "public enum $classname$\n" - " implements com.google.protobuf.Internal.EnumLite {\n", - "classname", descriptor_->name()); - } + printer->Print( + "public enum $classname$\n" + " implements com.google.protobuf.ProtocolMessageEnum {\n", + "classname", descriptor_->name()); printer->Indent(); for (int i = 0; i < canonical_values_.size(); i++) { @@ -311,7 +304,6 @@ void EnumGenerator::Generate(io::Printer* printer) { "}\n" "\n"); - // index is only used for reflection; lite implementation does not need it printer->Print("private final int index;\n"); } diff --git a/src/google/protobuf/compiler/java/java_enum_field_lite.cc b/src/google/protobuf/compiler/java/java_enum_field_lite.cc index e8bf15d0..e3e87c58 100644 --- a/src/google/protobuf/compiler/java/java_enum_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_enum_field_lite.cc @@ -596,9 +596,7 @@ GenerateInterfaceMembers(io::Printer* printer) const { void RepeatedImmutableEnumFieldLiteGenerator:: GenerateMembers(io::Printer* printer) const { printer->Print(variables_, - // TODO(dweis): Switch to IntList? - "private com.google.protobuf.Internal.ProtobufList<\n" - " java.lang.Integer> $name$_;\n" + "private com.google.protobuf.Internal.IntList $name$_;\n" "private static final com.google.protobuf.Internal.ListAdapter.Converter<\n" " java.lang.Integer, $type$> $name$_converter_ =\n" " new com.google.protobuf.Internal.ListAdapter.Converter<\n" @@ -623,7 +621,7 @@ GenerateMembers(io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$public $type$ get$capitalized_name$(int index) {\n" - " return $name$_converter_.convert($name$_.get(index));\n" + " return $name$_converter_.convert($name$_.getInt(index));\n" "}\n"); if (SupportUnknownEnumValue(descriptor_->file())) { WriteFieldDocComment(printer, descriptor_); @@ -635,7 +633,7 @@ GenerateMembers(io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$public int get$capitalized_name$Value(int index) {\n" - " return $name$_.get(index);\n" + " return $name$_.getInt(index);\n" "}\n"); } @@ -649,7 +647,7 @@ GenerateMembers(io::Printer* printer) const { printer->Print(variables_, "private void ensure$capitalized_name$IsMutable() {\n" " if (!$is_mutable$) {\n" - " $name$_ = newProtobufList($name$_);\n" + " $name$_ = newIntList($name$_);\n" " }\n" "}\n"); WriteFieldDocComment(printer, descriptor_); @@ -660,7 +658,7 @@ GenerateMembers(io::Printer* printer) const { " throw new NullPointerException();\n" " }\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.set(index, value.getNumber());\n" + " $name$_.setInt(index, value.getNumber());\n" "}\n"); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -669,7 +667,7 @@ GenerateMembers(io::Printer* printer) const { " throw new NullPointerException();\n" " }\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.add(value.getNumber());\n" + " $name$_.addInt(value.getNumber());\n" "}\n"); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -677,13 +675,13 @@ GenerateMembers(io::Printer* printer) const { " java.lang.Iterable values) {\n" " ensure$capitalized_name$IsMutable();\n" " for ($type$ value : values) {\n" - " $name$_.add(value.getNumber());\n" + " $name$_.addInt(value.getNumber());\n" " }\n" "}\n"); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "private void clear$capitalized_name$() {\n" - " $name$_ = emptyProtobufList();\n" + " $name$_ = emptyIntList();\n" "}\n"); if (SupportUnknownEnumValue(descriptor_->file())) { @@ -692,13 +690,13 @@ GenerateMembers(io::Printer* printer) const { "private void set$capitalized_name$Value(\n" " int index, int value) {\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.set(index, value);\n" + " $name$_.setInt(index, value);\n" "}\n"); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "private void add$capitalized_name$Value(int value) {\n" " ensure$capitalized_name$IsMutable();\n" - " $name$_.add(value);\n" + " $name$_.addInt(value);\n" "}\n"); WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, @@ -706,7 +704,7 @@ GenerateMembers(io::Printer* printer) const { " java.lang.Iterable values) {\n" " ensure$capitalized_name$IsMutable();\n" " for (int value : values) {\n" - " $name$_.add(value);\n" + " $name$_.addInt(value);\n" " }\n" "}\n"); } @@ -805,7 +803,7 @@ GenerateFieldBuilderInitializationCode(io::Printer* printer) const { void RepeatedImmutableEnumFieldLiteGenerator:: GenerateInitializationCode(io::Printer* printer) const { - printer->Print(variables_, "$name$_ = emptyProtobufList();\n"); + printer->Print(variables_, "$name$_ = emptyIntList();\n"); } void RepeatedImmutableEnumFieldLiteGenerator:: @@ -840,9 +838,9 @@ GenerateParsingCode(io::Printer* printer) const { printer->Print(variables_, "int rawValue = input.readEnum();\n" "if (!$is_mutable$) {\n" - " $name$_ = newProtobufList();\n" + " $name$_ = newIntList();\n" "}\n" - "$name$_.add(rawValue);\n"); + "$name$_.addInt(rawValue);\n"); } else { printer->Print(variables_, "int rawValue = input.readEnum();\n" @@ -855,9 +853,9 @@ GenerateParsingCode(io::Printer* printer) const { printer->Print(variables_, "} else {\n" " if (!$is_mutable$) {\n" - " $name$_ = newProtobufList();\n" + " $name$_ = newIntList();\n" " }\n" - " $name$_.add(rawValue);\n" + " $name$_.addInt(rawValue);\n" "}\n"); } } @@ -897,12 +895,12 @@ GenerateSerializationCode(io::Printer* printer) const { " output.writeRawVarint32($name$MemoizedSerializedSize);\n" "}\n" "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.writeEnumNoTag($name$_.get(i));\n" + " output.writeEnumNoTag($name$_.getInt(i));\n" "}\n"); } else { printer->Print(variables_, "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.writeEnum($number$, $name$_.get(i));\n" + " output.writeEnum($number$, $name$_.getInt(i));\n" "}\n"); } } @@ -917,7 +915,7 @@ GenerateSerializedSizeCode(io::Printer* printer) const { printer->Print(variables_, "for (int i = 0; i < $name$_.size(); i++) {\n" " dataSize += com.google.protobuf.CodedOutputStream\n" - " .computeEnumSizeNoTag($name$_.get(i));\n" + " .computeEnumSizeNoTag($name$_.getInt(i));\n" "}\n"); printer->Print( "size += dataSize;\n"); diff --git a/src/google/protobuf/compiler/java/java_enum_lite.cc b/src/google/protobuf/compiler/java/java_enum_lite.cc index 62186386..ed415eee 100644 --- a/src/google/protobuf/compiler/java/java_enum_lite.cc +++ b/src/google/protobuf/compiler/java/java_enum_lite.cc @@ -85,34 +85,26 @@ EnumLiteGenerator::~EnumLiteGenerator() {} void EnumLiteGenerator::Generate(io::Printer* printer) { WriteEnumDocComment(printer, descriptor_); - if (HasDescriptorMethods(descriptor_)) { - printer->Print( - "public enum $classname$\n" - " implements com.google.protobuf.ProtocolMessageEnum {\n", - "classname", descriptor_->name()); - } else { - printer->Print( - "public enum $classname$\n" - " implements com.google.protobuf.Internal.EnumLite {\n", - "classname", descriptor_->name()); - } + printer->Print( + "public enum $classname$\n" + " implements com.google.protobuf.Internal.EnumLite {\n", + "classname", descriptor_->name()); printer->Indent(); for (int i = 0; i < canonical_values_.size(); i++) { map vars; vars["name"] = canonical_values_[i]->name(); - vars["index"] = SimpleItoa(canonical_values_[i]->index()); vars["number"] = SimpleItoa(canonical_values_[i]->number()); WriteEnumValueDocComment(printer, canonical_values_[i]); if (canonical_values_[i]->options().deprecated()) { printer->Print("@java.lang.Deprecated\n"); } printer->Print(vars, - "$name$($index$, $number$),\n"); + "$name$($number$),\n"); } if (SupportUnknownEnumValue(descriptor_->file())) { - printer->Print("UNRECOGNIZED(-1, -1),\n"); + printer->Print("UNRECOGNIZED(-1),\n"); } printer->Print( @@ -145,15 +137,7 @@ void EnumLiteGenerator::Generate(io::Printer* printer) { printer->Print( "\n" - "public final int getNumber() {\n"); - if (SupportUnknownEnumValue(descriptor_->file())) { - printer->Print( - " if (index == -1) {\n" - " throw new java.lang.IllegalArgumentException(\n" - " \"Can't get the number of an unknown enum value.\");\n" - " }\n"); - } - printer->Print( + "public final int getNumber() {\n" " return value;\n" "}\n" "\n" @@ -193,7 +177,7 @@ void EnumLiteGenerator::Generate(io::Printer* printer) { printer->Print( "private final int value;\n\n" - "private $classname$(int index, int value) {\n", + "private $classname$(int value) {\n", "classname", descriptor_->name()); printer->Print( " this.value = value;\n" diff --git a/src/google/protobuf/compiler/java/java_file.cc b/src/google/protobuf/compiler/java/java_file.cc index 68b47ee1..c8172330 100644 --- a/src/google/protobuf/compiler/java/java_file.cc +++ b/src/google/protobuf/compiler/java/java_file.cc @@ -42,6 +42,7 @@ #include #include +#include #include #include #include @@ -281,8 +282,13 @@ void FileGenerator::Generate(io::Printer* printer) { if (!MultipleJavaFiles(file_, immutable_api_)) { for (int i = 0; i < file_->enum_type_count(); i++) { - EnumGenerator(file_->enum_type(i), immutable_api_, context_.get()) - .Generate(printer); + if (HasDescriptorMethods(file_)) { + EnumGenerator(file_->enum_type(i), immutable_api_, context_.get()) + .Generate(printer); + } else { + EnumLiteGenerator(file_->enum_type(i), immutable_api_, context_.get()) + .Generate(printer); + } } for (int i = 0; i < file_->message_type_count(); i++) { message_generators_[i]->GenerateInterface(printer); @@ -543,13 +549,23 @@ void FileGenerator::GenerateSiblings(const string& package_dir, vector* file_list) { if (MultipleJavaFiles(file_, immutable_api_)) { for (int i = 0; i < file_->enum_type_count(); i++) { - EnumGenerator generator(file_->enum_type(i), immutable_api_, - context_.get()); - GenerateSibling(package_dir, java_package_, - file_->enum_type(i), - context, file_list, "", - &generator, - &EnumGenerator::Generate); + if (HasDescriptorMethods(file_)) { + EnumGenerator generator(file_->enum_type(i), immutable_api_, + context_.get()); + GenerateSibling(package_dir, java_package_, + file_->enum_type(i), + context, file_list, "", + &generator, + &EnumGenerator::Generate); + } else { + EnumLiteGenerator generator(file_->enum_type(i), immutable_api_, + context_.get()); + GenerateSibling(package_dir, java_package_, + file_->enum_type(i), + context, file_list, "", + &generator, + &EnumLiteGenerator::Generate); + } } for (int i = 0; i < file_->message_type_count(); i++) { if (immutable_api_) { diff --git a/src/google/protobuf/compiler/java/java_primitive_field_lite.cc b/src/google/protobuf/compiler/java/java_primitive_field_lite.cc index 392333b8..5a7bf82d 100644 --- a/src/google/protobuf/compiler/java/java_primitive_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_primitive_field_lite.cc @@ -87,11 +87,13 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor, (*variables)["field_list_type"] = "com.google.protobuf.Internal." + capitalized_type + "List"; (*variables)["new_list"] = "new" + capitalized_type + "List"; + (*variables)["new_list_with_capacity"] = + "new" + capitalized_type + "ListWithCapacity"; (*variables)["empty_list"] = "empty" + capitalized_type + "List()"; (*variables)["make_name_unmodifiable"] = (*variables)["name"] + "_.makeImmutable()"; - (*variables)["repeated_index_get"] = - (*variables)["name"] + "_.get" + capitalized_type + "(index)"; + (*variables)["repeated_get"] = + (*variables)["name"] + "_.get" + capitalized_type; (*variables)["repeated_add"] = (*variables)["name"] + "_.add" + capitalized_type; (*variables)["repeated_set"] = @@ -102,11 +104,11 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor, "com.google.protobuf.Internal.ProtobufList<" + (*variables)["boxed_type"] + ">"; (*variables)["new_list"] = "newProtobufList"; + (*variables)["new_list_with_capacity"] = "newProtobufListWithCapacity"; (*variables)["empty_list"] = "emptyProtobufList()"; (*variables)["make_name_unmodifiable"] = (*variables)["name"] + "_.makeImmutable()"; - (*variables)["repeated_index_get"] = - (*variables)["name"] + "_.get(index)"; + (*variables)["repeated_get"] = (*variables)["name"] + "_.get"; (*variables)["repeated_add"] = (*variables)["name"] + "_.add"; (*variables)["repeated_set"] = (*variables)["name"] + "_.set"; } @@ -629,7 +631,7 @@ GenerateMembers(io::Printer* printer) const { WriteFieldDocComment(printer, descriptor_); printer->Print(variables_, "$deprecation$public $type$ get$capitalized_name$(int index) {\n" - " return $repeated_index_get$;\n" + " return $repeated_get$(index);\n" "}\n"); if (descriptor_->options().packed() && @@ -773,6 +775,9 @@ GenerateDynamicMethodMakeImmutableCode(io::Printer* printer) const { void RepeatedImmutablePrimitiveFieldLiteGenerator:: GenerateParsingCode(io::Printer* printer) const { + // TODO(dweis): Scan the input buffer to count, then initialize + // appropriately. + // TODO(dweis): Scan the input buffer to count and ensure capacity. printer->Print(variables_, "if (!$is_mutable$) {\n" " $name$_ = $new_list$();\n" @@ -785,8 +790,21 @@ GenerateParsingCodeFromPacked(io::Printer* printer) const { printer->Print(variables_, "int length = input.readRawVarint32();\n" "int limit = input.pushLimit(length);\n" - "if (!$is_mutable$ && input.getBytesUntilLimit() > 0) {\n" - " $name$_ = $new_list$();\n" + "if (!$is_mutable$ && input.getBytesUntilLimit() > 0) {\n"); + + int fixed_size = FixedSize(GetType(descriptor_)); + if (fixed_size == -1) { + // TODO(dweis): Scan the input buffer to count, then initialize + // appropriately. + printer->Print(variables_, + " $name$_ = $new_list$();\n"); + } else { + printer->Print(variables_, + " $name$_ = $new_list_with_capacity$(length/$fixed_size$);\n"); + } + + // TODO(dweis): Scan the input buffer to count and ensure capacity. + printer->Print(variables_, "}\n" "while (input.getBytesUntilLimit() > 0) {\n" " $repeated_add$(input.read$capitalized_type$());\n" @@ -814,12 +832,12 @@ GenerateSerializationCode(io::Printer* printer) const { " output.writeRawVarint32($name$MemoizedSerializedSize);\n" "}\n" "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.write$capitalized_type$NoTag($name$_.get(i));\n" + " output.write$capitalized_type$NoTag($repeated_get$(i));\n" "}\n"); } else { printer->Print(variables_, "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.write$capitalized_type$($number$, $name$_.get(i));\n" + " output.write$capitalized_type$($number$, $repeated_get$(i));\n" "}\n"); } } @@ -835,7 +853,7 @@ GenerateSerializedSizeCode(io::Printer* printer) const { printer->Print(variables_, "for (int i = 0; i < $name$_.size(); i++) {\n" " dataSize += com.google.protobuf.CodedOutputStream\n" - " .compute$capitalized_type$SizeNoTag($name$_.get(i));\n" + " .compute$capitalized_type$SizeNoTag($repeated_get$(i));\n" "}\n"); } else { printer->Print(variables_, diff --git a/src/google/protobuf/compiler/java/java_string_field.cc b/src/google/protobuf/compiler/java/java_string_field.cc index 72ebaeca..7f757e47 100644 --- a/src/google/protobuf/compiler/java/java_string_field.cc +++ b/src/google/protobuf/compiler/java/java_string_field.cc @@ -405,7 +405,7 @@ void ImmutableStringFieldGenerator:: GenerateParsingCode(io::Printer* printer) const { if (CheckUtf8(descriptor_)) { printer->Print(variables_, - "String s = input.readStringRequireUtf8();\n" + "java.lang.String s = input.readStringRequireUtf8();\n" "$set_has_field_bit_message$\n" "$name$_ = s;\n"); } else if (!HasDescriptorMethods(descriptor_->file())) { @@ -414,7 +414,7 @@ GenerateParsingCode(io::Printer* printer) const { // spurious intermediary ByteString allocations, cutting overall allocations // in half. printer->Print(variables_, - "String s = input.readString();\n" + "java.lang.String s = input.readString();\n" "$set_has_field_bit_message$\n" "$name$_ = s;\n"); } else { @@ -665,7 +665,7 @@ void ImmutableStringOneofFieldGenerator:: GenerateParsingCode(io::Printer* printer) const { if (CheckUtf8(descriptor_)) { printer->Print(variables_, - "String s = input.readStringRequireUtf8();\n" + "java.lang.String s = input.readStringRequireUtf8();\n" "$set_oneof_case_message$;\n" "$oneof_name$_ = s;\n"); } else if (!HasDescriptorMethods(descriptor_->file())) { @@ -674,7 +674,7 @@ GenerateParsingCode(io::Printer* printer) const { // spurious intermediary ByteString allocations, cutting overall allocations // in half. printer->Print(variables_, - "String s = input.readString();\n" + "java.lang.String s = input.readString();\n" "$set_oneof_case_message$;\n" "$oneof_name$_ = s;\n"); } else { @@ -773,12 +773,6 @@ GenerateMembers(io::Printer* printer) const { " get$capitalized_name$Bytes(int index) {\n" " return $name$_.getByteString(index);\n" "}\n"); - - if (descriptor_->options().packed() && - HasGeneratedMethods(descriptor_->containing_type())) { - printer->Print(variables_, - "private int $name$MemoizedSerializedSize = -1;\n"); - } } void RepeatedImmutableStringFieldGenerator:: @@ -939,14 +933,14 @@ void RepeatedImmutableStringFieldGenerator:: GenerateParsingCode(io::Printer* printer) const { if (CheckUtf8(descriptor_)) { printer->Print(variables_, - "String s = input.readStringRequireUtf8();\n"); + "java.lang.String s = input.readStringRequireUtf8();\n"); } else if (!HasDescriptorMethods(descriptor_->file())) { // Lite runtime should attempt to reduce allocations by attempting to // construct the string directly from the input stream buffer. This avoids // spurious intermediary ByteString allocations, cutting overall allocations // in half. printer->Print(variables_, - "String s = input.readString();\n"); + "java.lang.String s = input.readString();\n"); } else { printer->Print(variables_, "com.google.protobuf.ByteString bs = input.readBytes();\n"); @@ -965,30 +959,6 @@ GenerateParsingCode(io::Printer* printer) const { } } -void RepeatedImmutableStringFieldGenerator:: -GenerateParsingCodeFromPacked(io::Printer* printer) const { - printer->Print(variables_, - "int length = input.readRawVarint32();\n" - "int limit = input.pushLimit(length);\n" - "if (!$get_mutable_bit_parser$ && input.getBytesUntilLimit() > 0) {\n" - " $name$_ = new com.google.protobuf.LazyStringArrayList();\n" - " $set_mutable_bit_parser$;\n" - "}\n" - "while (input.getBytesUntilLimit() > 0) {\n"); - if (CheckUtf8(descriptor_)) { - printer->Print(variables_, - " String s = input.readStringRequireUtf8();\n"); - } else { - printer->Print(variables_, - " String s = input.readString();\n"); - } - printer->Print(variables_, - " $name$.add(s);\n"); - printer->Print(variables_, - "}\n" - "input.popLimit(limit);\n"); -} - void RepeatedImmutableStringFieldGenerator:: GenerateParsingDoneCode(io::Printer* printer) const { printer->Print(variables_, @@ -999,21 +969,10 @@ GenerateParsingDoneCode(io::Printer* printer) const { void RepeatedImmutableStringFieldGenerator:: GenerateSerializationCode(io::Printer* printer) const { - if (descriptor_->options().packed()) { - printer->Print(variables_, - "if (get$capitalized_name$List().size() > 0) {\n" - " output.writeRawVarint32($tag$);\n" - " output.writeRawVarint32($name$MemoizedSerializedSize);\n" - "}\n" - "for (int i = 0; i < $name$_.size(); i++) {\n" - " writeStringNoTag(output, $name$_.getRaw(i));\n" - "}\n"); - } else { - printer->Print(variables_, - "for (int i = 0; i < $name$_.size(); i++) {\n" - " $writeString$(output, $number$, $name$_.getRaw(i));\n" - "}\n"); - } + printer->Print(variables_, + "for (int i = 0; i < $name$_.size(); i++) {\n" + " $writeString$(output, $number$, $name$_.getRaw(i));\n" + "}\n"); } void RepeatedImmutableStringFieldGenerator:: @@ -1031,23 +990,8 @@ GenerateSerializedSizeCode(io::Printer* printer) const { printer->Print( "size += dataSize;\n"); - if (descriptor_->options().packed()) { - printer->Print(variables_, - "if (!get$capitalized_name$List().isEmpty()) {\n" - " size += $tag_size$;\n" - " size += com.google.protobuf.CodedOutputStream\n" - " .computeInt32SizeNoTag(dataSize);\n" - "}\n"); - } else { - printer->Print(variables_, - "size += $tag_size$ * get$capitalized_name$List().size();\n"); - } - - // cache the data size for packed fields. - if (descriptor_->options().packed()) { - printer->Print(variables_, - "$name$MemoizedSerializedSize = dataSize;\n"); - } + printer->Print(variables_, + "size += $tag_size$ * get$capitalized_name$List().size();\n"); printer->Outdent(); printer->Print("}\n"); diff --git a/src/google/protobuf/compiler/java/java_string_field.h b/src/google/protobuf/compiler/java/java_string_field.h index 1ea44dec..a3b57351 100644 --- a/src/google/protobuf/compiler/java/java_string_field.h +++ b/src/google/protobuf/compiler/java/java_string_field.h @@ -131,7 +131,6 @@ class RepeatedImmutableStringFieldGenerator : public ImmutableFieldGenerator { void GenerateMergingCode(io::Printer* printer) const; void GenerateBuildingCode(io::Printer* printer) const; void GenerateParsingCode(io::Printer* printer) const; - void GenerateParsingCodeFromPacked(io::Printer* printer) const; void GenerateParsingDoneCode(io::Printer* printer) const; void GenerateSerializationCode(io::Printer* printer) const; void GenerateSerializedSizeCode(io::Printer* printer) const; diff --git a/src/google/protobuf/compiler/java/java_string_field_lite.cc b/src/google/protobuf/compiler/java/java_string_field_lite.cc index 092e3c29..eb5964bd 100644 --- a/src/google/protobuf/compiler/java/java_string_field_lite.cc +++ b/src/google/protobuf/compiler/java/java_string_field_lite.cc @@ -647,12 +647,6 @@ GenerateMembers(io::Printer* printer) const { " $name$_.get(index));\n" "}\n"); - if (descriptor_->options().packed() && - HasGeneratedMethods(descriptor_->containing_type())) { - printer->Print(variables_, - "private int $name$MemoizedSerializedSize = -1;\n"); - } - printer->Print(variables_, "private void ensure$capitalized_name$IsMutable() {\n" " if (!$is_mutable$) {\n" @@ -834,29 +828,6 @@ GenerateParsingCode(io::Printer* printer) const { } } -void RepeatedImmutableStringFieldLiteGenerator:: -GenerateParsingCodeFromPacked(io::Printer* printer) const { - printer->Print(variables_, - "int length = input.readRawVarint32();\n" - "int limit = input.pushLimit(length);\n" - "if (!$is_mutable$ && input.getBytesUntilLimit() > 0) {\n" - " $name$_ = com.google.protobuf.GeneratedMessageLite.newProtobufList();\n" - "}\n" - "while (input.getBytesUntilLimit() > 0) {\n"); - if (CheckUtf8(descriptor_)) { - printer->Print(variables_, - " String s = input.readStringRequireUtf8();\n"); - } else { - printer->Print(variables_, - " String s = input.readString();\n"); - } - printer->Print(variables_, - " $name$.add(s);\n"); - printer->Print(variables_, - "}\n" - "input.popLimit(limit);\n"); -} - void RepeatedImmutableStringFieldLiteGenerator:: GenerateParsingDoneCode(io::Printer* printer) const { printer->Print(variables_, @@ -870,21 +841,10 @@ GenerateSerializationCode(io::Printer* printer) const { // Lite runtime should reduce allocations by serializing the string directly. // This avoids spurious intermediary ByteString allocations, cutting overall // allocations in half. - if (descriptor_->options().packed()) { - printer->Print(variables_, - "if (get$capitalized_name$List().size() > 0) {\n" - " output.writeRawVarint32($tag$);\n" - " output.writeRawVarint32($name$MemoizedSerializedSize);\n" - "}\n" - "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.writeStringNoTag($name$_.get(i));\n" - "}\n"); - } else { - printer->Print(variables_, - "for (int i = 0; i < $name$_.size(); i++) {\n" - " output.writeString($number$, $name$_.get(i));\n" - "}\n"); - } + printer->Print(variables_, + "for (int i = 0; i < $name$_.size(); i++) {\n" + " output.writeString($number$, $name$_.get(i));\n" + "}\n"); } void RepeatedImmutableStringFieldLiteGenerator:: @@ -906,23 +866,9 @@ GenerateSerializedSizeCode(io::Printer* printer) const { printer->Print( "size += dataSize;\n"); - if (descriptor_->options().packed()) { - printer->Print(variables_, - "if (!get$capitalized_name$List().isEmpty()) {\n" - " size += $tag_size$;\n" - " size += com.google.protobuf.CodedOutputStream\n" - " .computeInt32SizeNoTag(dataSize);\n" - "}\n"); - } else { - printer->Print(variables_, - "size += $tag_size$ * get$capitalized_name$List().size();\n"); - } - // cache the data size for packed fields. - if (descriptor_->options().packed()) { - printer->Print(variables_, - "$name$MemoizedSerializedSize = dataSize;\n"); - } + printer->Print(variables_, + "size += $tag_size$ * get$capitalized_name$List().size();\n"); printer->Outdent(); printer->Print("}\n"); diff --git a/src/google/protobuf/compiler/java/java_string_field_lite.h b/src/google/protobuf/compiler/java/java_string_field_lite.h index 9d93b307..4d9b4bd7 100644 --- a/src/google/protobuf/compiler/java/java_string_field_lite.h +++ b/src/google/protobuf/compiler/java/java_string_field_lite.h @@ -129,7 +129,6 @@ class RepeatedImmutableStringFieldLiteGenerator void GenerateMergingCode(io::Printer* printer) const; void GenerateDynamicMethodMakeImmutableCode(io::Printer* printer) const; void GenerateParsingCode(io::Printer* printer) const; - void GenerateParsingCodeFromPacked(io::Printer* printer) const; void GenerateParsingDoneCode(io::Printer* printer) const; void GenerateSerializationCode(io::Printer* printer) const; void GenerateSerializedSizeCode(io::Printer* printer) const; diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc new file mode 100755 index 00000000..1e308464 --- /dev/null +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -0,0 +1,2620 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include +#include +#include +#include +#ifndef _SHARED_PTR_H +#include +#endif +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace js { + +// Sorted list of JavaScript keywords. These cannot be used as names. If they +// appear, we prefix them with "pb_". +const char* kKeyword[] = { + "abstract", + "boolean", + "break", + "byte", + "case", + "catch", + "char", + "class", + "const", + "continue", + "debugger", + "default", + "delete", + "do", + "double", + "else", + "enum", + "export", + "extends", + "false", + "final", + "finally", + "float", + "for", + "function", + "goto", + "if", + "implements", + "import", + "in", + "instanceof", + "int", + "interface", + "long", + "native", + "new", + "null", + "package", + "private", + "protected", + "public", + "return", + "short", + "static", + "super", + "switch", + "synchronized", + "this", + "throw", + "throws", + "transient", + "try", + "typeof", + "var", + "void", + "volatile", + "while", + "with", +}; + +static const int kNumKeyword = sizeof(kKeyword) / sizeof(char*); + +namespace { + +bool IsReserved(const string& ident) { + for (int i = 0; i < kNumKeyword; i++) { + if (ident == kKeyword[i]) { + return true; + } + } + return false; +} + +// Returns a copy of |filename| with any trailing ".protodevel" or ".proto +// suffix stripped. +string StripProto(const string& filename) { + const char* suffix = HasSuffixString(filename, ".protodevel") + ? ".protodevel" : ".proto"; + return StripSuffixString(filename, suffix); +} + +// Returns the fully normalized JavaScript path for the given +// file descriptor's package. +string GetPath(const GeneratorOptions& options, + const FileDescriptor* file) { + if (!options.namespace_prefix.empty()) { + return options.namespace_prefix; + } else if (!file->package().empty()) { + return "proto." + file->package(); + } else { + return "proto"; + } +} + +// Forward declare, so that GetPrefix can call this method, +// which in turn, calls GetPrefix. +string GetPath(const GeneratorOptions& options, + const Descriptor* descriptor); + +// Returns the path prefix for a message or enumeration that +// lives under the given file and containing type. +string GetPrefix(const GeneratorOptions& options, + const FileDescriptor* file_descriptor, + const Descriptor* containing_type) { + string prefix = ""; + + if (containing_type == NULL) { + prefix = GetPath(options, file_descriptor); + } else { + prefix = GetPath(options, containing_type); + } + + if (!prefix.empty()) { + prefix += "."; + } + + return prefix; +} + + +// Returns the fully normalized JavaScript path for the given +// message descriptor. +string GetPath(const GeneratorOptions& options, + const Descriptor* descriptor) { + return GetPrefix( + options, descriptor->file(), + descriptor->containing_type()) + descriptor->name(); +} + + +// Returns the fully normalized JavaScript path for the given +// field's containing message descriptor. +string GetPath(const GeneratorOptions& options, + const FieldDescriptor* descriptor) { + return GetPath(options, descriptor->containing_type()); +} + +// Returns the fully normalized JavaScript path for the given +// enumeration descriptor. +string GetPath(const GeneratorOptions& options, + const EnumDescriptor* enum_descriptor) { + return GetPrefix( + options, enum_descriptor->file(), + enum_descriptor->containing_type()) + enum_descriptor->name(); +} + + +// Returns the fully normalized JavaScript path for the given +// enumeration value descriptor. +string GetPath(const GeneratorOptions& options, + const EnumValueDescriptor* value_descriptor) { + return GetPath( + options, + value_descriptor->type()) + "." + value_descriptor->name(); +} + +// - Object field name: LOWER_UNDERSCORE -> LOWER_CAMEL, except for group fields +// (UPPER_CAMEL -> LOWER_CAMEL), with "List" (or "Map") appended if appropriate, +// and with reserved words triggering a "pb_" prefix. +// - Getters/setters: LOWER_UNDERSCORE -> UPPER_CAMEL, except for group fields +// (use the name directly), then append "List" if appropriate, then append "$" +// if resulting name is equal to a reserved word. +// - Enums: just uppercase. + +// Locale-independent version of ToLower that deals only with ASCII A-Z. +char ToLowerASCII(char c) { + if (c >= 'A' && c <= 'Z') { + return (c - 'A') + 'a'; + } else { + return c; + } +} + +vector ParseLowerUnderscore(const string& input) { + vector words; + string running = ""; + for (int i = 0; i < input.size(); i++) { + if (input[i] == '_') { + if (!running.empty()) { + words.push_back(running); + running.clear(); + } + } else { + running += ToLowerASCII(input[i]); + } + } + if (!running.empty()) { + words.push_back(running); + } + return words; +} + +vector ParseUpperCamel(const string& input) { + vector words; + string running = ""; + for (int i = 0; i < input.size(); i++) { + if (input[i] >= 'A' && input[i] <= 'Z' && !running.empty()) { + words.push_back(running); + running.clear(); + } + running += ToLowerASCII(input[i]); + } + if (!running.empty()) { + words.push_back(running); + } + return words; +} + +string ToLowerCamel(const vector& words) { + string result; + for (int i = 0; i < words.size(); i++) { + string word = words[i]; + if (i == 0 && (word[0] >= 'A' && word[0] <= 'Z')) { + word[0] = (word[0] - 'A') + 'a'; + } else if (i != 0 && (word[0] >= 'a' && word[0] <= 'z')) { + word[0] = (word[0] - 'a') + 'A'; + } + result += word; + } + return result; +} + +string ToUpperCamel(const vector& words) { + string result; + for (int i = 0; i < words.size(); i++) { + string word = words[i]; + if (word[0] >= 'a' && word[0] <= 'z') { + word[0] = (word[0] - 'a') + 'A'; + } + result += word; + } + return result; +} + +// Based on code from descriptor.cc (Thanks Kenton!) +// Uppercases the entire string, turning ValueName into +// VALUENAME. +string ToEnumCase(const string& input) { + string result; + result.reserve(input.size()); + + for (int i = 0; i < input.size(); i++) { + if ('a' <= input[i] && input[i] <= 'z') { + result.push_back(input[i] - 'a' + 'A'); + } else { + result.push_back(input[i]); + } + } + + return result; +} + +string ToFileName(const string& input) { + string result; + result.reserve(input.size()); + + for (int i = 0; i < input.size(); i++) { + if ('A' <= input[i] && input[i] <= 'Z') { + result.push_back(input[i] - 'A' + 'a'); + } else { + result.push_back(input[i]); + } + } + + return result; +} + +// Returns the message/response ID, if set. +string GetMessageId(const Descriptor* desc) { + return string(); +} + + +// Used inside Google only -- do not remove. +bool IsResponse(const Descriptor* desc) { return false; } +bool IgnoreField(const FieldDescriptor* field) { return false; } + + +// Does JSPB ignore this entire oneof? True only if all fields are ignored. +bool IgnoreOneof(const OneofDescriptor* oneof) { + for (int i = 0; i < oneof->field_count(); i++) { + if (!IgnoreField(oneof->field(i))) { + return false; + } + } + return true; +} + +string JSIdent(const FieldDescriptor* field, + bool is_upper_camel, + bool is_map) { + string result; + if (field->type() == FieldDescriptor::TYPE_GROUP) { + result = is_upper_camel ? + ToUpperCamel(ParseUpperCamel(field->message_type()->name())) : + ToLowerCamel(ParseUpperCamel(field->message_type()->name())); + } else { + result = is_upper_camel ? + ToUpperCamel(ParseLowerUnderscore(field->name())) : + ToLowerCamel(ParseLowerUnderscore(field->name())); + } + if (is_map) { + result += "Map"; + } else if (field->is_repeated()) { + result += "List"; + } + return result; +} + +string JSObjectFieldName(const FieldDescriptor* field) { + string name = JSIdent( + field, + /* is_upper_camel = */ false, + /* is_map = */ false); + if (IsReserved(name)) { + name = "pb_" + name; + } + return name; +} + +// Returns the field name as a capitalized portion of a getter/setter method +// name, e.g. MyField for .getMyField(). +string JSGetterName(const FieldDescriptor* field) { + string name = JSIdent(field, + /* is_upper_camel = */ true, + /* is_map = */ false); + if (name == "Extension" || name == "JsPbMessageId") { + // Avoid conflicts with base-class names. + name += "$"; + } + return name; +} + +string JSMapGetterName(const FieldDescriptor* field) { + return JSIdent(field, + /* is_upper_camel = */ true, + /* is_map = */ true); +} + + + +string JSOneofName(const OneofDescriptor* oneof) { + return ToUpperCamel(ParseLowerUnderscore(oneof->name())); +} + +// Returns the index corresponding to this field in the JSPB array (underlying +// data storage array). +string JSFieldIndex(const FieldDescriptor* field) { + // Determine whether this field is a member of a group. Group fields are a bit + // wonky: their "containing type" is a message type created just for the + // group, and that type's parent type has a field with the group-message type + // as its message type and TYPE_GROUP as its field type. For such fields, the + // index we use is relative to the field number of the group submessage field. + // For all other fields, we just use the field number. + const Descriptor* containing_type = field->containing_type(); + const Descriptor* parent_type = containing_type->containing_type(); + if (parent_type != NULL) { + for (int i = 0; i < parent_type->field_count(); i++) { + if (parent_type->field(i)->type() == FieldDescriptor::TYPE_GROUP && + parent_type->field(i)->message_type() == containing_type) { + return SimpleItoa(field->number() - parent_type->field(i)->number()); + } + } + } + return SimpleItoa(field->number()); +} + +string JSOneofIndex(const OneofDescriptor* oneof) { + int index = -1; + for (int i = 0; i < oneof->containing_type()->oneof_decl_count(); i++) { + const OneofDescriptor* o = oneof->containing_type()->oneof_decl(i); + // If at least one field in this oneof is not JSPB-ignored, count the oneof. + for (int j = 0; j < o->field_count(); j++) { + const FieldDescriptor* f = o->field(j); + if (!IgnoreField(f)) { + index++; + break; // inner loop + } + } + if (o == oneof) { + break; + } + } + return SimpleItoa(index); +} + +// Decodes a codepoint in \x0000 -- \xFFFF. Since JS strings are UTF-16, we only +// need to handle the BMP (16-bit range) here. +uint16_t DecodeUTF8Codepoint(uint8_t* bytes, size_t* length) { + if (*length == 0) { + return 0; + } + size_t expected = 0; + if ((*bytes & 0x80) == 0) { + expected = 1; + } else if ((*bytes & 0xe0) == 0xc0) { + expected = 2; + } else if ((*bytes & 0xf0) == 0xe0) { + expected = 3; + } else { + // Too long -- don't accept. + *length = 0; + return 0; + } + + if (*length < expected) { + // Not enough bytes -- don't accept. + *length = 0; + return 0; + } + + *length = expected; + switch (expected) { + case 1: return bytes[0]; + case 2: return ((bytes[0] & 0x1F) << 6) | + ((bytes[1] & 0x3F) << 0); + case 3: return ((bytes[0] & 0x0F) << 12) | + ((bytes[1] & 0x3F) << 6) | + ((bytes[2] & 0x3F) << 0); + default: return 0; + } +} + +// Escapes the contents of a string to be included within double-quotes ("") in +// JavaScript. |is_utf8| determines whether the input data (in a C++ string of +// chars) is UTF-8 encoded (in which case codepoints become JavaScript string +// characters, escaped with 16-bit hex escapes where necessary) or raw binary +// (in which case bytes become JavaScript string characters 0 -- 255). +string EscapeJSString(const string& in, bool is_utf8) { + string result; + size_t decoded = 0; + for (size_t i = 0; i < in.size(); i += decoded) { + uint16_t codepoint = 0; + if (is_utf8) { + // Decode the next UTF-8 codepoint. + size_t have_bytes = in.size() - i; + uint8_t bytes[3] = { + static_cast(in[i]), + static_cast(((i + 1) < in.size()) ? in[i + 1] : 0), + static_cast(((i + 2) < in.size()) ? in[i + 2] : 0), + }; + codepoint = DecodeUTF8Codepoint(bytes, &have_bytes); + if (have_bytes == 0) { + break; + } + decoded = have_bytes; + } else { + codepoint = static_cast(static_cast(in[i])); + decoded = 1; + } + + // Next byte -- used for minimal octal escapes below. + char next_byte = (i + decoded) < in.size() ? + in[i + decoded] : 0; + bool pad_octal = (next_byte >= '0' && next_byte <= '7'); + + switch (codepoint) { + case '\0': result += pad_octal ? "\\000" : "\\0"; break; + case '\b': result += "\\\b"; break; + case '\t': result += "\\\t"; break; + case '\n': result += "\\\n"; break; + case '\r': result += "\\\r"; break; + case '\f': result += "\\\f"; break; + case '\\': result += "\\\\"; break; + case '"': result += pad_octal ? "\\042" : "\\42"; break; + case '&': result += pad_octal ? "\\046" : "\\46"; break; + case '\'': result += pad_octal ? "\\047" : "\\47"; break; + case '<': result += pad_octal ? "\\074" : "\\74"; break; + case '=': result += pad_octal ? "\\075" : "\\75"; break; + case '>': result += pad_octal ? "\\076" : "\\76"; break; + default: + // All other non-ASCII codepoints are escaped. + // Original codegen uses hex for >= 0x100 and octal for others. + if (codepoint >= 0x20 && codepoint <= 0x7e) { + result += static_cast(codepoint); + } else { + if (codepoint >= 0x100) { + result += StringPrintf("\\u%04x", codepoint); + } else { + if (pad_octal || codepoint >= 0100) { + result += "\\"; + result += ('0' + ((codepoint >> 6) & 07)); + result += ('0' + ((codepoint >> 3) & 07)); + result += ('0' + ((codepoint >> 0) & 07)); + } else if (codepoint >= 010) { + result += "\\"; + result += ('0' + ((codepoint >> 3) & 07)); + result += ('0' + ((codepoint >> 0) & 07)); + } else { + result += "\\"; + result += ('0' + ((codepoint >> 0) & 07)); + } + } + } + break; + } + } + return result; +} + +string EscapeBase64(const string& in) { + static const char* kAlphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + string result; + + for (size_t i = 0; i < in.size(); i += 3) { + int value = (in[i] << 16) | + (((i + 1) < in.size()) ? (in[i + 1] << 8) : 0) | + (((i + 2) < in.size()) ? (in[i + 2] << 0) : 0); + result += kAlphabet[(value >> 18) & 0x3f]; + result += kAlphabet[(value >> 12) & 0x3f]; + if ((i + 1) < in.size()) { + result += kAlphabet[(value >> 6) & 0x3f]; + } else { + result += '='; + } + if ((i + 2) < in.size()) { + result += kAlphabet[(value >> 0) & 0x3f]; + } else { + result += '='; + } + } + + return result; +} + +// Post-process the result of SimpleFtoa/SimpleDtoa to *exactly* match the +// original codegen's formatting (which is just .toString() on java.lang.Double +// or java.lang.Float). +string PostProcessFloat(string result) { + // If inf, -inf or nan, replace with +Infinity, -Infinity or NaN. + if (result == "inf") { + return "Infinity"; + } else if (result == "-inf") { + return "-Infinity"; + } else if (result == "nan") { + return "NaN"; + } + + // If scientific notation (e.g., "1e10"), (i) capitalize the "e", (ii) + // ensure that the mantissa (portion prior to the "e") has at least one + // fractional digit (after the decimal point), and (iii) strip any unnecessary + // leading zeroes and/or '+' signs from the exponent. + string::size_type exp_pos = result.find('e'); + if (exp_pos != string::npos) { + string mantissa = result.substr(0, exp_pos); + string exponent = result.substr(exp_pos + 1); + + // Add ".0" to mantissa if no fractional part exists. + if (mantissa.find('.') == string::npos) { + mantissa += ".0"; + } + + // Strip the sign off the exponent and store as |exp_neg|. + bool exp_neg = false; + if (!exponent.empty() && exponent[0] == '+') { + exponent = exponent.substr(1); + } else if (!exponent.empty() && exponent[0] == '-') { + exp_neg = true; + exponent = exponent.substr(1); + } + + // Strip any leading zeroes off the exponent. + while (exponent.size() > 1 && exponent[0] == '0') { + exponent = exponent.substr(1); + } + + return mantissa + "E" + string(exp_neg ? "-" : "") + exponent; + } + + // Otherwise, this is an ordinary decimal number. Append ".0" if result has no + // decimal/fractional part in order to match output of original codegen. + if (result.find('.') == string::npos) { + result += ".0"; + } + + return result; +} + +string FloatToString(float value) { + string result = SimpleFtoa(value); + return PostProcessFloat(result); +} + +string DoubleToString(double value) { + string result = SimpleDtoa(value); + return PostProcessFloat(result); +} + +string MaybeNumberString(const FieldDescriptor* field, const string& orig) { + return orig; +} + +string JSFieldDefault(const FieldDescriptor* field) { + assert(field->has_default_value()); + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return MaybeNumberString( + field, SimpleItoa(field->default_value_int32())); + case FieldDescriptor::CPPTYPE_UINT32: + // The original codegen is in Java, and Java protobufs store unsigned + // integer values as signed integer values. In order to exactly match the + // output, we need to reinterpret as base-2 signed. Ugh. + return MaybeNumberString( + field, SimpleItoa(static_cast(field->default_value_uint32()))); + case FieldDescriptor::CPPTYPE_INT64: + return MaybeNumberString( + field, SimpleItoa(field->default_value_int64())); + case FieldDescriptor::CPPTYPE_UINT64: + // See above note for uint32 -- reinterpreting as signed. + return MaybeNumberString( + field, SimpleItoa(static_cast(field->default_value_uint64()))); + case FieldDescriptor::CPPTYPE_ENUM: + return SimpleItoa(field->default_value_enum()->number()); + case FieldDescriptor::CPPTYPE_BOOL: + return field->default_value_bool() ? "true" : "false"; + case FieldDescriptor::CPPTYPE_FLOAT: + return FloatToString(field->default_value_float()); + case FieldDescriptor::CPPTYPE_DOUBLE: + return DoubleToString(field->default_value_double()); + case FieldDescriptor::CPPTYPE_STRING: + if (field->type() == FieldDescriptor::TYPE_STRING) { + return "\"" + EscapeJSString(field->default_value_string(), true) + + "\""; + } else { + return "\"" + EscapeBase64(field->default_value_string()) + + "\""; + } + case FieldDescriptor::CPPTYPE_MESSAGE: + return "null"; + } +} + +string ProtoTypeName(const GeneratorOptions& options, + const FieldDescriptor* field) { + switch (field->type()) { + case FieldDescriptor::TYPE_BOOL: + return "bool"; + case FieldDescriptor::TYPE_INT32: + return "int32"; + case FieldDescriptor::TYPE_UINT32: + return "uint32"; + case FieldDescriptor::TYPE_SINT32: + return "sint32"; + case FieldDescriptor::TYPE_FIXED32: + return "fixed32"; + case FieldDescriptor::TYPE_SFIXED32: + return "sfixed32"; + case FieldDescriptor::TYPE_INT64: + return "int64"; + case FieldDescriptor::TYPE_UINT64: + return "uint64"; + case FieldDescriptor::TYPE_SINT64: + return "sint64"; + case FieldDescriptor::TYPE_FIXED64: + return "fixed64"; + case FieldDescriptor::TYPE_SFIXED64: + return "sfixed64"; + case FieldDescriptor::TYPE_FLOAT: + return "float"; + case FieldDescriptor::TYPE_DOUBLE: + return "double"; + case FieldDescriptor::TYPE_STRING: + return "string"; + case FieldDescriptor::TYPE_BYTES: + return "bytes"; + case FieldDescriptor::TYPE_GROUP: + return GetPath(options, field->message_type()); + case FieldDescriptor::TYPE_ENUM: + return GetPath(options, field->enum_type()); + case FieldDescriptor::TYPE_MESSAGE: + return GetPath(options, field->message_type()); + default: + return ""; + } +} + +string JSIntegerTypeName(const FieldDescriptor* field) { + return "number"; +} + +string JSTypeName(const GeneratorOptions& options, + const FieldDescriptor* field) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_BOOL: + return "boolean"; + case FieldDescriptor::CPPTYPE_INT32: + return JSIntegerTypeName(field); + case FieldDescriptor::CPPTYPE_INT64: + return JSIntegerTypeName(field); + case FieldDescriptor::CPPTYPE_UINT32: + return JSIntegerTypeName(field); + case FieldDescriptor::CPPTYPE_UINT64: + return JSIntegerTypeName(field); + case FieldDescriptor::CPPTYPE_FLOAT: + return "number"; + case FieldDescriptor::CPPTYPE_DOUBLE: + return "number"; + case FieldDescriptor::CPPTYPE_STRING: + return "string"; + case FieldDescriptor::CPPTYPE_ENUM: + return GetPath(options, field->enum_type()); + case FieldDescriptor::CPPTYPE_MESSAGE: + return GetPath(options, field->message_type()); + default: + return ""; + } +} + +bool HasFieldPresence(const FieldDescriptor* field); + +string JSFieldTypeAnnotation(const GeneratorOptions& options, + const FieldDescriptor* field, + bool force_optional, + bool force_present, + bool singular_if_not_packed, + bool always_singular) { + bool is_primitive = + (field->cpp_type() != FieldDescriptor::CPPTYPE_ENUM && + field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE); + + string jstype = JSTypeName(options, field); + + if (field->is_repeated() && + !always_singular && + (field->is_packed() || !singular_if_not_packed)) { + if (!is_primitive) { + jstype = "!" + jstype; + } + jstype = "Array.<" + jstype + ">"; + if (!force_optional) { + jstype = "!" + jstype; + } + } + + if (field->is_optional() && is_primitive && + (!field->has_default_value() || force_optional) && !force_present) { + jstype += "?"; + } else if (field->is_required() && !is_primitive && !force_optional) { + jstype = "!" + jstype; + } + + if (force_optional && HasFieldPresence(field)) { + jstype += "|undefined"; + } + if (force_present && jstype[0] != '!' && !is_primitive) { + jstype = "!" + jstype; + } + + return jstype; +} + +string JSBinaryReaderMethodType(const FieldDescriptor* field) { + string name = field->type_name(); + if (name[0] >= 'a' && name[0] <= 'z') { + name[0] = (name[0] - 'a') + 'A'; + } + + return name; +} + +string JSBinaryReadWriteMethodName(const FieldDescriptor* field, + bool is_writer) { + string name = JSBinaryReaderMethodType(field); + if (is_writer && field->type() == FieldDescriptor::TYPE_BYTES) { + // Override for `bytes` fields: treat string as raw bytes, not base64. + name = "BytesRawString"; + } + if (field->is_packed()) { + name = "Packed" + name; + } else if (is_writer && field->is_repeated()) { + name = "Repeated" + name; + } + return name; +} + +string JSBinaryReaderMethodName(const FieldDescriptor* field) { + return "read" + JSBinaryReadWriteMethodName(field, /* is_writer = */ false); +} + +string JSBinaryWriterMethodName(const FieldDescriptor* field) { + return "write" + JSBinaryReadWriteMethodName(field, /* is_writer = */ true); +} + +string JSReturnClause(const FieldDescriptor* desc) { + return ""; +} + +string JSReturnDoc(const GeneratorOptions& options, + const FieldDescriptor* desc) { + return ""; +} + +bool HasRepeatedFields(const Descriptor* desc) { + for (int i = 0; i < desc->field_count(); i++) { + if (desc->field(i)->is_repeated()) { + return true; + } + } + return false; +} + +static const char* kRepeatedFieldArrayName = ".repeatedFields_"; + +string RepeatedFieldsArrayName(const GeneratorOptions& options, + const Descriptor* desc) { + return HasRepeatedFields(desc) ? + (GetPath(options, desc) + kRepeatedFieldArrayName) : "null"; +} + +bool HasOneofFields(const Descriptor* desc) { + for (int i = 0; i < desc->field_count(); i++) { + if (desc->field(i)->containing_oneof()) { + return true; + } + } + return false; +} + +static const char* kOneofGroupArrayName = ".oneofGroups_"; + +string OneofFieldsArrayName(const GeneratorOptions& options, + const Descriptor* desc) { + return HasOneofFields(desc) ? + (GetPath(options, desc) + kOneofGroupArrayName) : "null"; +} + +string RepeatedFieldNumberList(const Descriptor* desc) { + std::vector numbers; + for (int i = 0; i < desc->field_count(); i++) { + if (desc->field(i)->is_repeated()) { + numbers.push_back(JSFieldIndex(desc->field(i))); + } + } + return "[" + Join(numbers, ",") + "]"; +} + +string OneofGroupList(const Descriptor* desc) { + // List of arrays (one per oneof), each of which is a list of field indices + std::vector oneof_entries; + for (int i = 0; i < desc->oneof_decl_count(); i++) { + const OneofDescriptor* oneof = desc->oneof_decl(i); + if (IgnoreOneof(oneof)) { + continue; + } + + std::vector oneof_fields; + for (int j = 0; j < oneof->field_count(); j++) { + if (IgnoreField(oneof->field(j))) { + continue; + } + oneof_fields.push_back(JSFieldIndex(oneof->field(j))); + } + oneof_entries.push_back("[" + Join(oneof_fields, ",") + "]"); + } + return "[" + Join(oneof_entries, ",") + "]"; +} + +string JSOneofArray(const GeneratorOptions& options, + const FieldDescriptor* field) { + return OneofFieldsArrayName(options, field->containing_type()) + "[" + + JSOneofIndex(field->containing_oneof()) + "]"; +} + +string RelativeTypeName(const FieldDescriptor* field) { + assert(field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM || + field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE); + // For a field with an enum or message type, compute a name relative to the + // path name of the message type containing this field. + string package = field->file()->package(); + string containing_type = field->containing_type()->full_name() + "."; + string type = (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) ? + field->enum_type()->full_name() : field->message_type()->full_name(); + + // |prefix| is advanced as we find separators '.' past the common package + // prefix that yield common prefixes in the containing type's name and this + // type's name. + int prefix = 0; + for (int i = 0; i < type.size() && i < containing_type.size(); i++) { + if (type[i] != containing_type[i]) { + break; + } + if (type[i] == '.' && i >= package.size()) { + prefix = i + 1; + } + } + + return type.substr(prefix); +} + +string JSExtensionsObjectName(const GeneratorOptions& options, + const Descriptor* desc) { + if (desc->full_name() == "google.protobuf.bridge.MessageSet") { + return "jspb.Message.messageSetExtensions"; + } else { + return GetPath(options, desc) + ".extensions"; + } +} + +string FieldDefinition(const GeneratorOptions& options, + const FieldDescriptor* field) { + string qualifier = field->is_repeated() ? "repeated" : + (field->is_optional() ? "optional" : "required"); + string type, name; + if (field->type() == FieldDescriptor::TYPE_ENUM || + field->type() == FieldDescriptor::TYPE_MESSAGE) { + type = RelativeTypeName(field); + name = field->name(); + } else if (field->type() == FieldDescriptor::TYPE_GROUP) { + type = "group"; + name = field->message_type()->name(); + } else { + type = ProtoTypeName(options, field); + name = field->name(); + } + return StringPrintf("%s %s %s = %d;", + qualifier.c_str(), + type.c_str(), + name.c_str(), + field->number()); +} + +string FieldComments(const FieldDescriptor* field) { + string comments; + if (field->cpp_type() == FieldDescriptor::CPPTYPE_BOOL) { + comments += + " * Note that Boolean fields may be set to 0/1 when serialized from " + "a Java server.\n" + " * You should avoid comparisons like {@code val === true/false} in " + "those cases.\n"; + } + if (field->is_repeated()) { + comments += + " * If you change this array by adding, removing or replacing " + "elements, or if you\n" + " * replace the array itself, then you must call the setter to " + "update it.\n"; + } + return comments; +} + +bool ShouldGenerateExtension(const FieldDescriptor* field) { + return + field->is_extension() && + !IgnoreField(field); +} + +bool HasExtensions(const Descriptor* desc) { + if (desc->extension_count() > 0) { + return true; + } + for (int i = 0; i < desc->nested_type_count(); i++) { + if (HasExtensions(desc->nested_type(i))) { + return true; + } + } + return false; +} + +bool HasExtensions(const FileDescriptor* file) { + for (int i = 0; i < file->extension_count(); i++) { + if (ShouldGenerateExtension(file->extension(i))) { + return true; + } + } + for (int i = 0; i < file->message_type_count(); i++) { + if (HasExtensions(file->message_type(i))) { + return true; + } + } + return false; +} + +bool IsExtendable(const Descriptor* desc) { + return desc->extension_range_count() > 0; +} + +// Returns the max index in the underlying data storage array beyond which the +// extension object is used. +string GetPivot(const Descriptor* desc) { + static const int kDefaultPivot = (1 << 29); // max field number (29 bits) + + // Find the max field number + int max_field_number = 0; + for (int i = 0; i < desc->field_count(); i++) { + if (!IgnoreField(desc->field(i)) && + desc->field(i)->number() > max_field_number) { + max_field_number = desc->field(i)->number(); + } + } + + int pivot = -1; + if (IsExtendable(desc)) { + pivot = ((max_field_number + 1) < kDefaultPivot) ? + (max_field_number + 1) : kDefaultPivot; + } + + return SimpleItoa(pivot); +} + +// Returns true for fields that represent "null" as distinct from the default +// value. See https://go/proto3#heading=h.kozewqqcqhuz for more information. +bool HasFieldPresence(const FieldDescriptor* field) { + return + (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) || + (field->containing_oneof() != NULL) || + (field->file()->syntax() != FileDescriptor::SYNTAX_PROTO3); +} + +// For proto3 fields without presence, returns a string representing the default +// value in JavaScript. See https://go/proto3#heading=h.kozewqqcqhuz for more +// information. +string Proto3PrimitiveFieldDefault(const FieldDescriptor* field) { + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + case FieldDescriptor::CPPTYPE_INT64: + case FieldDescriptor::CPPTYPE_UINT32: + case FieldDescriptor::CPPTYPE_UINT64: { + return "0"; + } + + case FieldDescriptor::CPPTYPE_ENUM: + case FieldDescriptor::CPPTYPE_FLOAT: + case FieldDescriptor::CPPTYPE_DOUBLE: + return "0"; + + case FieldDescriptor::CPPTYPE_BOOL: + return "false"; + + case FieldDescriptor::CPPTYPE_STRING: + return "\"\""; + + default: + // BYTES and MESSAGE are handled separately. + assert(false); + return ""; + } +} + +} // anonymous namespace + +void Generator::GenerateHeader(const GeneratorOptions& options, + io::Printer* printer) const { + printer->Print("/**\n" + " * @fileoverview\n" + " * @enhanceable\n" + " * @public\n" + " */\n" + "// GENERATED CODE -- DO NOT EDIT!\n" + "\n"); +} + +void Generator::FindProvides(const GeneratorOptions& options, + io::Printer* printer, + const vector& files, + std::set* provided) const { + for (int i = 0; i < files.size(); i++) { + for (int j = 0; j < files[i]->message_type_count(); j++) { + FindProvidesForMessage(options, printer, files[i]->message_type(j), + provided); + } + for (int j = 0; j < files[i]->enum_type_count(); j++) { + FindProvidesForEnum(options, printer, files[i]->enum_type(j), + provided); + } + } + + printer->Print("\n"); +} + +void Generator::FindProvidesForMessage( + const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc, + std::set* provided) const { + string name = GetPath(options, desc); + provided->insert(name); + + for (int i = 0; i < desc->enum_type_count(); i++) { + FindProvidesForEnum(options, printer, desc->enum_type(i), + provided); + } + for (int i = 0; i < desc->nested_type_count(); i++) { + FindProvidesForMessage(options, printer, desc->nested_type(i), + provided); + } +} + +void Generator::FindProvidesForEnum(const GeneratorOptions& options, + io::Printer* printer, + const EnumDescriptor* enumdesc, + std::set* provided) const { + string name = GetPath(options, enumdesc); + provided->insert(name); +} + +void Generator::FindProvidesForFields( + const GeneratorOptions& options, + io::Printer* printer, + const vector& fields, + std::set* provided) const { + for (int i = 0; i < fields.size(); i++) { + const FieldDescriptor* field = fields[i]; + + if (IgnoreField(field)) { + continue; + } + + string name = + GetPath(options, field->file()) + "." + JSObjectFieldName(field); + provided->insert(name); + } +} + +void Generator::GenerateProvides(const GeneratorOptions& options, + io::Printer* printer, + std::set* provided) const { + for (std::set::iterator it = provided->begin(); + it != provided->end(); ++it) { + printer->Print("goog.provide('$name$');\n", + "name", *it); + } +} + +void Generator::GenerateRequires(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc, + std::set* provided) const { + std::set required; + std::set forwards; + bool have_message = false; + FindRequiresForMessage(options, desc, + &required, &forwards, &have_message); + + GenerateRequiresImpl(options, printer, &required, &forwards, provided, + /* require_jspb = */ have_message, + /* require_extension = */ HasExtensions(desc)); +} + +void Generator::GenerateRequires(const GeneratorOptions& options, + io::Printer* printer, + const vector& files, + std::set* provided) const { + std::set required; + std::set forwards; + bool have_extensions = false; + bool have_message = false; + + for (int i = 0; i < files.size(); i++) { + for (int j = 0; j < files[i]->message_type_count(); j++) { + FindRequiresForMessage(options, + files[i]->message_type(j), + &required, &forwards, &have_message); + } + if (!have_extensions && HasExtensions(files[i])) { + have_extensions = true; + } + + for (int j = 0; j < files[i]->extension_count(); j++) { + const FieldDescriptor* extension = files[i]->extension(j); + if (IgnoreField(extension)) { + continue; + } + if (extension->containing_type()->full_name() != + "google.protobuf.bridge.MessageSet") { + required.insert(GetPath(options, extension->containing_type())); + } + FindRequiresForField(options, extension, &required, &forwards); + have_extensions = true; + } + } + + GenerateRequiresImpl(options, printer, &required, &forwards, provided, + /* require_jspb = */ have_message, + /* require_extension = */ have_extensions); +} + +void Generator::GenerateRequires(const GeneratorOptions& options, + io::Printer* printer, + const vector& fields, + std::set* provided) const { + std::set required; + std::set forwards; + for (int i = 0; i < fields.size(); i++) { + const FieldDescriptor* field = fields[i]; + if (IgnoreField(field)) { + continue; + } + FindRequiresForExtension(options, field, &required, &forwards); + } + + GenerateRequiresImpl(options, printer, &required, &forwards, provided, + /* require_jspb = */ false, + /* require_extension = */ fields.size() > 0); +} + +void Generator::GenerateRequiresImpl(const GeneratorOptions& options, + io::Printer* printer, + std::set* required, + std::set* forwards, + std::set* provided, + bool require_jspb, + bool require_extension) const { + if (require_jspb) { + printer->Print( + "goog.require('jspb.Message');\n"); + if (options.binary) { + printer->Print( + "goog.require('jspb.BinaryReader');\n" + "goog.require('jspb.BinaryWriter');\n"); + } + } + if (require_extension) { + printer->Print( + "goog.require('jspb.ExtensionFieldInfo');\n"); + } + + std::set::iterator it; + for (it = required->begin(); it != required->end(); ++it) { + if (provided->find(*it) != provided->end()) { + continue; + } + printer->Print("goog.require('$name$');\n", + "name", *it); + } + + printer->Print("\n"); + + for (it = forwards->begin(); it != forwards->end(); ++it) { + if (provided->find(*it) != provided->end()) { + continue; + } + printer->Print("goog.forwardDeclare('$name$');\n", + "name", *it); + } +} + +bool NamespaceOnly(const Descriptor* desc) { + return false; +} + +void Generator::FindRequiresForMessage( + const GeneratorOptions& options, + const Descriptor* desc, + std::set* required, + std::set* forwards, + bool* have_message) const { + + + if (!NamespaceOnly(desc)) { + *have_message = true; + for (int i = 0; i < desc->field_count(); i++) { + const FieldDescriptor* field = desc->field(i); + if (IgnoreField(field)) { + continue; + } + FindRequiresForField(options, field, required, forwards); + } + } + + for (int i = 0; i < desc->extension_count(); i++) { + const FieldDescriptor* field = desc->extension(i); + if (IgnoreField(field)) { + continue; + } + FindRequiresForExtension(options, field, required, forwards); + } + + for (int i = 0; i < desc->nested_type_count(); i++) { + FindRequiresForMessage(options, desc->nested_type(i), required, forwards, + have_message); + } +} + +void Generator::FindRequiresForField(const GeneratorOptions& options, + const FieldDescriptor* field, + std::set* required, + std::set* forwards) const { + if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM && + // N.B.: file-level extensions with enum type do *not* create + // dependencies, as per original codegen. + !(field->is_extension() && field->extension_scope() == NULL)) { + if (options.add_require_for_enums) { + required->insert(GetPath(options, field->enum_type())); + } else { + forwards->insert(GetPath(options, field->enum_type())); + } + } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + required->insert(GetPath(options, field->message_type())); + } +} + +void Generator::FindRequiresForExtension(const GeneratorOptions& options, + const FieldDescriptor* field, + std::set* required, + std::set* forwards) const { + if (field->containing_type()->full_name() != "google.protobuf.bridge.MessageSet") { + required->insert(GetPath(options, field->containing_type())); + } + FindRequiresForField(options, field, required, forwards); +} + +void Generator::GenerateTestOnly(const GeneratorOptions& options, + io::Printer* printer) const { + if (options.testonly) { + printer->Print("goog.setTestOnly();\n\n"); + } + printer->Print("\n"); +} + +void Generator::GenerateClassesAndEnums(const GeneratorOptions& options, + io::Printer* printer, + const FileDescriptor* file) const { + for (int i = 0; i < file->message_type_count(); i++) { + GenerateClass(options, printer, file->message_type(i)); + } + for (int i = 0; i < file->enum_type_count(); i++) { + GenerateEnum(options, printer, file->enum_type(i)); + } +} + +void Generator::GenerateClass(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + if (!NamespaceOnly(desc)) { + printer->Print("\n"); + GenerateClassConstructor(options, printer, desc); + GenerateClassFieldInfo(options, printer, desc); + + + GenerateClassToObject(options, printer, desc); + if (options.binary) { + // These must come *before* the extension-field info generation in + // GenerateClassRegistration so that references to the binary + // serialization/deserialization functions may be placed in the extension + // objects. + GenerateClassDeserializeBinary(options, printer, desc); + GenerateClassSerializeBinary(options, printer, desc); + } + GenerateClassClone(options, printer, desc); + GenerateClassRegistration(options, printer, desc); + GenerateClassFields(options, printer, desc); + if (IsExtendable(desc) && desc->full_name() != "google.protobuf.bridge.MessageSet") { + GenerateClassExtensionFieldInfo(options, printer, desc); + } + } + + // Recurse on nested types. + for (int i = 0; i < desc->enum_type_count(); i++) { + GenerateEnum(options, printer, desc->enum_type(i)); + } + for (int i = 0; i < desc->nested_type_count(); i++) { + GenerateClass(options, printer, desc->nested_type(i)); + } +} + +void Generator::GenerateClassConstructor(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + printer->Print( + "/**\n" + " * Generated by JsPbCodeGenerator.\n" + " * @param {Array=} opt_data Optional initial data array, typically " + "from a\n" + " * server response, or constructed directly in Javascript. The array " + "is used\n" + " * in place and becomes part of the constructed object. It is not " + "cloned.\n" + " * If no data is provided, the constructed object will be empty, but " + "still\n" + " * valid.\n" + " * @extends {jspb.Message}\n" + " * @constructor\n" + " */\n" + "$classname$ = function(opt_data) {\n", + "classname", GetPath(options, desc)); + string message_id = GetMessageId(desc); + printer->Print( + " jspb.Message.initialize(this, opt_data, $messageId$, $pivot$, " + "$rptfields$, $oneoffields$);\n", + "messageId", !message_id.empty() ? + ("'" + message_id + "'") : + (IsResponse(desc) ? "''" : "0"), + "pivot", GetPivot(desc), + "rptfields", RepeatedFieldsArrayName(options, desc), + "oneoffields", OneofFieldsArrayName(options, desc)); + printer->Print( + "};\n" + "goog.inherits($classname$, jspb.Message);\n" + "if (goog.DEBUG && !COMPILED) {\n" + " $classname$.displayName = '$classname$';\n" + "}\n", + "classname", GetPath(options, desc)); +} + +void Generator::GenerateClassFieldInfo(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + if (HasRepeatedFields(desc)) { + printer->Print( + "/**\n" + " * List of repeated fields within this message type.\n" + " * @private {!Array}\n" + " * @const\n" + " */\n" + "$classname$$rptfieldarray$ = $rptfields$;\n" + "\n", + "classname", GetPath(options, desc), + "rptfieldarray", kRepeatedFieldArrayName, + "rptfields", RepeatedFieldNumberList(desc)); + } + + if (HasOneofFields(desc)) { + printer->Print( + "/**\n" + " * Oneof group definitions for this message. Each group defines the " + "field\n" + " * numbers belonging to that group. When of these fields' value is " + "set, all\n" + " * other fields in the group are cleared. During deserialization, if " + "multiple\n" + " * fields are encountered for a group, only the last value seen will " + "be kept.\n" + " * @private {!Array>}\n" + " * @const\n" + " */\n" + "$classname$$oneofgrouparray$ = $oneofgroups$;\n" + "\n", + "classname", GetPath(options, desc), + "oneofgrouparray", kOneofGroupArrayName, + "oneofgroups", OneofGroupList(desc)); + + for (int i = 0; i < desc->oneof_decl_count(); i++) { + if (IgnoreOneof(desc->oneof_decl(i))) { + continue; + } + GenerateOneofCaseDefinition(options, printer, desc->oneof_decl(i)); + } + } +} + +void Generator::GenerateClassXid(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + printer->Print( + "\n" + "\n" + "$class$.prototype.messageXid = xid('$class$');\n", + "class", GetPath(options, desc)); +} + +void Generator::GenerateOneofCaseDefinition( + const GeneratorOptions& options, + io::Printer* printer, + const OneofDescriptor* oneof) const { + printer->Print( + "/**\n" + " * @enum {number}\n" + " */\n" + "$classname$.$oneof$Case = {\n" + " $upcase$_NOT_SET: 0", + "classname", GetPath(options, oneof->containing_type()), + "oneof", JSOneofName(oneof), + "upcase", ToEnumCase(oneof->name())); + + for (int i = 0; i < oneof->field_count(); i++) { + if (IgnoreField(oneof->field(i))) { + continue; + } + + printer->Print( + ",\n" + " $upcase$: $number$", + "upcase", ToEnumCase(oneof->field(i)->name()), + "number", JSFieldIndex(oneof->field(i))); + } + + printer->Print( + "\n" + "};\n" + "\n" + "/**\n" + " * @return {$class$.$oneof$Case}\n" + " */\n" + "$class$.prototype.get$oneof$Case = function() {\n" + " return /** @type {$class$.$oneof$Case} */(jspb.Message." + "computeOneofCase(this, $class$.oneofGroups_[$oneofindex$]));\n" + "};\n" + "\n", + "class", GetPath(options, oneof->containing_type()), + "oneof", JSOneofName(oneof), + "oneofindex", JSOneofIndex(oneof)); +} + +void Generator::GenerateClassToObject(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + printer->Print( + "\n" + "\n" + "if (jspb.Message.GENERATE_TO_OBJECT) {\n" + "/**\n" + " * Creates an object representation of this proto suitable for use in " + "Soy templates.\n" + " * Field names that are reserved in JavaScript and will be renamed to " + "pb_name.\n" + " * To access a reserved field use, foo.pb_, eg, foo.pb_default.\n" + " * For the list of reserved names please see:\n" + " * com.google.apps.jspb.JsClassTemplate.JS_RESERVED_WORDS.\n" + " * @param {boolean=} opt_includeInstance Whether to include the JSPB " + "instance\n" + " * for transitional soy proto support: http://goto/soy-param-" + "migration\n" + " * @return {!Object}\n" + " */\n" + "$classname$.prototype.toObject = function(opt_includeInstance) {\n" + " return $classname$.toObject(opt_includeInstance, this);\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Static version of the {@see toObject} method.\n" + " * @param {boolean|undefined} includeInstance Whether to include the " + "JSPB\n" + " * instance for transitional soy proto support:\n" + " * http://goto/soy-param-migration\n" + " * @param {!$classname$} msg The msg instance to transform.\n" + " * @return {!Object}\n" + " */\n" + "$classname$.toObject = function(includeInstance, msg) {\n" + " var f, obj = {", + "classname", GetPath(options, desc)); + + bool first = true; + for (int i = 0; i < desc->field_count(); i++) { + const FieldDescriptor* field = desc->field(i); + if (IgnoreField(field)) { + continue; + } + + if (!first) { + printer->Print(",\n "); + } else { + printer->Print("\n "); + first = false; + } + + GenerateClassFieldToObject(options, printer, field); + } + + if (!first) { + printer->Print("\n };\n\n"); + } else { + printer->Print("\n\n };\n\n"); + } + + if (IsExtendable(desc)) { + printer->Print( + " jspb.Message.toObjectExtension(/** @type {!jspb.Message} */ (msg), " + "obj,\n" + " $extObject$, $class$.prototype.getExtension,\n" + " includeInstance);\n", + "extObject", JSExtensionsObjectName(options, desc), + "class", GetPath(options, desc)); + } + + printer->Print( + " if (includeInstance) {\n" + " obj.$$jspbMessageInstance = msg\n" + " }\n" + " return obj;\n" + "};\n" + "}\n" + "\n" + "\n", + "classname", GetPath(options, desc)); +} + +void Generator::GenerateClassFieldToObject(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const { + printer->Print("$fieldname$: ", + "fieldname", JSObjectFieldName(field)); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + // Message field. + if (field->is_repeated()) { + { + printer->Print("jspb.Message.toObjectList(msg.get$getter$(),\n" + " $type$.toObject, includeInstance)", + "getter", JSGetterName(field), + "type", GetPath(options, field->message_type())); + } + } else { + printer->Print("(f = msg.get$getter$()) && " + "$type$.toObject(includeInstance, f)", + "getter", JSGetterName(field), + "type", GetPath(options, field->message_type())); + } + } else { + // Simple field (singular or repeated). + if (!HasFieldPresence(field) && !field->is_repeated()) { + // Delegate to the generated get() method in order not to duplicate + // the proto3-field-default-value logic here. + printer->Print("msg.get$getter$()", + "getter", JSGetterName(field)); + } else { + if (field->has_default_value()) { + printer->Print("jspb.Message.getField(msg, $index$) != null ? " + "jspb.Message.getField(msg, $index$) : $defaultValue$", + "index", JSFieldIndex(field), + "defaultValue", JSFieldDefault(field)); + } else { + printer->Print("jspb.Message.getField(msg, $index$)", + "index", JSFieldIndex(field)); + } + } + } +} + +void Generator::GenerateClassFromObject(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + printer->Print( + "if (jspb.Message.GENERATE_FROM_OBJECT) {\n" + "/**\n" + " * Loads data from an object into a new instance of this proto.\n" + " * @param {!Object} obj The object representation of this proto to\n" + " * load the data from.\n" + " * @return {!$classname$}\n" + " */\n" + "$classname$.fromObject = function(obj) {\n" + " var f, msg = new $classname$();\n", + "classname", GetPath(options, desc)); + + for (int i = 0; i < desc->field_count(); i++) { + const FieldDescriptor* field = desc->field(i); + GenerateClassFieldFromObject(options, printer, field); + } + + printer->Print( + " return msg;\n" + "};\n" + "}\n"); +} + +void Generator::GenerateClassFieldFromObject( + const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const { + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + // Message field (singular or repeated) + if (field->is_repeated()) { + { + printer->Print( + " goog.isDef(obj.$name$) && " + "jspb.Message.setRepeatedWrapperField(\n" + " msg, $index$, goog.array.map(obj.$name$, function(i) {\n" + " return $fieldclass$.fromObject(i);\n" + " }));\n", + "name", JSObjectFieldName(field), + "index", JSFieldIndex(field), + "fieldclass", GetPath(options, field->message_type())); + } + } else { + printer->Print( + " goog.isDef(obj.$name$) && jspb.Message.setWrapperField(\n" + " msg, $index$, $fieldclass$.fromObject(obj.$name$));\n", + "name", JSObjectFieldName(field), + "index", JSFieldIndex(field), + "fieldclass", GetPath(options, field->message_type())); + } + } else { + // Simple (primitive) field. + printer->Print( + " goog.isDef(obj.$name$) && jspb.Message.setField(msg, $index$, " + "obj.$name$);\n", + "name", JSObjectFieldName(field), + "index", JSFieldIndex(field)); + } +} + +void Generator::GenerateClassClone(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + printer->Print( + "/**\n" + " * Creates a deep clone of this proto. No data is shared with the " + "original.\n" + " * @return {!$name$} The clone.\n" + " */\n" + "$name$.prototype.cloneMessage = function() {\n" + " return /** @type {!$name$} */ (jspb.Message.cloneMessage(this));\n" + "};\n\n\n", + "name", GetPath(options, desc)); +} + +void Generator::GenerateClassRegistration(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + // Register any extensions defined inside this message type. + for (int i = 0; i < desc->extension_count(); i++) { + const FieldDescriptor* extension = desc->extension(i); + if (ShouldGenerateExtension(extension)) { + GenerateExtension(options, printer, extension); + } + } + +} + +void Generator::GenerateClassFields(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + for (int i = 0; i < desc->field_count(); i++) { + if (!IgnoreField(desc->field(i))) { + GenerateClassField(options, printer, desc->field(i)); + } + } +} + +void Generator::GenerateClassField(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const { + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + printer->Print( + "/**\n" + " * $fielddef$\n" + "$comment$" + " * @return {$type$}\n" + " */\n", + "fielddef", FieldDefinition(options, field), + "comment", FieldComments(field), + "type", JSFieldTypeAnnotation(options, field, + /* force_optional = */ false, + /* force_present = */ false, + /* singular_if_not_packed = */ false, + /* always_singular = */ false)); + printer->Print( + "$class$.prototype.get$name$ = function() {\n" + " return /** @type{$type$} */ (\n" + " jspb.Message.get$rpt$WrapperField(this, $wrapperclass$, " + "$index$$required$));\n" + "};\n" + "\n" + "\n", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(field), + "type", JSFieldTypeAnnotation(options, field, + /* force_optional = */ false, + /* force_present = */ false, + /* singular_if_not_packed = */ false, + /* always_singular = */ false), + "rpt", (field->is_repeated() ? "Repeated" : ""), + "index", JSFieldIndex(field), + "wrapperclass", GetPath(options, field->message_type()), + "required", (field->label() == FieldDescriptor::LABEL_REQUIRED ? + ", 1" : "")); + printer->Print( + "/** @param {$optionaltype$} value $returndoc$ */\n" + "$class$.prototype.set$name$ = function(value) {\n" + " jspb.Message.set$oneoftag$$repeatedtag$WrapperField(", + "optionaltype", + JSFieldTypeAnnotation(options, field, + /* force_optional = */ true, + /* force_present = */ false, + /* singular_if_not_packed = */ false, + /* always_singular = */ false), + "returndoc", JSReturnDoc(options, field), + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(field), + "oneoftag", (field->containing_oneof() ? "Oneof" : ""), + "repeatedtag", (field->is_repeated() ? "Repeated" : "")); + + printer->Print( + "this, $index$$oneofgroup$, value);$returnvalue$\n" + "};\n" + "\n" + "\n", + "index", JSFieldIndex(field), + "oneofgroup", (field->containing_oneof() ? + (", " + JSOneofArray(options, field)) : ""), + "returnvalue", JSReturnClause(field)); + + printer->Print( + "$class$.prototype.clear$name$ = function() {\n" + " this.set$name$($clearedvalue$);$returnvalue$\n" + "};\n" + "\n" + "\n", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(field), + "clearedvalue", (field->is_repeated() ? "[]" : "undefined"), + "returnvalue", JSReturnClause(field)); + + } else { + string typed_annotation; + + // Simple (primitive) field, either singular or repeated. + { + typed_annotation = JSFieldTypeAnnotation(options, field, + /* force_optional = */ false, + /* force_present = */ !HasFieldPresence(field), + /* singular_if_not_packed = */ false, + /* always_singular = */ false), + printer->Print( + "/**\n" + " * $fielddef$\n" + "$comment$" + " * @return {$type$}\n" + " */\n", + "fielddef", FieldDefinition(options, field), + "comment", FieldComments(field), + "type", typed_annotation); + } + + printer->Print( + "$class$.prototype.get$name$ = function() {\n", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(field)); + + { + printer->Print( + " return /** @type {$type$} */ (", + "type", typed_annotation); + } + + // For proto3 fields without presence, use special getters that will return + // defaults when the field is unset, possibly constructing a value if + // required. + if (!HasFieldPresence(field) && !field->is_repeated()) { + printer->Print("jspb.Message.getFieldProto3(this, $index$, $default$)", + "index", JSFieldIndex(field), + "default", Proto3PrimitiveFieldDefault(field)); + } else { + if (field->has_default_value()) { + printer->Print("jspb.Message.getField(this, $index$) != null ? " + "jspb.Message.getField(this, $index$) : $defaultValue$", + "index", JSFieldIndex(field), + "defaultValue", JSFieldDefault(field)); + } else { + printer->Print("jspb.Message.getField(this, $index$)", + "index", JSFieldIndex(field)); + } + } + + { + printer->Print( + ");\n" + "};\n" + "\n" + "\n"); + } + + { + printer->Print( + "/** @param {$optionaltype$} value $returndoc$ */\n", + "optionaltype", + JSFieldTypeAnnotation(options, field, + /* force_optional = */ true, + /* force_present = */ !HasFieldPresence(field), + /* singular_if_not_packed = */ false, + /* always_singular = */ false), + "returndoc", JSReturnDoc(options, field)); + } + + printer->Print( + "$class$.prototype.set$name$ = function(value) {\n" + " jspb.Message.set$oneoftag$Field(this, $index$", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(field), + "oneoftag", (field->containing_oneof() ? "Oneof" : ""), + "index", JSFieldIndex(field)); + printer->Print( + "$oneofgroup$, $type$value$rptvalueinit$$typeclose$);$returnvalue$\n" + "};\n" + "\n" + "\n", + "type", "", + "typeclose", "", + "oneofgroup", + (field->containing_oneof() ? (", " + JSOneofArray(options, field)) + : ""), + "returnvalue", JSReturnClause(field), "rptvalueinit", + (field->is_repeated() ? " || []" : "")); + + + if (HasFieldPresence(field)) { + printer->Print( + "$class$.prototype.clear$name$ = function() {\n" + " jspb.Message.set$oneoftag$Field(this, $index$$oneofgroup$, ", + "class", GetPath(options, field->containing_type()), + "name", JSGetterName(field), + "oneoftag", (field->containing_oneof() ? "Oneof" : ""), + "oneofgroup", (field->containing_oneof() ? + (", " + JSOneofArray(options, field)) : ""), + "index", JSFieldIndex(field)); + printer->Print( + "$clearedvalue$);$returnvalue$\n" + "};\n" + "\n" + "\n", + "clearedvalue", (field->is_repeated() ? "[]" : "undefined"), + "returnvalue", JSReturnClause(field)); + } + } +} + +void Generator::GenerateClassExtensionFieldInfo(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + if (IsExtendable(desc)) { + printer->Print( + "\n" + "/**\n" + " * The extensions registered with this message class. This is a " + "map of\n" + " * extension field number to fieldInfo object.\n" + " *\n" + " * For example:\n" + " * { 123: {fieldIndex: 123, fieldName: {my_field_name: 0}, " + "ctor: proto.example.MyMessage} }\n" + " *\n" + " * fieldName contains the JsCompiler renamed field name property " + "so that it\n" + " * works in OPTIMIZED mode.\n" + " *\n" + " * @type {!Object.}\n" + " */\n" + "$class$.extensions = {};\n" + "\n", + "class", GetPath(options, desc)); + } +} + + +void Generator::GenerateClassDeserializeBinary(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + // TODO(cfallin): Handle lazy decoding when requested by field option and/or + // by default for 'bytes' fields and packed repeated fields. + + printer->Print( + "/**\n" + " * Deserializes binary data (in protobuf wire format).\n" + " * @param {jspb.ByteSource} bytes The bytes to deserialize.\n" + " * @return {!$class$}\n" + " */\n" + "$class$.deserializeBinary = function(bytes) {\n" + " var reader = new jspb.BinaryReader(bytes);\n" + " var msg = new $class$;\n" + " return $class$.deserializeBinaryFromReader(msg, reader);\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Deserializes binary data (in protobuf wire format) from the\n" + " * given reader into the given message object.\n" + " * @param {!$class$} msg The message object to deserialize into.\n" + " * @param {!jspb.BinaryReader} reader The BinaryReader to use.\n" + " * @return {!$class$}\n" + " */\n" + "$class$.deserializeBinaryFromReader = function(msg, reader) {\n" + " while (reader.nextField()) {\n" + " if (reader.isEndGroup()) {\n" + " break;\n" + " }\n" + " var field = reader.getFieldNumber();\n" + " switch (field) {\n", + "class", GetPath(options, desc)); + + for (int i = 0; i < desc->field_count(); i++) { + GenerateClassDeserializeBinaryField(options, printer, desc->field(i)); + } + + printer->Print( + " default:\n"); + if (IsExtendable(desc)) { + printer->Print( + " jspb.Message.readBinaryExtension(msg, reader, $extobj$,\n" + " $class$.prototype.getExtension,\n" + " $class$.prototype.setExtension);\n" + " break;\n", + "extobj", JSExtensionsObjectName(options, desc), + "class", GetPath(options, desc)); + } else { + printer->Print( + " reader.skipField();\n" + " break;\n"); + } + + printer->Print( + " }\n" + " }\n" + " return msg;\n" + "};\n" + "\n" + "\n"); +} + +void Generator::GenerateClassDeserializeBinaryField( + const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const { + + printer->Print(" case $num$:\n", + "num", SimpleItoa(field->number())); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + printer->Print( + " var value = new $fieldclass$;\n" + " reader.read$msgOrGroup$($grpfield$value," + "$fieldclass$.deserializeBinaryFromReader);\n", + "fieldclass", GetPath(options, field->message_type()), + "msgOrGroup", (field->type() == FieldDescriptor::TYPE_GROUP) ? + "Group" : "Message", + "grpfield", (field->type() == FieldDescriptor::TYPE_GROUP) ? + (SimpleItoa(field->number()) + ", ") : ""); + } else { + printer->Print( + " var value = /** @type {$fieldtype$} */ (reader.$reader$());\n", + "fieldtype", JSFieldTypeAnnotation(options, field, false, true, + /* singular_if_not_packed = */ true, + /* always_singular = */ false), + "reader", JSBinaryReaderMethodName(field)); + } + + if (field->is_repeated() && !field->is_packed()) { + // Repeated fields receive a |value| one at at a time; append to array + // returned by get$name$(). + printer->Print( + " msg.get$name$().push(value);\n", + "name", JSGetterName(field)); + } else { + // Singular fields, and packed repeated fields, receive a |value| either as + // the field's value or as the array of all the field's values; set this as + // the field's value directly. + printer->Print( + " msg.set$name$(value);\n", + "name", JSGetterName(field)); + } + + printer->Print(" break;\n"); +} + +void Generator::GenerateClassSerializeBinary(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const { + printer->Print( + "/**\n" + " * Class method variant: serializes the given message to binary data\n" + " * (in protobuf wire format), writing to the given BinaryWriter.\n" + " * @param {!$class$} message\n" + " * @param {!jspb.BinaryWriter} writer\n" + " */\n" + "$class$.serializeBinaryToWriter = function(message, " + "writer) {\n" + " message.serializeBinaryToWriter(writer);\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Serializes the message to binary data (in protobuf wire format).\n" + " * @return {!Uint8Array}\n" + " */\n" + "$class$.prototype.serializeBinary = function() {\n" + " var writer = new jspb.BinaryWriter();\n" + " this.serializeBinaryToWriter(writer);\n" + " return writer.getResultBuffer();\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Serializes the message to binary data (in protobuf wire format),\n" + " * writing to the given BinaryWriter.\n" + " * @param {!jspb.BinaryWriter} writer\n" + " */\n" + "$class$.prototype.serializeBinaryToWriter = function (writer) {\n" + " var f = undefined;\n", + "class", GetPath(options, desc)); + + for (int i = 0; i < desc->field_count(); i++) { + GenerateClassSerializeBinaryField(options, printer, desc->field(i)); + } + + if (IsExtendable(desc)) { + printer->Print( + " jspb.Message.serializeBinaryExtensions(this, writer, $extobj$,\n" + " $class$.prototype.getExtension);\n", + "extobj", JSExtensionsObjectName(options, desc), + "class", GetPath(options, desc)); + } + + printer->Print( + "};\n" + "\n" + "\n"); +} + +void Generator::GenerateClassSerializeBinaryField( + const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const { + printer->Print( + " f = this.get$name$();\n", + "name", JSGetterName(field)); + + if (field->is_repeated()) { + printer->Print( + " if (f.length > 0) {\n"); + } else { + if (HasFieldPresence(field)) { + printer->Print( + " if (f != null) {\n"); + } else { + // No field presence: serialize onto the wire only if value is + // non-default. Defaults are documented here: + // https://goto.google.com/lhdfm + switch (field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + case FieldDescriptor::CPPTYPE_INT64: + case FieldDescriptor::CPPTYPE_UINT32: + case FieldDescriptor::CPPTYPE_UINT64: { + { + printer->Print(" if (f !== 0) {\n"); + } + break; + } + + case FieldDescriptor::CPPTYPE_ENUM: + case FieldDescriptor::CPPTYPE_FLOAT: + case FieldDescriptor::CPPTYPE_DOUBLE: + printer->Print( + " if (f !== 0.0) {\n"); + break; + case FieldDescriptor::CPPTYPE_BOOL: + printer->Print( + " if (f) {\n"); + break; + case FieldDescriptor::CPPTYPE_STRING: + printer->Print( + " if (f.length > 0) {\n"); + break; + default: + assert(false); + break; + } + } + } + + printer->Print( + " writer.$writer$(\n" + " $index$,\n" + " f", + "writer", JSBinaryWriterMethodName(field), + "name", JSGetterName(field), + "index", SimpleItoa(field->number())); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + printer->Print( + ",\n" + " $submsg$.serializeBinaryToWriter\n", + "submsg", GetPath(options, field->message_type())); + } else { + printer->Print("\n"); + } + printer->Print( + " );\n" + " }\n"); +} + +void Generator::GenerateEnum(const GeneratorOptions& options, + io::Printer* printer, + const EnumDescriptor* enumdesc) const { + printer->Print( + "/**\n" + " * @enum {number}\n" + " */\n" + "$name$ = {\n", + "name", GetPath(options, enumdesc)); + + for (int i = 0; i < enumdesc->value_count(); i++) { + const EnumValueDescriptor* value = enumdesc->value(i); + printer->Print( + " $name$: $value$$comma$\n", + "name", ToEnumCase(value->name()), + "value", SimpleItoa(value->number()), + "comma", (i == enumdesc->value_count() - 1) ? "" : ","); + } + + printer->Print( + "};\n" + "\n"); +} + +void Generator::GenerateExtension(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const { + string extension_scope = + (field->extension_scope() ? + GetPath(options, field->extension_scope()) : + GetPath(options, field->file())); + + printer->Print( + "\n" + "/**\n" + " * A tuple of {field number, class constructor} for the extension\n" + " * field named `$name$`.\n" + " * @type {!jspb.ExtensionFieldInfo.<$extensionType$>}\n" + " */\n" + "$class$.$name$ = new jspb.ExtensionFieldInfo(\n", + "name", JSObjectFieldName(field), + "class", extension_scope, + "extensionType", JSFieldTypeAnnotation( + options, field, + /* force_optional = */ false, + /* force_present = */ true, + /* singular_if_not_packed = */ false, + /* always_singular = */ false)); + printer->Print( + " $index$,\n" + " {$name$: 0},\n" + " $ctor$,\n" + " /** @type {?function((boolean|undefined),!jspb.Message=): " + "!Object} */ (\n" + " $toObject$),\n" + " $repeated$", + "index", SimpleItoa(field->number()), + "name", JSObjectFieldName(field), + "ctor", (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ? + GetPath(options, field->message_type()) : string("null")), + "toObject", (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE ? + (GetPath(options, field->message_type()) + ".toObject") : + string("null")), + "repeated", (field->is_repeated() ? "1" : "0")); + + if (options.binary) { + printer->Print( + ",\n" + " jspb.BinaryReader.prototype.$binaryReaderFn$,\n" + " jspb.BinaryWriter.prototype.$binaryWriterFn$,\n" + " $binaryMessageSerializeFn$,\n" + " $binaryMessageDeserializeFn$,\n" + " $isPacked$);\n", + "binaryReaderFn", JSBinaryReaderMethodName(field), + "binaryWriterFn", JSBinaryWriterMethodName(field), + "binaryMessageSerializeFn", + (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) ? + (GetPath(options, field->message_type()) + + ".serializeBinaryToWriter") : "null", + "binaryMessageDeserializeFn", + (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) ? + (GetPath(options, field->message_type()) + + ".deserializeBinaryFromReader") : "null", + "isPacked", (field->is_packed() ? "true" : "false")); + } else { + printer->Print(");\n"); + } + + printer->Print( + "// This registers the extension field with the extended class, so that\n" + "// toObject() will function correctly.\n" + "$extendName$[$index$] = $class$.$name$;\n" + "\n", + "extendName", JSExtensionsObjectName(options, field->containing_type()), + "index", SimpleItoa(field->number()), + "class", extension_scope, + "name", JSObjectFieldName(field)); +} + +bool GeneratorOptions::ParseFromOptions( + const vector< pair< string, string > >& options, + string* error) { + for (int i = 0; i < options.size(); i++) { + if (options[i].first == "add_require_for_enums") { + if (options[i].second != "") { + *error = "Unexpected option value for add_require_for_enums"; + return false; + } + add_require_for_enums = true; + } else if (options[i].first == "binary") { + if (options[i].second != "") { + *error = "Unexpected option value for binary"; + return false; + } + binary = true; + } else if (options[i].first == "testonly") { + if (options[i].second != "") { + *error = "Unexpected option value for testonly"; + return false; + } + testonly = true; + } else if (options[i].first == "error_on_name_conflict") { + if (options[i].second != "") { + *error = "Unexpected option value for error_on_name_conflict"; + return false; + } + error_on_name_conflict = true; + } else if (options[i].first == "output_dir") { + output_dir = options[i].second; + } else if (options[i].first == "namespace_prefix") { + namespace_prefix = options[i].second; + } else if (options[i].first == "library") { + library = options[i].second; + } else { + // Assume any other option is an output directory, as long as it is a bare + // `key` rather than a `key=value` option. + if (options[i].second != "") { + *error = "Unknown option: " + options[i].first; + return false; + } + output_dir = options[i].first; + } + } + + return true; +} + +void Generator::GenerateFilesInDepOrder( + const GeneratorOptions& options, + io::Printer* printer, + const vector& files) const { + // Build a std::set over all files so that the DFS can detect when it recurses + // into a dep not specified in the user's command line. + std::set all_files(files.begin(), files.end()); + // Track the in-progress set of files that have been generated already. + std::set generated; + for (int i = 0; i < files.size(); i++) { + GenerateFileAndDeps(options, printer, files[i], &all_files, &generated); + } +} + +void Generator::GenerateFileAndDeps( + const GeneratorOptions& options, + io::Printer* printer, + const FileDescriptor* root, + std::set* all_files, + std::set* generated) const { + // Skip if already generated. + if (generated->find(root) != generated->end()) { + return; + } + generated->insert(root); + + // Generate all dependencies before this file's content. + for (int i = 0; i < root->dependency_count(); i++) { + const FileDescriptor* dep = root->dependency(i); + GenerateFileAndDeps(options, printer, dep, all_files, generated); + } + + // Generate this file's content. Only generate if the file is part of the + // original set requested to be generated; i.e., don't take all transitive + // deps down to the roots. + if (all_files->find(root) != all_files->end()) { + GenerateClassesAndEnums(options, printer, root); + } +} + +bool Generator::GenerateAll(const vector& files, + const string& parameter, + GeneratorContext* context, + string* error) const { + vector< pair< string, string > > option_pairs; + ParseGeneratorParameter(parameter, &option_pairs); + GeneratorOptions options; + if (!options.ParseFromOptions(option_pairs, error)) { + return false; + } + + + // We're either generating a single library file with definitions for message + // and enum types in *all* FileDescriptor inputs, or we're generating a single + // file for each type. + if (options.library != "") { + string filename = options.output_dir + "/" + options.library + ".js"; + google::protobuf::scoped_ptr output(context->Open(filename)); + GOOGLE_CHECK(output.get()); + io::Printer printer(output.get(), '$'); + + // Pull out all extensions -- we need these to generate all + // provides/requires. + vector extensions; + for (int i = 0; i < files.size(); i++) { + for (int j = 0; j < files[i]->extension_count(); j++) { + const FieldDescriptor* extension = files[i]->extension(j); + extensions.push_back(extension); + } + } + + GenerateHeader(options, &printer); + + std::set provided; + FindProvides(options, &printer, files, &provided); + FindProvidesForFields(options, &printer, extensions, &provided); + GenerateProvides(options, &printer, &provided); + GenerateTestOnly(options, &printer); + GenerateRequires(options, &printer, files, &provided); + + GenerateFilesInDepOrder(options, &printer, files); + + for (int i = 0; i < extensions.size(); i++) { + if (ShouldGenerateExtension(extensions[i])) { + GenerateExtension(options, &printer, extensions[i]); + } + } + + if (printer.failed()) { + return false; + } + } else { + // Collect all types, and print each type to a separate file. Pull out + // free-floating extensions while we make this pass. + map< string, vector > extensions_by_namespace; + + // If we're generating code in file-per-type mode, avoid overwriting files + // by choosing the last descriptor that writes each filename and permitting + // only those to generate code. + + // Current descriptor that will generate each filename, indexed by filename. + map desc_by_filename; + // Set of descriptors allowed to generate files. + set allowed_descs; + + for (int i = 0; i < files.size(); i++) { + // Collect all (descriptor, filename) pairs. + map descs_in_file; + for (int j = 0; j < files[i]->message_type_count(); j++) { + const Descriptor* desc = files[i]->message_type(j); + string filename = + options.output_dir + "/" + ToFileName(desc->name()) + ".js"; + descs_in_file[desc] = filename; + } + for (int j = 0; j < files[i]->enum_type_count(); j++) { + const EnumDescriptor* desc = files[i]->enum_type(j); + string filename = + options.output_dir + "/" + ToFileName(desc->name()) + ".js"; + descs_in_file[desc] = filename; + } + + // For each (descriptor, filename) pair, update the + // descriptors-by-filename map, and if a previous descriptor was already + // writing the filename, remove it from the allowed-descriptors set. + map::iterator it; + for (it = descs_in_file.begin(); it != descs_in_file.end(); ++it) { + const void* desc = it->first; + const string& filename = it->second; + if (desc_by_filename.find(filename) != desc_by_filename.end()) { + if (options.error_on_name_conflict) { + *error = "Name conflict: file name " + filename + + " would be generated by two descriptors"; + return false; + } + allowed_descs.erase(desc_by_filename[filename]); + } + desc_by_filename[filename] = desc; + allowed_descs.insert(desc); + } + } + + // Generate code. + for (int i = 0; i < files.size(); i++) { + const FileDescriptor* file = files[i]; + for (int j = 0; j < file->message_type_count(); j++) { + const Descriptor* desc = file->message_type(j); + if (allowed_descs.find(desc) == allowed_descs.end()) { + continue; + } + + string filename = options.output_dir + "/" + + ToFileName(desc->name()) + ".js"; + google::protobuf::scoped_ptr output( + context->Open(filename)); + GOOGLE_CHECK(output.get()); + io::Printer printer(output.get(), '$'); + + GenerateHeader(options, &printer); + + std::set provided; + FindProvidesForMessage(options, &printer, desc, &provided); + GenerateProvides(options, &printer, &provided); + GenerateTestOnly(options, &printer); + GenerateRequires(options, &printer, desc, &provided); + + GenerateClass(options, &printer, desc); + + if (printer.failed()) { + return false; + } + } + for (int j = 0; j < file->enum_type_count(); j++) { + const EnumDescriptor* enumdesc = file->enum_type(j); + if (allowed_descs.find(enumdesc) == allowed_descs.end()) { + continue; + } + + string filename = options.output_dir + "/" + + ToFileName(enumdesc->name()) + ".js"; + + google::protobuf::scoped_ptr output( + context->Open(filename)); + GOOGLE_CHECK(output.get()); + io::Printer printer(output.get(), '$'); + + GenerateHeader(options, &printer); + + std::set provided; + FindProvidesForEnum(options, &printer, enumdesc, &provided); + GenerateProvides(options, &printer, &provided); + GenerateTestOnly(options, &printer); + + GenerateEnum(options, &printer, enumdesc); + + if (printer.failed()) { + return false; + } + } + // Pull out all free-floating extensions and generate files for those too. + for (int j = 0; j < file->extension_count(); j++) { + const FieldDescriptor* extension = file->extension(j); + extensions_by_namespace[GetPath(options, files[i])] + .push_back(extension); + } + } + + // Generate extensions in separate files. + map< string, vector >::iterator it; + for (it = extensions_by_namespace.begin(); + it != extensions_by_namespace.end(); + ++it) { + string filename = options.output_dir + "/" + + ToFileName(it->first) + ".js"; + + google::protobuf::scoped_ptr output( + context->Open(filename)); + GOOGLE_CHECK(output.get()); + io::Printer printer(output.get(), '$'); + + GenerateHeader(options, &printer); + + std::set provided; + FindProvidesForFields(options, &printer, it->second, &provided); + GenerateProvides(options, &printer, &provided); + GenerateTestOnly(options, &printer); + GenerateRequires(options, &printer, it->second, &provided); + + for (int j = 0; j < it->second.size(); j++) { + if (ShouldGenerateExtension(it->second[j])) { + GenerateExtension(options, &printer, it->second[j]); + } + } + } + } + + return true; +} + +} // namespace js +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/js/js_generator.h b/src/google/protobuf/compiler/js/js_generator.h new file mode 100755 index 00000000..db2dceb3 --- /dev/null +++ b/src/google/protobuf/compiler/js/js_generator.h @@ -0,0 +1,265 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_JS_GENERATOR_H__ +#define GOOGLE_PROTOBUF_COMPILER_JS_GENERATOR_H__ + +#include +#include + +#include + +namespace google { +namespace protobuf { + +class Descriptor; +class EnumDescriptor; +class FieldDescriptor; +class OneofDescriptor; +class FileDescriptor; + +namespace io { class Printer; } + +namespace compiler { +namespace js { + +struct GeneratorOptions { + // Add a `goog.requires()` call for each enum type used. If not set, a forward + // declaration with `goog.forwardDeclare` is produced instead. + bool add_require_for_enums; + // Set this as a test-only module via `goog.setTestOnly();`. + bool testonly; + // Output path. + string output_dir; + // Namespace prefix. + string namespace_prefix; + // Create a library with name _lib.js rather than a separate .js file + // per type? + string library; + // Error if there are two types that would generate the same output file? + bool error_on_name_conflict; + // Enable binary-format support? + bool binary; + + GeneratorOptions() + : add_require_for_enums(false), + testonly(false), + output_dir("."), + namespace_prefix(""), + library(""), + error_on_name_conflict(false), + binary(false) {} + + bool ParseFromOptions( + const vector< pair< string, string > >& options, + string* error); +}; + +class LIBPROTOC_EXPORT Generator : public CodeGenerator { + public: + Generator() {} + virtual ~Generator() {} + + virtual bool Generate(const FileDescriptor* file, + const string& parameter, + GeneratorContext* context, + string* error) const { + *error = "Unimplemented Generate() method. Call GenerateAll() instead."; + return false; + } + + virtual bool HasGenerateAll() const { return true; } + + virtual bool GenerateAll(const vector& files, + const string& parameter, + GeneratorContext* context, + string* error) const; + + private: + void GenerateHeader(const GeneratorOptions& options, + io::Printer* printer) const; + + // Generate goog.provides() calls. + void FindProvides(const GeneratorOptions& options, + io::Printer* printer, + const vector& file, + std::set* provided) const; + void FindProvidesForMessage(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc, + std::set* provided) const; + void FindProvidesForEnum(const GeneratorOptions& options, + io::Printer* printer, + const EnumDescriptor* enumdesc, + std::set* provided) const; + // For extension fields at file scope. + void FindProvidesForFields(const GeneratorOptions& options, + io::Printer* printer, + const vector& fields, + std::set* provided) const; + // Print the goog.provides() found by the methods above. + void GenerateProvides(const GeneratorOptions& options, + io::Printer* printer, + std::set* provided) const; + + // Generate goog.setTestOnly() if indicated. + void GenerateTestOnly(const GeneratorOptions& options, + io::Printer* printer) const; + + // Generate goog.requires() calls. + void GenerateRequires(const GeneratorOptions& options, + io::Printer* printer, + const vector& file, + std::set* provided) const; + void GenerateRequires(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc, + std::set* provided) const; + // For extension fields at file scope. + void GenerateRequires(const GeneratorOptions& options, + io::Printer* printer, + const vector& fields, + std::set* provided) const; + void GenerateRequiresImpl(const GeneratorOptions& options, + io::Printer* printer, + std::set* required, + std::set* forwards, + std::set* provided, + bool require_jspb, + bool require_extension) const; + void FindRequiresForMessage(const GeneratorOptions& options, + const Descriptor* desc, + std::set* required, + std::set* forwards, + bool* have_message) const; + void FindRequiresForField(const GeneratorOptions& options, + const FieldDescriptor* field, + std::set* required, + std::set* forwards) const; + void FindRequiresForExtension(const GeneratorOptions& options, + const FieldDescriptor* field, + std::set* required, + std::set* forwards) const; + + // Generate definitions for all message classes and enums in all files, + // processing the files in dependence order. + void GenerateFilesInDepOrder(const GeneratorOptions& options, + io::Printer* printer, + const vector& file) const; + // Helper for above. + void GenerateFileAndDeps(const GeneratorOptions& options, + io::Printer* printer, + const FileDescriptor* root, + std::set* all_files, + std::set* generated) const; + + // Generate definitions for all message classes and enums. + void GenerateClassesAndEnums(const GeneratorOptions& options, + io::Printer* printer, + const FileDescriptor* file) const; + + // Generate definition for one class. + void GenerateClass(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassConstructor(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassFieldInfo(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassXid(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateOneofCaseDefinition(const GeneratorOptions& options, + io::Printer* printer, + const OneofDescriptor* oneof) const; + void GenerateClassToObject(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassFieldToObject(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const; + void GenerateClassFromObject(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassFieldFromObject(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const; + void GenerateClassClone(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassRegistration(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassFields(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassField(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* desc) const; + void GenerateClassExtensionFieldInfo(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassDeserialize(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassDeserializeBinary(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassDeserializeBinaryField(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const; + void GenerateClassSerializeBinary(const GeneratorOptions& options, + io::Printer* printer, + const Descriptor* desc) const; + void GenerateClassSerializeBinaryField(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const; + + // Generate definition for one enum. + void GenerateEnum(const GeneratorOptions& options, + io::Printer* printer, + const EnumDescriptor* enumdesc) const; + + // Generate an extension definition. + void GenerateExtension(const GeneratorOptions& options, + io::Printer* printer, + const FieldDescriptor* field) const; + + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Generator); +}; + +} // namespace js +} // namespace compiler +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_COMPILER_JS_GENERATOR_H__ diff --git a/src/google/protobuf/compiler/main.cc b/src/google/protobuf/compiler/main.cc index 584e5a40..97df536e 100644 --- a/src/google/protobuf/compiler/main.cc +++ b/src/google/protobuf/compiler/main.cc @@ -38,6 +38,7 @@ #include #include #include +#include int main(int argc, char* argv[]) { @@ -80,5 +81,10 @@ int main(int argc, char* argv[]) { cli.RegisterGenerator("--objc_out", &objc_generator, "Generate Objective C header and source."); + // JavaScript + google::protobuf::compiler::js::Generator js_generator; + cli.RegisterGenerator("--js_out", &js_generator, + "Generate JavaScript source."); + return cli.Run(argc, argv); } diff --git a/src/google/protobuf/compiler/mock_code_generator.cc b/src/google/protobuf/compiler/mock_code_generator.cc index 98261431..121d917b 100644 --- a/src/google/protobuf/compiler/mock_code_generator.cc +++ b/src/google/protobuf/compiler/mock_code_generator.cc @@ -147,6 +147,12 @@ bool MockCodeGenerator::Generate( std::cerr << "Saw message type MockCodeGenerator_HasSourceCodeInfo: " << has_source_code_info << "." << std::endl; abort(); + } else if (command == "HasJsonName") { + FieldDescriptorProto field_descriptor_proto; + file->message_type(i)->field(0)->CopyTo(&field_descriptor_proto); + std::cerr << "Saw json_name: " + << field_descriptor_proto.has_json_name() << std::endl; + abort(); } else { GOOGLE_LOG(FATAL) << "Unknown MockCodeGenerator command: " << command; } diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index a389a4fc..ea792a9d 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -458,6 +458,61 @@ void Parser::SkipRestOfBlock() { // =================================================================== +bool Parser::ValidateEnum(const EnumDescriptorProto* proto) { + bool has_allow_alias = false; + bool allow_alias = false; + + for (int i = 0; i < proto->options().uninterpreted_option_size(); i++) { + const UninterpretedOption option = proto->options().uninterpreted_option(i); + if (option.name_size() > 1) { + continue; + } + if (!option.name(0).is_extension() && + option.name(0).name_part() == "allow_alias") { + has_allow_alias = true; + if (option.identifier_value() == "true") { + allow_alias = true; + } + break; + } + } + + if (has_allow_alias && !allow_alias) { + string error = + "\"" + proto->name() + + "\" declares 'option allow_alias = false;' which has no effect. " + "Please remove the declaration."; + // This needlessly clutters declarations with nops. + AddError(error); + return false; + } + + set used_values; + bool has_duplicates = false; + for (int i = 0; i < proto->value_size(); ++i) { + const EnumValueDescriptorProto enum_value = proto->value(i); + if (used_values.find(enum_value.number()) != used_values.end()) { + has_duplicates = true; + break; + } else { + used_values.insert(enum_value.number()); + } + } + if (allow_alias && !has_duplicates) { + string error = + "\"" + proto->name() + + "\" declares support for enum aliases but no enum values share field " + "numbers. Please remove the unnecessary 'option allow_alias = true;' " + "declaration."; + // Generate an error if an enum declares support for duplicate enum values + // and does not use it protect future authors. + AddError(error); + return false; + } + + return true; +} + bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { input_ = input; had_errors_ = false; @@ -1627,6 +1682,9 @@ bool Parser::ParseEnumDefinition(EnumDescriptorProto* enum_type, } DO(ParseEnumBlock(enum_type, enum_location, containing_file)); + + DO(ValidateEnum(enum_type)); + return true; } diff --git a/src/google/protobuf/compiler/parser.h b/src/google/protobuf/compiler/parser.h index 3ba1e170..2c561c23 100644 --- a/src/google/protobuf/compiler/parser.h +++ b/src/google/protobuf/compiler/parser.h @@ -498,6 +498,8 @@ class LIBPROTOBUF_EXPORT Parser { } + bool ValidateEnum(const EnumDescriptorProto* proto); + // ================================================================= io::Tokenizer* input_; diff --git a/src/google/protobuf/compiler/parser_unittest.cc b/src/google/protobuf/compiler/parser_unittest.cc index 0d729e0b..1d623dd9 100644 --- a/src/google/protobuf/compiler/parser_unittest.cc +++ b/src/google/protobuf/compiler/parser_unittest.cc @@ -1170,6 +1170,29 @@ TEST_F(ParseErrorTest, EnumValueOutOfRange) { "4:19: Integer out of range.\n"); } +TEST_F(ParseErrorTest, EnumAllowAliasFalse) { + ExpectHasErrors( + "enum Foo {\n" + " option allow_alias = false;\n" + " BAR = 1;\n" + " BAZ = 2;\n" + "}\n", + "5:0: \"Foo\" declares 'option allow_alias = false;' which has no effect. " + "Please remove the declaration.\n"); +} + +TEST_F(ParseErrorTest, UnnecessaryEnumAllowAlias) { + ExpectHasErrors( + "enum Foo {\n" + " option allow_alias = true;\n" + " BAR = 1;\n" + " BAZ = 2;\n" + "}\n", + "5:0: \"Foo\" declares support for enum aliases but no enum values share " + "field numbers. Please remove the unnecessary 'option allow_alias = true;' " + "declaration.\n"); +} + TEST_F(ParseErrorTest, DefaultValueMissing) { ExpectHasErrors( "message TestMessage {\n" @@ -1839,6 +1862,8 @@ TEST_F(ParseDescriptorDebugTest, TestCommentsInDebugString) { "// Detached comment before TestMessage1.\n" "\n" "// Message comment.\n" + "//\n" + "// More detail in message comment.\n" "message TestMessage1 {\n" "\n" " // Detached comment before foo.\n" @@ -1890,11 +1915,6 @@ TEST_F(ParseDescriptorDebugTest, TestCommentsInDebugString) { pool_.BuildFileCollectingErrors(parsed_desc, &collector); ASSERT_TRUE(descriptor != NULL); - DebugStringOptions debug_string_options; - debug_string_options.include_comments = true; - const string debug_string = - descriptor->DebugStringWithOptions(debug_string_options); - // Ensure that each of the comments appears somewhere in the DebugString(). // We don't test the exact comment placement or formatting, because we do not // want to be too fragile here. @@ -1905,6 +1925,7 @@ TEST_F(ParseDescriptorDebugTest, TestCommentsInDebugString) { "Package comment.", "Detached comment before TestMessage1.", "Message comment.", + "More detail in message comment.", "Detached comment before foo.", "Field comment", "Detached comment before NestedMessage.", @@ -1919,11 +1940,28 @@ TEST_F(ParseDescriptorDebugTest, TestCommentsInDebugString) { "RPC comment", }; - for (int i = 0; i < GOOGLE_ARRAYSIZE(expected_comments); ++i) { - string::size_type found_pos = debug_string.find(expected_comments[i]); - EXPECT_TRUE(found_pos != string::npos) - << "\"" << expected_comments[i] << "\" not found."; + DebugStringOptions debug_string_options; + debug_string_options.include_comments = true; + + { + const string debug_string = + descriptor->DebugStringWithOptions(debug_string_options); + + for (int i = 0; i < GOOGLE_ARRAYSIZE(expected_comments); ++i) { + string::size_type found_pos = debug_string.find(expected_comments[i]); + EXPECT_TRUE(found_pos != string::npos) + << "\"" << expected_comments[i] << "\" not found."; + } + + // Result of DebugStringWithOptions should be parseable. + SetupParser(debug_string.c_str()); + FileDescriptorProto parsed; + parser_->Parse(input_.get(), &parsed); + EXPECT_EQ(io::Tokenizer::TYPE_END, input_->current().type); + ASSERT_EQ("", error_collector_.text_) + << "Failed to parse:\n" << debug_string; } + } TEST_F(ParseDescriptorDebugTest, TestMaps) { diff --git a/src/google/protobuf/compiler/plugin.pb.cc b/src/google/protobuf/compiler/plugin.pb.cc index 636673ae..a2da8eee 100644 --- a/src/google/protobuf/compiler/plugin.pb.cc +++ b/src/google/protobuf/compiler/plugin.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/compiler/subprocess.cc b/src/google/protobuf/compiler/subprocess.cc index 85429924..a30ac305 100644 --- a/src/google/protobuf/compiler/subprocess.cc +++ b/src/google/protobuf/compiler/subprocess.cc @@ -47,6 +47,7 @@ #include #include + namespace google { namespace protobuf { namespace compiler { diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 9a1c6fe8..1aac360b 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -2050,6 +2050,7 @@ class SourceLocationCommentPrinter { } private: + bool have_source_loc_; SourceLocation source_loc_; DebugStringOptions options_; @@ -2951,7 +2952,8 @@ class DescriptorBuilder { const ServiceDescriptor* parent, MethodDescriptor* result); - void LogUnusedDependency(const FileDescriptor* result); + void LogUnusedDependency(const FileDescriptorProto& proto, + const FileDescriptor* result); // Must be run only after building. // @@ -3996,7 +3998,7 @@ const FileDescriptor* DescriptorBuilder::BuildFile( if (!unused_dependency_.empty()) { - LogUnusedDependency(result); + LogUnusedDependency(proto, result); } if (had_errors_) { @@ -4143,6 +4145,7 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, } } + void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, const Descriptor* parent, FieldDescriptor* result, @@ -4248,8 +4251,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, } else if (proto.default_value() == "nan") { result->default_value_float_ = numeric_limits::quiet_NaN(); } else { - result->default_value_float_ = - io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos); + result->default_value_float_ = io::SafeDoubleToFloat( + io::NoLocaleStrtod(proto.default_value().c_str(), &end_pos)); } break; case FieldDescriptor::CPPTYPE_DOUBLE: @@ -4421,6 +4424,7 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, AllocateOptions(proto.options(), result); } + AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result)); } @@ -5533,11 +5537,19 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( // UnknownFieldSet and wait there until the message is parsed by something // that does know about the options. string buf; - options->AppendToString(&buf); - GOOGLE_CHECK(options->ParseFromString(buf)) + GOOGLE_CHECK(options->AppendPartialToString(&buf)) + << "Protocol message could not be serialized."; + GOOGLE_CHECK(options->ParsePartialFromString(buf)) << "Protocol message serialized itself in invalid fashion."; + if (!options->IsInitialized()) { + builder_->AddWarning( + options_to_interpret->element_name, *original_options, + DescriptorPool::ErrorCollector::OTHER, + "Options could not be fully parsed using the proto descriptors " + "compiled into this binary. Missing required fields: " + + options->InitializationErrorString()); + } } - return !failed; } @@ -6191,7 +6203,8 @@ void DescriptorBuilder::OptionInterpreter::SetUInt64(int number, uint64 value, } } -void DescriptorBuilder::LogUnusedDependency(const FileDescriptor* result) { +void DescriptorBuilder::LogUnusedDependency(const FileDescriptorProto& proto, + const FileDescriptor* result) { if (!unused_dependency_.empty()) { std::set annotation_extensions; @@ -6217,9 +6230,9 @@ void DescriptorBuilder::LogUnusedDependency(const FileDescriptor* result) { } // Log warnings for unused imported files. if (i == (*it)->extension_count()) { - GOOGLE_LOG(WARNING) << "Warning: Unused import: \"" << result->name() - << "\" imports \"" << (*it)->name() - << "\" which is not used."; + string error_message = "Import " + (*it)->name() + " but not used."; + AddWarning((*it)->name(), proto, DescriptorPool::ErrorCollector::OTHER, + error_message); } } } diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index bc846609..eda6280f 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index ccd0650d..be8e0b72 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -178,6 +178,61 @@ void AddEmptyEnum(FileDescriptorProto* file, const string& name) { AddEnumValue(AddEnum(file, name), name + "_DUMMY", 1); } +class MockErrorCollector : public DescriptorPool::ErrorCollector { + public: + MockErrorCollector() {} + ~MockErrorCollector() {} + + string text_; + string warning_text_; + + // implements ErrorCollector --------------------------------------- + void AddError(const string& filename, + const string& element_name, const Message* descriptor, + ErrorLocation location, const string& message) { + const char* location_name = NULL; + switch (location) { + case NAME : location_name = "NAME" ; break; + case NUMBER : location_name = "NUMBER" ; break; + case TYPE : location_name = "TYPE" ; break; + case EXTENDEE : location_name = "EXTENDEE" ; break; + case DEFAULT_VALUE: location_name = "DEFAULT_VALUE"; break; + case OPTION_NAME : location_name = "OPTION_NAME" ; break; + case OPTION_VALUE : location_name = "OPTION_VALUE" ; break; + case INPUT_TYPE : location_name = "INPUT_TYPE" ; break; + case OUTPUT_TYPE : location_name = "OUTPUT_TYPE" ; break; + case OTHER : location_name = "OTHER" ; break; + } + + strings::SubstituteAndAppend( + &text_, "$0: $1: $2: $3\n", + filename, element_name, location_name, message); + } + + // implements ErrorCollector --------------------------------------- + void AddWarning(const string& filename, const string& element_name, + const Message* descriptor, ErrorLocation location, + const string& message) { + const char* location_name = NULL; + switch (location) { + case NAME : location_name = "NAME" ; break; + case NUMBER : location_name = "NUMBER" ; break; + case TYPE : location_name = "TYPE" ; break; + case EXTENDEE : location_name = "EXTENDEE" ; break; + case DEFAULT_VALUE: location_name = "DEFAULT_VALUE"; break; + case OPTION_NAME : location_name = "OPTION_NAME" ; break; + case OPTION_VALUE : location_name = "OPTION_VALUE" ; break; + case INPUT_TYPE : location_name = "INPUT_TYPE" ; break; + case OUTPUT_TYPE : location_name = "OUTPUT_TYPE" ; break; + case OTHER : location_name = "OTHER" ; break; + } + + strings::SubstituteAndAppend( + &warning_text_, "$0: $1: $2: $3\n", + filename, element_name, location_name, message); + } +}; + // =================================================================== // Test simple files. @@ -3120,77 +3175,95 @@ TEST(CustomOptions, UnusedImportWarning) { ->file()->CopyTo(&file_proto); ASSERT_TRUE(pool.BuildFile(file_proto) != NULL); - pool.AddUnusedImportTrackFile("custom_options_import.proto"); ASSERT_TRUE(TextFormat::ParseFromString( "name: \"custom_options_import.proto\" " "package: \"protobuf_unittest\" " "dependency: \"google/protobuf/unittest_custom_options.proto\" ", &file_proto)); - pool.BuildFile(file_proto); -} -// =================================================================== + MockErrorCollector error_collector; + EXPECT_TRUE(pool.BuildFileCollectingErrors(file_proto, &error_collector)); + EXPECT_EQ("", error_collector.warning_text_); +} -// The tests below trigger every unique call to AddError() in descriptor.cc, -// in the order in which they appear in that file. I'm using TextFormat here -// to specify the input descriptors because building them using code would -// be too bulky. +// Verifies that proto files can correctly be parsed, even if the +// custom options defined in the file are incompatible with those +// compiled in the binary. See http://b/19276250. +TEST(CustomOptions, OptionsWithRequiredEnums) { + DescriptorPool pool; -class MockErrorCollector : public DescriptorPool::ErrorCollector { - public: - MockErrorCollector() {} - ~MockErrorCollector() {} + FileDescriptorProto file_proto; + MessageOptions::descriptor()->file()->CopyTo(&file_proto); + ASSERT_TRUE(pool.BuildFile(file_proto) != NULL); - string text_; - string warning_text_; + // Create a new file descriptor proto containing a subset of the + // messages defined in google/protobuf/unittest_custom_options.proto. + file_proto.Clear(); + file_proto.set_name("unittest_custom_options.proto"); + file_proto.set_package("protobuf_unittest"); + file_proto.add_dependency("google/protobuf/descriptor.proto"); - // implements ErrorCollector --------------------------------------- - void AddError(const string& filename, - const string& element_name, const Message* descriptor, - ErrorLocation location, const string& message) { - const char* location_name = NULL; - switch (location) { - case NAME : location_name = "NAME" ; break; - case NUMBER : location_name = "NUMBER" ; break; - case TYPE : location_name = "TYPE" ; break; - case EXTENDEE : location_name = "EXTENDEE" ; break; - case DEFAULT_VALUE: location_name = "DEFAULT_VALUE"; break; - case OPTION_NAME : location_name = "OPTION_NAME" ; break; - case OPTION_VALUE : location_name = "OPTION_VALUE" ; break; - case INPUT_TYPE : location_name = "INPUT_TYPE" ; break; - case OUTPUT_TYPE : location_name = "OUTPUT_TYPE" ; break; - case OTHER : location_name = "OTHER" ; break; - } + // Add the "required_enum_opt" extension. + FieldDescriptorProto* extension = file_proto.add_extension(); + protobuf_unittest::OldOptionType::descriptor()->file() + ->FindExtensionByName("required_enum_opt")->CopyTo(extension); + + // Add a test message that uses the "required_enum_opt" option. + DescriptorProto* test_message_type = file_proto.add_message_type(); + protobuf_unittest::TestMessageWithRequiredEnumOption::descriptor() + ->CopyTo(test_message_type); + + // Instruct the extension to use NewOptionType instead of + // OldOptionType, and add the descriptor of NewOptionType. + extension->set_type_name(".protobuf_unittest.NewOptionType"); + DescriptorProto* new_option_type = file_proto.add_message_type(); + protobuf_unittest::NewOptionType::descriptor() + ->CopyTo(new_option_type); + + // Replace the value of the "required_enum_opt" option used in the + // test message with an enum value that only exists in NewOptionType. + ASSERT_TRUE(TextFormat::ParseFromString( + "uninterpreted_option { " + " name { " + " name_part: 'required_enum_opt' " + " is_extension: true " + " } " + " aggregate_value: 'value: NEW_VALUE' " + "}", + test_message_type->mutable_options())); - strings::SubstituteAndAppend( - &text_, "$0: $1: $2: $3\n", - filename, element_name, location_name, message); - } + // Add the file descriptor to the pool. + ASSERT_TRUE(pool.BuildFile(file_proto) != NULL); - // implements ErrorCollector --------------------------------------- - void AddWarning(const string& filename, const string& element_name, - const Message* descriptor, ErrorLocation location, - const string& message) { - const char* location_name = NULL; - switch (location) { - case NAME : location_name = "NAME" ; break; - case NUMBER : location_name = "NUMBER" ; break; - case TYPE : location_name = "TYPE" ; break; - case EXTENDEE : location_name = "EXTENDEE" ; break; - case DEFAULT_VALUE: location_name = "DEFAULT_VALUE"; break; - case OPTION_NAME : location_name = "OPTION_NAME" ; break; - case OPTION_VALUE : location_name = "OPTION_VALUE" ; break; - case INPUT_TYPE : location_name = "INPUT_TYPE" ; break; - case OUTPUT_TYPE : location_name = "OUTPUT_TYPE" ; break; - case OTHER : location_name = "OTHER" ; break; - } + // Find the test message. + const Descriptor* test_message = pool.FindMessageTypeByName( + "protobuf_unittest.TestMessageWithRequiredEnumOption"); + ASSERT_TRUE(test_message != NULL); + + const MessageOptions& options = test_message->options(); + // Extract the "required_enum_opt" option. Since the binary does not + // know that the extension was updated, this will still return an + // OldOptionType message. + ASSERT_TRUE( + options.HasExtension(protobuf_unittest::required_enum_opt)); + const protobuf_unittest::OldOptionType& old_enum_opt = + options.GetExtension(protobuf_unittest::required_enum_opt); + + // Confirm that the required enum field is missing. + EXPECT_FALSE(old_enum_opt.IsInitialized()); + EXPECT_FALSE(old_enum_opt.has_value()); + + string buf; + // Verify that the required enum field does show up when the option + // is re-parsed as a NewOptionType message; + protobuf_unittest::NewOptionType new_enum_opt; + EXPECT_TRUE(old_enum_opt.AppendPartialToString(&buf)); + EXPECT_TRUE(new_enum_opt.ParseFromString(buf)); + EXPECT_EQ(protobuf_unittest::NewOptionType::NEW_VALUE, new_enum_opt.value()); +} - strings::SubstituteAndAppend( - &warning_text_, "$0: $1: $2: $3\n", - filename, element_name, location_name, message); - } -}; +// =================================================================== class ValidationErrorTest : public testing::Test { protected: @@ -5123,7 +5196,6 @@ TEST_F(ValidationErrorTest, AllowEnumAlias) { } TEST_F(ValidationErrorTest, UnusedImportWarning) { - pool_.AddUnusedImportTrackFile("bar.proto"); BuildFile( "name: \"bar.proto\" " @@ -5155,7 +5227,7 @@ TEST_F(ValidationErrorTest, UnusedImportWarning) { // } // pool_.AddUnusedImportTrackFile("forward.proto"); - BuildFile( + BuildFileWithWarnings( "name: \"forward.proto\"" "dependency: \"base.proto\"" "dependency: \"bar.proto\"" @@ -5165,7 +5237,8 @@ TEST_F(ValidationErrorTest, UnusedImportWarning) { "message_type {" " name: \"Forward\"" " field { name:\"base\" number:1 label:LABEL_OPTIONAL type_name:\"Base\" }" - "}"); + "}", + "forward.proto: bar.proto: OTHER: Import bar.proto but not used.\n"); } namespace { diff --git a/src/google/protobuf/duration.pb.cc b/src/google/protobuf/duration.pb.cc index c7eed0d5..b325944e 100644 --- a/src/google/protobuf/duration.pb.cc +++ b/src/google/protobuf/duration.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/duration.proto b/src/google/protobuf/duration.proto index 7f172aa6..78bcc74b 100644 --- a/src/google/protobuf/duration.proto +++ b/src/google/protobuf/duration.proto @@ -27,15 +27,16 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "DurationProto"; -option java_package = "com.google.protobuf"; option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DurationProto"; +option java_multiple_files = true; +option java_generate_equals_and_hash = true; option objc_class_prefix = "GPB"; // A Duration represents a signed, fixed-length span of time represented diff --git a/src/google/protobuf/dynamic_message.cc b/src/google/protobuf/dynamic_message.cc index 091fc975..bb400476 100644 --- a/src/google/protobuf/dynamic_message.cc +++ b/src/google/protobuf/dynamic_message.cc @@ -69,8 +69,8 @@ #include #endif -#include #include +#include #include #include diff --git a/src/google/protobuf/empty.pb.cc b/src/google/protobuf/empty.pb.cc index 4a4f6eae..f2eec782 100644 --- a/src/google/protobuf/empty.pb.cc +++ b/src/google/protobuf/empty.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -79,9 +80,9 @@ void protobuf_AddDesc_google_2fprotobuf_2fempty_2eproto() { ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( "\n\033google/protobuf/empty.proto\022\017google.pr" - "otobuf\"\007\n\005EmptyBM\n\023com.google.protobufB\n" - "EmptyProtoP\001\240\001\001\242\002\003GPB\252\002\036Google.Protobuf." - "WellKnownTypesb\006proto3", 142); + "otobuf\"\007\n\005EmptyBP\n\023com.google.protobufB\n" + "EmptyProtoP\001\240\001\001\370\001\001\242\002\003GPB\252\002\036Google.Protob" + "uf.WellKnownTypesb\006proto3", 145); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "google/protobuf/empty.proto", &protobuf_RegisterTypes); Empty::default_instance_ = new Empty(); @@ -117,6 +118,14 @@ Empty::Empty() // @@protoc_insertion_point(constructor:google.protobuf.Empty) } +Empty::Empty(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.Empty) +} + void Empty::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -140,10 +149,20 @@ Empty::~Empty() { } void Empty::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void Empty::ArenaDtor(void* object) { + Empty* _this = reinterpret_cast< Empty* >(object); + (void)_this; +} +void Empty::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void Empty::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -162,11 +181,7 @@ const Empty& Empty::default_instance() { Empty* Empty::default_instance_ = NULL; Empty* Empty::New(::google::protobuf::Arena* arena) const { - Empty* n = new Empty; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void Empty::Clear() { @@ -255,6 +270,18 @@ bool Empty::IsInitialized() const { void Empty::Swap(Empty* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + Empty temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void Empty::UnsafeArenaSwap(Empty* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void Empty::InternalSwap(Empty* other) { diff --git a/src/google/protobuf/empty.pb.h b/src/google/protobuf/empty.pb.h index 20876bea..868009fc 100644 --- a/src/google/protobuf/empty.pb.h +++ b/src/google/protobuf/empty.pb.h @@ -53,9 +53,14 @@ class LIBPROTOBUF_EXPORT Empty : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const Empty& default_instance(); + void UnsafeArenaSwap(Empty* other); void Swap(Empty* other); // implements Message ---------------------------------------------- @@ -82,6 +87,11 @@ class LIBPROTOBUF_EXPORT Empty : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(Empty* other); + protected: + explicit Empty(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -101,6 +111,9 @@ class LIBPROTOBUF_EXPORT Empty : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; mutable int _cached_size_; friend void LIBPROTOBUF_EXPORT protobuf_AddDesc_google_2fprotobuf_2fempty_2eproto(); diff --git a/src/google/protobuf/empty.proto b/src/google/protobuf/empty.proto index 9dddc6c5..b96daf28 100644 --- a/src/google/protobuf/empty.proto +++ b/src/google/protobuf/empty.proto @@ -27,16 +27,18 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_multiple_files = true; -option java_outer_classname = "EmptyProto"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option java_package = "com.google.protobuf"; +option java_outer_classname = "EmptyProto"; +option java_multiple_files = true; option java_generate_equals_and_hash = true; -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; // A generic empty message that you can re-use to avoid defining duplicated // empty messages in your APIs. A typical example is to use it as the request diff --git a/src/google/protobuf/field_mask.pb.cc b/src/google/protobuf/field_mask.pb.cc index f834363b..01a6ce56 100644 --- a/src/google/protobuf/field_mask.pb.cc +++ b/src/google/protobuf/field_mask.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/field_mask.proto b/src/google/protobuf/field_mask.proto index 8b21c692..908c8a86 100644 --- a/src/google/protobuf/field_mask.proto +++ b/src/google/protobuf/field_mask.proto @@ -27,16 +27,17 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "FieldMaskProto"; -option java_package = "com.google.protobuf"; option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "FieldMaskProto"; +option java_multiple_files = true; option objc_class_prefix = "GPB"; +option java_generate_equals_and_hash = true; // `FieldMask` represents a set of symbolic field paths, for example: // @@ -69,7 +70,7 @@ option objc_class_prefix = "GPB"; // z: 8 // // The result will not contain specific values for fields x,y and z -// (there value will be set to the default, and omitted in proto text +// (their value will be set to the default, and omitted in proto text // output): // // diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index cb1869a7..2da096c5 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -109,6 +109,7 @@ #ifndef GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ #define GOOGLE_PROTOBUF_IO_CODED_STREAM_H__ +#include #include #include #ifdef _MSC_VER diff --git a/src/google/protobuf/io/strtod.cc b/src/google/protobuf/io/strtod.cc index 579de9aa..a90bb9a3 100644 --- a/src/google/protobuf/io/strtod.cc +++ b/src/google/protobuf/io/strtod.cc @@ -32,6 +32,7 @@ #include #include +#include #include #include @@ -109,6 +110,16 @@ double NoLocaleStrtod(const char* text, char** original_endptr) { return result; } +float SafeDoubleToFloat(double value) { + if (value > std::numeric_limits::max()) { + return std::numeric_limits::infinity(); + } else if (value < -std::numeric_limits::max()) { + return -std::numeric_limits::infinity(); + } else { + return static_cast(value); + } +} + } // namespace io } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/io/strtod.h b/src/google/protobuf/io/strtod.h index c2efc8d3..f56e41c8 100644 --- a/src/google/protobuf/io/strtod.h +++ b/src/google/protobuf/io/strtod.h @@ -43,6 +43,11 @@ namespace io { // uses a dot as the decimal separator. double NoLocaleStrtod(const char* str, char** endptr); +// Casts a double value to a float value. If the value is outside of the +// representable range of float, it will be converted to positive or negative +// infinity. +float SafeDoubleToFloat(double value); + } // namespace io } // namespace protobuf diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index 7ccd633d..3d57707c 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -375,7 +375,7 @@ void Tokenizer::ConsumeString(char delimiter) { // Possibly followed by two more octal digits, but these will // just be consumed by the main loop anyway so we don't need // to do so explicitly here. - } else if (TryConsume('x') || TryConsume('X')) { + } else if (TryConsume('x')) { if (!TryConsumeOne()) { AddError("Expected hex digits for escape sequence."); } diff --git a/src/google/protobuf/io/tokenizer_unittest.cc b/src/google/protobuf/io/tokenizer_unittest.cc index 6526056a..20d50a2c 100644 --- a/src/google/protobuf/io/tokenizer_unittest.cc +++ b/src/google/protobuf/io/tokenizer_unittest.cc @@ -875,6 +875,8 @@ ErrorCase kErrorCases[] = { // String errors. { "'\\l' foo", true, "0:2: Invalid escape sequence in string literal.\n" }, + { "'\\X' foo", true, + "0:2: Invalid escape sequence in string literal.\n" }, { "'\\x' foo", true, "0:3: Expected hex digits for escape sequence.\n" }, { "'foo", false, diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h index 4bd76f25..dfc62420 100644 --- a/src/google/protobuf/map.h +++ b/src/google/protobuf/map.h @@ -188,7 +188,6 @@ class LIBPROTOBUF_EXPORT MapKey { case FieldDescriptor::CPPTYPE_ENUM: case FieldDescriptor::CPPTYPE_MESSAGE: GOOGLE_LOG(FATAL) << "Can't get here."; - return false; } GOOGLE_LOG(FATAL) << "Can't get here."; return false; @@ -497,7 +496,7 @@ class Map { insert(other.begin(), other.end()); } template - explicit Map(const InputIt& first, const InputIt& last) + Map(const InputIt& first, const InputIt& last) : arena_(NULL), allocator_(arena_), elements_(0, hasher(), key_equal(), allocator_), @@ -546,9 +545,7 @@ class Map { } #if __cplusplus >= 201103L && !defined(GOOGLE_PROTOBUF_OS_APPLE) && \ - !defined(GOOGLE_PROTOBUF_OS_NACL) && \ - !defined(GOOGLE_PROTOBUF_OS_ANDROID) && \ - !defined(GOOGLE_PROTOBUF_OS_EMSCRIPTEN) + !defined(GOOGLE_PROTOBUF_OS_NACL) && !defined(GOOGLE_PROTOBUF_OS_ANDROID) template void construct(NodeType* p, Args&&... args) { new (static_cast(p)) NodeType(std::forward(args)...); @@ -586,21 +583,22 @@ class Map { private: typedef void DestructorSkippable_; - Arena* arena_; + Arena* const arena_; template friend class MapAllocator; }; - public: typedef MapAllocator*> > Allocator; + typedef hash_map, equal_to, Allocator> + InnerMap; + public: // Iterators class const_iterator : public std::iterator { - typedef typename hash_map, equal_to, - Allocator>::const_iterator InnerIt; + typedef typename InnerMap::const_iterator InnerIt; public: const_iterator() {} @@ -627,8 +625,7 @@ class Map { }; class iterator : public std::iterator { - typedef typename hash_map, - Allocator>::iterator InnerIt; + typedef typename InnerMap::iterator InnerIt; public: iterator() {} @@ -744,8 +741,7 @@ class Map { // Erase size_type erase(const key_type& key) { - typename hash_map, equal_to, - Allocator>::iterator it = elements_.find(key); + typename InnerMap::iterator it = elements_.find(key); if (it == elements_.end()) { return 0; } else { @@ -815,7 +811,7 @@ class Map { Arena* arena_; Allocator allocator_; - hash_map, equal_to, Allocator> elements_; + InnerMap elements_; int default_enum_value_; friend class ::google::protobuf::Arena; @@ -854,7 +850,6 @@ struct hash { case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: GOOGLE_LOG(FATAL) << "Can't get here."; - return 0; } GOOGLE_LOG(FATAL) << "Can't get here."; return 0; @@ -879,7 +874,6 @@ struct hash { case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: GOOGLE_LOG(FATAL) << "Can't get here."; - return true; } GOOGLE_LOG(FATAL) << "Can't get here."; return true; diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index 5530fefe..432236ce 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -208,10 +208,19 @@ class RepeatedField { // sizeof(*this) int SpaceUsedExcludingSelf() const; - // Remove the element referenced by position. + // Removes the element referenced by position. + // + // Returns an iterator to the element immediately following the removed + // element. + // + // Invalidates all iterators at or after the removed element, including end(). iterator erase(const_iterator position); - // Remove the elements in the range [first, last). + // Removes the elements in the range [first, last). + // + // Returns an iterator to the element immediately following the removed range. + // + // Invalidates all iterators at or after the removed range, including end(). iterator erase(const_iterator first, const_iterator last); // Get the Arena on which this RepeatedField stores its elements. @@ -885,10 +894,19 @@ class RepeatedPtrField : public internal::RepeatedPtrFieldBase { // so will trigger a GOOGLE_DCHECK-failure. Element* ReleaseCleared(); - // Remove the element referenced by position. + // Removes the element referenced by position. + // + // Returns an iterator to the element immediately following the removed + // element. + // + // Invalidates all iterators at or after the removed element, including end(). iterator erase(const_iterator position); // Removes the elements in the range [first, last). + // + // Returns an iterator to the element immediately following the removed range. + // + // Invalidates all iterators at or after the removed range, including end(). iterator erase(const_iterator first, const_iterator last); // Gets the arena on which this RepeatedPtrField stores its elements. diff --git a/src/google/protobuf/source_context.pb.cc b/src/google/protobuf/source_context.pb.cc index 1be3297e..f2eb7ae7 100644 --- a/src/google/protobuf/source_context.pb.cc +++ b/src/google/protobuf/source_context.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/source_context.proto b/src/google/protobuf/source_context.proto index e9a27d65..d76252ca 100644 --- a/src/google/protobuf/source_context.proto +++ b/src/google/protobuf/source_context.proto @@ -27,15 +27,16 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_multiple_files = true; -option java_outer_classname = "SourceContextProto"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option java_package = "com.google.protobuf"; +option java_outer_classname = "SourceContextProto"; +option java_multiple_files = true; option java_generate_equals_and_hash = true; -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option objc_class_prefix = "GPB"; // `SourceContext` represents information about the source of a diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index 273645d9..e020597a 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include diff --git a/src/google/protobuf/struct.proto b/src/google/protobuf/struct.proto index b3e9e699..8562e2c1 100644 --- a/src/google/protobuf/struct.proto +++ b/src/google/protobuf/struct.proto @@ -27,15 +27,16 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "StructProto"; -option java_package = "com.google.protobuf"; option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "StructProto"; +option java_multiple_files = true; +option java_generate_equals_and_hash = true; option objc_class_prefix = "GPB"; diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc index 8442f2ce..7ba92e8f 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc @@ -524,27 +524,81 @@ int CEscapeInternal(const char* src, int src_len, char* dest, return used; } -int CEscapeString(const char* src, int src_len, char* dest, int dest_len) { - return CEscapeInternal(src, src_len, dest, dest_len, false, false); +// Calculates the length of the C-style escaped version of 'src'. +// Assumes that non-printable characters are escaped using octal sequences, and +// that UTF-8 bytes are not handled specially. +static inline size_t CEscapedLength(StringPiece src) { + static char c_escaped_len[256] = { + 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 4, 4, 2, 4, 4, // \t, \n, \r + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // ", ' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // '0'..'9' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 'A'..'O' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, // 'P'..'Z', '\' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 'a'..'o' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, // 'p'..'z', DEL + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + }; + + size_t escaped_len = 0; + for (int i = 0; i < src.size(); ++i) { + unsigned char c = static_cast(src[i]); + escaped_len += c_escaped_len[c]; + } + return escaped_len; } // ---------------------------------------------------------------------- -// CEscape() -// CHexEscape() -// Copies 'src' to result, escaping dangerous characters using -// C-style escape sequences. This is very useful for preparing query -// flags. 'src' and 'dest' should not overlap. The 'Hex' version -// hexadecimal rather than octal sequences. -// -// Currently only \n, \r, \t, ", ', \ and !isprint() chars are escaped. +// Escapes 'src' using C-style escape sequences, and appends the escaped string +// to 'dest'. This version is faster than calling CEscapeInternal as it computes +// the required space using a lookup table, and also does not do any special +// handling for Hex or UTF-8 characters. // ---------------------------------------------------------------------- +void CEscapeAndAppend(StringPiece src, string* dest) { + size_t escaped_len = CEscapedLength(src); + if (escaped_len == src.size()) { + dest->append(src.data(), src.size()); + return; + } + + size_t cur_dest_len = dest->size(); + dest->resize(cur_dest_len + escaped_len); + char* append_ptr = &(*dest)[cur_dest_len]; + + for (int i = 0; i < src.size(); ++i) { + unsigned char c = static_cast(src[i]); + switch (c) { + case '\n': *append_ptr++ = '\\'; *append_ptr++ = 'n'; break; + case '\r': *append_ptr++ = '\\'; *append_ptr++ = 'r'; break; + case '\t': *append_ptr++ = '\\'; *append_ptr++ = 't'; break; + case '\"': *append_ptr++ = '\\'; *append_ptr++ = '\"'; break; + case '\'': *append_ptr++ = '\\'; *append_ptr++ = '\''; break; + case '\\': *append_ptr++ = '\\'; *append_ptr++ = '\\'; break; + default: + if (!isprint(c)) { + *append_ptr++ = '\\'; + *append_ptr++ = '0' + c / 64; + *append_ptr++ = '0' + (c % 64) / 8; + *append_ptr++ = '0' + c % 8; + } else { + *append_ptr++ = c; + } + break; + } + } +} + string CEscape(const string& src) { - const int dest_length = src.size() * 4 + 1; // Maximum possible expansion - scoped_array dest(new char[dest_length]); - const int len = CEscapeInternal(src.data(), src.size(), - dest.get(), dest_length, false, false); - GOOGLE_DCHECK_GE(len, 0); - return string(dest.get(), len); + string dest; + CEscapeAndAppend(src, &dest); + return dest; } namespace strings { diff --git a/src/google/protobuf/stubs/strutil.h b/src/google/protobuf/stubs/strutil.h index b22066b6..27d47575 100644 --- a/src/google/protobuf/stubs/strutil.h +++ b/src/google/protobuf/stubs/strutil.h @@ -314,26 +314,20 @@ LIBPROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest, LIBPROTOBUF_EXPORT string UnescapeCEscapeString(const string& src); // ---------------------------------------------------------------------- -// CEscapeString() -// Copies 'src' to 'dest', escaping dangerous characters using -// C-style escape sequences. This is very useful for preparing query -// flags. 'src' and 'dest' should not overlap. -// Returns the number of bytes written to 'dest' (not including the \0) -// or -1 if there was insufficient space. +// CEscape() +// Escapes 'src' using C-style escape sequences and returns the resulting +// string. // -// Currently only \n, \r, \t, ", ', \ and !isprint() chars are escaped. +// Escaped chars: \n, \r, \t, ", ', \, and !isprint(). // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT int CEscapeString(const char* src, int src_len, - char* dest, int dest_len); +LIBPROTOBUF_EXPORT string CEscape(const string& src); // ---------------------------------------------------------------------- -// CEscape() -// More convenient form of CEscapeString: returns result as a "string". -// This version is slower than CEscapeString() because it does more -// allocation. However, it is much more convenient to use in -// non-speed-critical code like logging messages etc. +// CEscapeAndAppend() +// Escapes 'src' using C-style escape sequences, and appends the escaped +// string to 'dest'. // ---------------------------------------------------------------------- -LIBPROTOBUF_EXPORT string CEscape(const string& src); +LIBPROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, string* dest); namespace strings { // Like CEscape() but does not escape bytes with the upper bit set. diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc index 38b9069b..b0a5ce63 100644 --- a/src/google/protobuf/text_format.cc +++ b/src/google/protobuf/text_format.cc @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -380,6 +381,7 @@ class TextFormat::Parser::ParserImpl { string full_type_name, prefix; DO(ConsumeAnyTypeUrl(&full_type_name, &prefix)); DO(Consume("]")); + TryConsume(":"); // ':' is optional between message labels and values. string serialized_value; DO(ConsumeAnyValue(full_type_name, message->GetDescriptor()->file()->pool(), @@ -389,7 +391,6 @@ class TextFormat::Parser::ParserImpl { string(prefix + full_type_name)); reflection->SetString(message, any_value_field, serialized_value); return true; - // Fall through. } if (TryConsume("[")) { // Extension. @@ -656,7 +657,7 @@ class TextFormat::Parser::ParserImpl { case FieldDescriptor::CPPTYPE_FLOAT: { double value; DO(ConsumeDouble(&value)); - SET_FIELD(Float, static_cast(value)); + SET_FIELD(Float, io::SafeDoubleToFloat(value)); break; } @@ -1357,7 +1358,10 @@ string TextFormat::FieldValuePrinter::PrintDouble(double val) const { return SimpleDtoa(val); } string TextFormat::FieldValuePrinter::PrintString(const string& val) const { - return StrCat("\"", CEscape(val), "\""); + string printed("\""); + CEscapeAndAppend(val, &printed); + printed.push_back('\"'); + return printed; } string TextFormat::FieldValuePrinter::PrintBytes(const string& val) const { return PrintString(val); @@ -1423,7 +1427,8 @@ TextFormat::Printer::Printer() use_short_repeated_primitives_(false), hide_unknown_fields_(false), print_message_fields_in_index_order_(false), - expand_any_(false) { + expand_any_(false), + truncate_string_field_longer_than_(0LL) { SetUseUtf8StringEscaping(false); } @@ -1775,11 +1780,21 @@ void TextFormat::Printer::PrintFieldValue( ? reflection->GetRepeatedStringReference( message, field, index, &scratch) : reflection->GetStringReference(message, field, &scratch); + int64 size = value.size(); + if (truncate_string_field_longer_than_ > 0) { + size = std::min(truncate_string_field_longer_than_, + static_cast(value.size())); + } + string truncated_value(value.substr(0, size) + "......"); + const string* value_to_print = &value; + if (size < value.size()) { + value_to_print = &truncated_value; + } if (field->type() == FieldDescriptor::TYPE_STRING) { - generator.Print(printer->PrintString(value)); + generator.Print(printer->PrintString(*value_to_print)); } else { GOOGLE_DCHECK_EQ(field->type(), FieldDescriptor::TYPE_BYTES); - generator.Print(printer->PrintBytes(value)); + generator.Print(printer->PrintBytes(*value_to_print)); } break; } @@ -1926,14 +1941,10 @@ void TextFormat::Printer::PrintUnknownFields( } else { // This field is not parseable as a Message. // So it is probably just a plain string. - generator.Print(": \""); - generator.Print(CEscape(value)); - generator.Print("\""); - if (single_line_mode_) { - generator.Print(" "); - } else { - generator.Print("\n"); - } + string printed(": \""); + CEscapeAndAppend(value, &printed); + printed.append(single_line_mode_ ? "\" " : "\"\n"); + generator.Print(printed); } break; } diff --git a/src/google/protobuf/text_format.h b/src/google/protobuf/text_format.h index 6717aecd..ef3d4a8f 100644 --- a/src/google/protobuf/text_format.h +++ b/src/google/protobuf/text_format.h @@ -219,6 +219,18 @@ class LIBPROTOBUF_EXPORT TextFormat { expand_any_ = expand; } + // If non-zero, we truncate all string fields that are longer than this + // threshold. This is useful when the proto message has very long strings, + // e.g., dump of encoded image file. + // + // NOTE(hfgong): Setting a non-zero value breaks round-trip safe + // property of TextFormat::Printer. That is, from the printed message, we + // cannot fully recover the original string field any more. + void SetTruncateStringFieldLongerThan( + const int64 truncate_string_field_longer_than) { + truncate_string_field_longer_than_ = truncate_string_field_longer_than; + } + // Register a custom field-specific FieldValuePrinter for fields // with a particular FieldDescriptor. // Returns "true" if the registration succeeded, or "false", if there is @@ -286,6 +298,8 @@ class LIBPROTOBUF_EXPORT TextFormat { bool expand_any_; + int64 truncate_string_field_longer_than_; + google::protobuf::scoped_ptr default_field_value_printer_; typedef map CustomPrinterMap; diff --git a/src/google/protobuf/timestamp.pb.cc b/src/google/protobuf/timestamp.pb.cc index f0b09195..c1c4402c 100644 --- a/src/google/protobuf/timestamp.pb.cc +++ b/src/google/protobuf/timestamp.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -82,9 +83,9 @@ void protobuf_AddDesc_google_2fprotobuf_2ftimestamp_2eproto() { ::google::protobuf::DescriptorPool::InternalAddGeneratedFile( "\n\037google/protobuf/timestamp.proto\022\017googl" "e.protobuf\"+\n\tTimestamp\022\017\n\007seconds\030\001 \001(\003" - "\022\r\n\005nanos\030\002 \001(\005BQ\n\023com.google.protobufB\016" - "TimestampProtoP\001\240\001\001\242\002\003GPB\252\002\036Google.Proto" - "buf.WellKnownTypesb\006proto3", 186); + "\022\r\n\005nanos\030\002 \001(\005BT\n\023com.google.protobufB\016" + "TimestampProtoP\001\240\001\001\370\001\001\242\002\003GPB\252\002\036Google.Pr" + "otobuf.WellKnownTypesb\006proto3", 189); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "google/protobuf/timestamp.proto", &protobuf_RegisterTypes); Timestamp::default_instance_ = new Timestamp(); @@ -122,6 +123,14 @@ Timestamp::Timestamp() // @@protoc_insertion_point(constructor:google.protobuf.Timestamp) } +Timestamp::Timestamp(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.Timestamp) +} + void Timestamp::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -147,10 +156,20 @@ Timestamp::~Timestamp() { } void Timestamp::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void Timestamp::ArenaDtor(void* object) { + Timestamp* _this = reinterpret_cast< Timestamp* >(object); + (void)_this; +} +void Timestamp::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void Timestamp::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -169,11 +188,7 @@ const Timestamp& Timestamp::default_instance() { Timestamp* Timestamp::default_instance_ = NULL; Timestamp* Timestamp::New(::google::protobuf::Arena* arena) const { - Timestamp* n = new Timestamp; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void Timestamp::Clear() { @@ -349,6 +364,18 @@ bool Timestamp::IsInitialized() const { void Timestamp::Swap(Timestamp* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + Timestamp temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void Timestamp::UnsafeArenaSwap(Timestamp* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void Timestamp::InternalSwap(Timestamp* other) { diff --git a/src/google/protobuf/timestamp.pb.h b/src/google/protobuf/timestamp.pb.h index 85fc1242..7bf62597 100644 --- a/src/google/protobuf/timestamp.pb.h +++ b/src/google/protobuf/timestamp.pb.h @@ -53,9 +53,14 @@ class LIBPROTOBUF_EXPORT Timestamp : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const Timestamp& default_instance(); + void UnsafeArenaSwap(Timestamp* other); void Swap(Timestamp* other); // implements Message ---------------------------------------------- @@ -82,6 +87,11 @@ class LIBPROTOBUF_EXPORT Timestamp : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(Timestamp* other); + protected: + explicit Timestamp(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -113,6 +123,9 @@ class LIBPROTOBUF_EXPORT Timestamp : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::int64 seconds_; ::google::protobuf::int32 nanos_; diff --git a/src/google/protobuf/timestamp.proto b/src/google/protobuf/timestamp.proto index 06b60e6f..b51fc3fa 100644 --- a/src/google/protobuf/timestamp.proto +++ b/src/google/protobuf/timestamp.proto @@ -27,15 +27,17 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "TimestampProto"; -option java_package = "com.google.protobuf"; option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option java_generate_equals_and_hash = true; option objc_class_prefix = "GPB"; // A Timestamp represents a point in time independent of any time zone @@ -92,6 +94,7 @@ option objc_class_prefix = "GPB"; // nanos = int((now - seconds) * 10**9) // timestamp = Timestamp(seconds=seconds, nanos=nanos) // +// message Timestamp { // Represents seconds of UTC time since Unix epoch diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index 5792dff2..7b47b3bd 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -70,7 +71,7 @@ void protobuf_AssignDesc_google_2fprotobuf_2ftype_2eproto() { GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Type, _internal_metadata_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Type, _is_default_instance_)); Field_descriptor_ = file->message_type(1); - static const int Field_offsets_[9] = { + static const int Field_offsets_[10] = { GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, kind_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, cardinality_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, number_), @@ -80,6 +81,7 @@ void protobuf_AssignDesc_google_2fprotobuf_2ftype_2eproto() { GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, packed_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, options_), GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, json_name_), + GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(Field, default_value_), }; Field_reflection_ = ::google::protobuf::internal::GeneratedMessageReflection::NewGeneratedMessageReflection( @@ -203,37 +205,37 @@ void protobuf_AddDesc_google_2fprotobuf_2ftype_2eproto() { "\030\004 \003(\0132\027.google.protobuf.Option\0226\n\016sourc" "e_context\030\005 \001(\0132\036.google.protobuf.Source" "Context\022\'\n\006syntax\030\006 \001(\0162\027.google.protobu" - "f.Syntax\"\276\005\n\005Field\022)\n\004kind\030\001 \001(\0162\033.googl" + "f.Syntax\"\325\005\n\005Field\022)\n\004kind\030\001 \001(\0162\033.googl" "e.protobuf.Field.Kind\0227\n\013cardinality\030\002 \001" "(\0162\".google.protobuf.Field.Cardinality\022\016" "\n\006number\030\003 \001(\005\022\014\n\004name\030\004 \001(\t\022\020\n\010type_url" "\030\006 \001(\t\022\023\n\013oneof_index\030\007 \001(\005\022\016\n\006packed\030\010 " "\001(\010\022(\n\007options\030\t \003(\0132\027.google.protobuf.O" - "ption\022\021\n\tjson_name\030\n \001(\t\"\310\002\n\004Kind\022\020\n\014TYP" - "E_UNKNOWN\020\000\022\017\n\013TYPE_DOUBLE\020\001\022\016\n\nTYPE_FLO" - "AT\020\002\022\016\n\nTYPE_INT64\020\003\022\017\n\013TYPE_UINT64\020\004\022\016\n" - "\nTYPE_INT32\020\005\022\020\n\014TYPE_FIXED64\020\006\022\020\n\014TYPE_" - "FIXED32\020\007\022\r\n\tTYPE_BOOL\020\010\022\017\n\013TYPE_STRING\020" - "\t\022\016\n\nTYPE_GROUP\020\n\022\020\n\014TYPE_MESSAGE\020\013\022\016\n\nT" - "YPE_BYTES\020\014\022\017\n\013TYPE_UINT32\020\r\022\r\n\tTYPE_ENU" - "M\020\016\022\021\n\rTYPE_SFIXED32\020\017\022\021\n\rTYPE_SFIXED64\020" - "\020\022\017\n\013TYPE_SINT32\020\021\022\017\n\013TYPE_SINT64\020\022\"t\n\013C" - "ardinality\022\027\n\023CARDINALITY_UNKNOWN\020\000\022\030\n\024C" - "ARDINALITY_OPTIONAL\020\001\022\030\n\024CARDINALITY_REQ" - "UIRED\020\002\022\030\n\024CARDINALITY_REPEATED\020\003\"\316\001\n\004En" - "um\022\014\n\004name\030\001 \001(\t\022-\n\tenumvalue\030\002 \003(\0132\032.go" - "ogle.protobuf.EnumValue\022(\n\007options\030\003 \003(\013" - "2\027.google.protobuf.Option\0226\n\016source_cont" - "ext\030\004 \001(\0132\036.google.protobuf.SourceContex" - "t\022\'\n\006syntax\030\005 \001(\0162\027.google.protobuf.Synt" - "ax\"S\n\tEnumValue\022\014\n\004name\030\001 \001(\t\022\016\n\006number\030" - "\002 \001(\005\022(\n\007options\030\003 \003(\0132\027.google.protobuf" - ".Option\";\n\006Option\022\014\n\004name\030\001 \001(\t\022#\n\005value" - "\030\002 \001(\0132\024.google.protobuf.Any*.\n\006Syntax\022\021" - "\n\rSYNTAX_PROTO2\020\000\022\021\n\rSYNTAX_PROTO3\020\001BL\n\023" - "com.google.protobufB\tTypeProtoP\001\240\001\001\242\002\003GP" - "B\252\002\036Google.Protobuf.WellKnownTypesb\006prot" - "o3", 1522); + "ption\022\021\n\tjson_name\030\n \001(\t\022\025\n\rdefault_valu" + "e\030\013 \001(\t\"\310\002\n\004Kind\022\020\n\014TYPE_UNKNOWN\020\000\022\017\n\013TY" + "PE_DOUBLE\020\001\022\016\n\nTYPE_FLOAT\020\002\022\016\n\nTYPE_INT6" + "4\020\003\022\017\n\013TYPE_UINT64\020\004\022\016\n\nTYPE_INT32\020\005\022\020\n\014" + "TYPE_FIXED64\020\006\022\020\n\014TYPE_FIXED32\020\007\022\r\n\tTYPE" + "_BOOL\020\010\022\017\n\013TYPE_STRING\020\t\022\016\n\nTYPE_GROUP\020\n" + "\022\020\n\014TYPE_MESSAGE\020\013\022\016\n\nTYPE_BYTES\020\014\022\017\n\013TY" + "PE_UINT32\020\r\022\r\n\tTYPE_ENUM\020\016\022\021\n\rTYPE_SFIXE" + "D32\020\017\022\021\n\rTYPE_SFIXED64\020\020\022\017\n\013TYPE_SINT32\020" + "\021\022\017\n\013TYPE_SINT64\020\022\"t\n\013Cardinality\022\027\n\023CAR" + "DINALITY_UNKNOWN\020\000\022\030\n\024CARDINALITY_OPTION" + "AL\020\001\022\030\n\024CARDINALITY_REQUIRED\020\002\022\030\n\024CARDIN" + "ALITY_REPEATED\020\003\"\316\001\n\004Enum\022\014\n\004name\030\001 \001(\t\022" + "-\n\tenumvalue\030\002 \003(\0132\032.google.protobuf.Enu" + "mValue\022(\n\007options\030\003 \003(\0132\027.google.protobu" + "f.Option\0226\n\016source_context\030\004 \001(\0132\036.googl" + "e.protobuf.SourceContext\022\'\n\006syntax\030\005 \001(\016" + "2\027.google.protobuf.Syntax\"S\n\tEnumValue\022\014" + "\n\004name\030\001 \001(\t\022\016\n\006number\030\002 \001(\005\022(\n\007options\030" + "\003 \003(\0132\027.google.protobuf.Option\";\n\006Option" + "\022\014\n\004name\030\001 \001(\t\022#\n\005value\030\002 \001(\0132\024.google.p" + "rotobuf.Any*.\n\006Syntax\022\021\n\rSYNTAX_PROTO2\020\000" + "\022\021\n\rSYNTAX_PROTO3\020\001BL\n\023com.google.protob" + "ufB\tTypeProtoP\001\240\001\001\242\002\003GPB\252\002\036Google.Protob" + "uf.WellKnownTypesb\006proto3", 1545); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "google/protobuf/type.proto", &protobuf_RegisterTypes); Type::default_instance_ = new Type(); @@ -1026,6 +1028,7 @@ const int Field::kOneofIndexFieldNumber; const int Field::kPackedFieldNumber; const int Field::kOptionsFieldNumber; const int Field::kJsonNameFieldNumber; +const int Field::kDefaultValueFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 Field::Field() @@ -1058,6 +1061,7 @@ void Field::SharedCtor() { oneof_index_ = 0; packed_ = false; json_name_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + default_value_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } Field::~Field() { @@ -1069,6 +1073,7 @@ void Field::SharedDtor() { name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); type_url_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); json_name_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + default_value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); if (this != default_instance_) { } } @@ -1113,6 +1118,7 @@ void Field::Clear() { type_url_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); packed_ = false; json_name_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + default_value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); #undef ZR_HELPER_ #undef ZR_ @@ -1270,6 +1276,23 @@ bool Field::MergePartialFromCodedStream( } else { goto handle_unusual; } + if (input->ExpectTag(90)) goto parse_default_value; + break; + } + + // optional string default_value = 11; + case 11: { + if (tag == 90) { + parse_default_value: + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_default_value())); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->default_value().data(), this->default_value().length(), + ::google::protobuf::internal::WireFormatLite::PARSE, + "google.protobuf.Field.default_value")); + } else { + goto handle_unusual; + } if (input->ExpectAtEnd()) goto success; break; } @@ -1361,6 +1384,16 @@ void Field::SerializeWithCachedSizes( 10, this->json_name(), output); } + // optional string default_value = 11; + if (this->default_value().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->default_value().data(), this->default_value().length(), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "google.protobuf.Field.default_value"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 11, this->default_value(), output); + } + // @@protoc_insertion_point(serialize_end:google.protobuf.Field) } @@ -1434,6 +1467,17 @@ void Field::SerializeWithCachedSizes( 10, this->json_name(), target); } + // optional string default_value = 11; + if (this->default_value().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->default_value().data(), this->default_value().length(), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "google.protobuf.Field.default_value"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 11, this->default_value(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:google.protobuf.Field) return target; } @@ -1493,6 +1537,13 @@ int Field::ByteSize() const { this->json_name()); } + // optional string default_value = 11; + if (this->default_value().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->default_value()); + } + // repeated .google.protobuf.Option options = 9; total_size += 1 * this->options_size(); for (int i = 0; i < this->options_size(); i++) { @@ -1549,6 +1600,10 @@ void Field::MergeFrom(const Field& from) { json_name_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.json_name_); } + if (from.default_value().size() > 0) { + + default_value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.default_value_); + } } void Field::CopyFrom(const ::google::protobuf::Message& from) { @@ -1582,6 +1637,7 @@ void Field::InternalSwap(Field* other) { std::swap(packed_, other->packed_); options_.UnsafeArenaSwap(&other->options_); json_name_.Swap(&other->json_name_); + default_value_.Swap(&other->default_value_); _internal_metadata_.Swap(&other->_internal_metadata_); std::swap(_cached_size_, other->_cached_size_); } @@ -1826,6 +1882,49 @@ void Field::clear_json_name() { // @@protoc_insertion_point(field_set_allocated:google.protobuf.Field.json_name) } +// optional string default_value = 11; +void Field::clear_default_value() { + default_value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + const ::std::string& Field::default_value() const { + // @@protoc_insertion_point(field_get:google.protobuf.Field.default_value) + return default_value_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + void Field::set_default_value(const ::std::string& value) { + + default_value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:google.protobuf.Field.default_value) +} + void Field::set_default_value(const char* value) { + + default_value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:google.protobuf.Field.default_value) +} + void Field::set_default_value(const char* value, size_t size) { + + default_value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:google.protobuf.Field.default_value) +} + ::std::string* Field::mutable_default_value() { + + // @@protoc_insertion_point(field_mutable:google.protobuf.Field.default_value) + return default_value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + ::std::string* Field::release_default_value() { + + return default_value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + void Field::set_allocated_default_value(::std::string* default_value) { + if (default_value != NULL) { + + } else { + + } + default_value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), default_value); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.Field.default_value) +} + #endif // PROTOBUF_INLINE_NOT_IN_HEADERS // =================================================================== diff --git a/src/google/protobuf/type.pb.h b/src/google/protobuf/type.pb.h index 2533eb4e..76fe8a65 100644 --- a/src/google/protobuf/type.pb.h +++ b/src/google/protobuf/type.pb.h @@ -471,6 +471,17 @@ class LIBPROTOBUF_EXPORT Field : public ::google::protobuf::Message { ::std::string* release_json_name(); void set_allocated_json_name(::std::string* json_name); + // optional string default_value = 11; + void clear_default_value(); + static const int kDefaultValueFieldNumber = 11; + const ::std::string& default_value() const; + void set_default_value(const ::std::string& value); + void set_default_value(const char* value); + void set_default_value(const char* value, size_t size); + ::std::string* mutable_default_value(); + ::std::string* release_default_value(); + void set_allocated_default_value(::std::string* default_value); + // @@protoc_insertion_point(class_scope:google.protobuf.Field) private: @@ -484,6 +495,7 @@ class LIBPROTOBUF_EXPORT Field : public ::google::protobuf::Message { ::google::protobuf::internal::ArenaStringPtr type_url_; ::google::protobuf::RepeatedPtrField< ::google::protobuf::Option > options_; ::google::protobuf::internal::ArenaStringPtr json_name_; + ::google::protobuf::internal::ArenaStringPtr default_value_; bool packed_; mutable int _cached_size_; friend void LIBPROTOBUF_EXPORT protobuf_AddDesc_google_2fprotobuf_2ftype_2eproto(); @@ -1264,6 +1276,49 @@ inline void Field::set_allocated_json_name(::std::string* json_name) { // @@protoc_insertion_point(field_set_allocated:google.protobuf.Field.json_name) } +// optional string default_value = 11; +inline void Field::clear_default_value() { + default_value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& Field::default_value() const { + // @@protoc_insertion_point(field_get:google.protobuf.Field.default_value) + return default_value_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void Field::set_default_value(const ::std::string& value) { + + default_value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:google.protobuf.Field.default_value) +} +inline void Field::set_default_value(const char* value) { + + default_value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:google.protobuf.Field.default_value) +} +inline void Field::set_default_value(const char* value, size_t size) { + + default_value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:google.protobuf.Field.default_value) +} +inline ::std::string* Field::mutable_default_value() { + + // @@protoc_insertion_point(field_mutable:google.protobuf.Field.default_value) + return default_value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* Field::release_default_value() { + + return default_value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void Field::set_allocated_default_value(::std::string* default_value) { + if (default_value != NULL) { + + } else { + + } + default_value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), default_value); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.Field.default_value) +} + // ------------------------------------------------------------------- // Enum diff --git a/src/google/protobuf/type.proto b/src/google/protobuf/type.proto index 4df95762..1c9cf53d 100644 --- a/src/google/protobuf/type.proto +++ b/src/google/protobuf/type.proto @@ -27,6 +27,7 @@ // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + syntax = "proto3"; package google.protobuf; @@ -34,22 +35,22 @@ package google.protobuf; import "google/protobuf/any.proto"; import "google/protobuf/source_context.proto"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option java_package = "com.google.protobuf"; option java_outer_classname = "TypeProto"; option java_multiple_files = true; option java_generate_equals_and_hash = true; -option csharp_namespace = "Google.Protobuf.WellKnownTypes"; option objc_class_prefix = "GPB"; -// A light-weight descriptor for a proto message type. +// A protocol buffer message type. message Type { // The fully qualified message name. string name = 1; // The list of fields. repeated Field fields = 2; - // The list of oneof definitions. - repeated string oneofs = 3; // The list of oneofs declared in this Type - // The proto options. + // The list of types appearing in `oneof` definitions in this type. + repeated string oneofs = 3; + // The protocol buffer options. repeated Option options = 4; // The source context. SourceContext source_context = 5; @@ -57,9 +58,9 @@ message Type { Syntax syntax = 6; } -// Field represents a single field of a message type. +// A single field of a message type. message Field { - // Kind represents a basic field type. + // Basic field types. enum Kind { // Field type unknown. TYPE_UNKNOWN = 0; @@ -81,7 +82,7 @@ message Field { TYPE_BOOL = 8; // Field type string. TYPE_STRING = 9; - // Field type group (deprecated proto2 type) + // Field type group. Proto2 syntax only, and deprecated. TYPE_GROUP = 10; // Field type message. TYPE_MESSAGE = 11; @@ -101,38 +102,40 @@ message Field { TYPE_SINT64 = 18; }; - // Cardinality represents whether a field is optional, required, or - // repeated. + // Whether a field is optional, required, or repeated. enum Cardinality { - // The field cardinality is unknown. Typically an error condition. + // For fields with unknown cardinality. CARDINALITY_UNKNOWN = 0; // For optional fields. CARDINALITY_OPTIONAL = 1; - // For required fields. Not used for proto3. + // For required fields. Proto2 syntax only. CARDINALITY_REQUIRED = 2; // For repeated fields. CARDINALITY_REPEATED = 3; }; - // The field kind. + // The field type. Kind kind = 1; - // The field cardinality, i.e. optional/required/repeated. + // The field cardinality. Cardinality cardinality = 2; - // The proto field number. + // The field number. int32 number = 3; // The field name. string name = 4; - // The type URL (without the scheme) when the type is MESSAGE or ENUM, - // such as `type.googleapis.com/google.protobuf.Empty`. + // The field type URL, without the scheme, for message or enumeration + // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. string type_url = 6; - // Index in Type.oneofs. Starts at 1. Zero means no oneof mapping. + // The index of the field type in `Type.oneofs`, for message or enumeration + // types. The first type has index 1; zero means the type is not in the list. int32 oneof_index = 7; // Whether to use alternative packed wire representation. bool packed = 8; - // The proto options. + // The protocol buffer options. repeated Option options = 9; - // The JSON name for this field. + // The field JSON name. string json_name = 10; + // The string value of the default value of this field. Proto2 syntax only. + string default_value = 11; } // Enum type definition. @@ -141,7 +144,7 @@ message Enum { string name = 1; // Enum value definitions. repeated EnumValue enumvalue = 2; - // Proto options for the enum type. + // Protocol buffer options. repeated Option options = 3; // The source context. SourceContext source_context = 4; @@ -155,22 +158,23 @@ message EnumValue { string name = 1; // Enum value number. int32 number = 2; - // Proto options for the enum value. + // Protocol buffer options. repeated Option options = 3; } -// Proto option attached to messages/fields/enums etc. +// A protocol buffer option, which can be attached to a message, field, +// enumeration, etc. message Option { - // Proto option name. + // The option's name. For example, `"java_package"`. string name = 1; - // Proto option value. + // The option's value. For example, `"com.google.protobuf"`. Any value = 2; } -// Syntax specifies the syntax in which a service element was defined. +// The syntax in which a protocol buffer element is defined. enum Syntax { - // Syntax "proto2" + // Syntax `proto2`. SYNTAX_PROTO2 = 0; - // Syntax "proto3" + // Syntax `proto3`. SYNTAX_PROTO3 = 1; } diff --git a/src/google/protobuf/unittest_custom_options.proto b/src/google/protobuf/unittest_custom_options.proto index d4d6e869..4cc0e362 100644 --- a/src/google/protobuf/unittest_custom_options.proto +++ b/src/google/protobuf/unittest_custom_options.proto @@ -392,3 +392,30 @@ message NestedOptionType { optional int32 nested_extension = 7912573 [(field_opt2) = 1005]; } } + +// Custom message option that has a required enum field. +// WARNING: this is strongly discouraged! +message OldOptionType { + enum TestEnum { + OLD_VALUE = 0; + } + required TestEnum value = 1; +} + +// Updated version of the custom option above. +message NewOptionType { + enum TestEnum { + OLD_VALUE = 0; + NEW_VALUE = 1; + } + required TestEnum value = 1; +} + +extend google.protobuf.MessageOptions { + optional OldOptionType required_enum_opt = 106161807; +} + +// Test message using the "required_enum_opt" option defined above. +message TestMessageWithRequiredEnumOption { + option (required_enum_opt) = { value: OLD_VALUE }; +} diff --git a/src/google/protobuf/util/field_comparator.h b/src/google/protobuf/util/field_comparator.h index ee676265..8b83c69f 100644 --- a/src/google/protobuf/util/field_comparator.h +++ b/src/google/protobuf/util/field_comparator.h @@ -93,7 +93,7 @@ class LIBPROTOBUF_EXPORT FieldComparator { // Basic implementation of FieldComparator. Supports four modes of floating // point value comparison: exact, approximate using MathUtil::AlmostEqual -// method, and arbitrarilly precise using MathUtil::WithinFracionOrMargin. +// method, and arbitrarilly precise using MathUtil::WithinFractionOrMargin. class LIBPROTOBUF_EXPORT DefaultFieldComparator : public FieldComparator { public: enum FloatComparison { diff --git a/src/google/protobuf/util/field_comparator_test.cc b/src/google/protobuf/util/field_comparator_test.cc index d3d34602..23f7d51d 100644 --- a/src/google/protobuf/util/field_comparator_test.cc +++ b/src/google/protobuf/util/field_comparator_test.cc @@ -35,7 +35,12 @@ #include #include #include - +// This gtest header is put after mathutil.h intentionally. We have to do +// this because mathutil.h includes mathlimits.h which requires cmath not +// being included to compile on some versions of gcc: +// https://github.com/google/protobuf/blob/818c5eee08840355d70d2f3bdf1a2f17986a5e70/src/google/protobuf/stubs/mathlimits.h#L48 +// and the opensource version gtest.h header includes cmath transitively +// somehow. #include namespace google { diff --git a/src/google/protobuf/util/internal/datapiece.cc b/src/google/protobuf/util/internal/datapiece.cc index ea360798..b557429f 100644 --- a/src/google/protobuf/util/internal/datapiece.cc +++ b/src/google/protobuf/util/internal/datapiece.cc @@ -35,8 +35,8 @@ #include #include #include -#include #include +#include namespace google { namespace protobuf { @@ -57,13 +57,8 @@ inline Status InvalidArgument(StringPiece value_str) { return Status(util::error::INVALID_ARGUMENT, value_str); } -// For general conversion between -// int32, int64, uint32, uint64, double and float -// except conversion between double and float. template -StatusOr NumberConvertAndCheck(From before) { - if (::google::protobuf::internal::is_same::value) return before; - To after = static_cast(before); +StatusOr ValidateNumberConversion(To after, From before) { if (after == before && MathUtil::Sign(before) == MathUtil::Sign(after)) { return after; @@ -76,6 +71,27 @@ StatusOr NumberConvertAndCheck(From before) { } } +// For general conversion between +// int32, int64, uint32, uint64, double and float +// except conversion between double and float. +template +StatusOr NumberConvertAndCheck(From before) { + if (::google::protobuf::internal::is_same::value) return before; + + To after = static_cast(before); + return ValidateNumberConversion(after, before); +} + +// For conversion to integer types (int32, int64, uint32, uint64) from floating +// point types (double, float) only. +template +StatusOr FloatingPointToIntConvertAndCheck(From before) { + if (::google::protobuf::internal::is_same::value) return before; + + To after = static_cast(before); + return ValidateNumberConversion(after, before); +} + // For conversion between double and float only. template StatusOr FloatingPointConvertAndCheck(From before) { @@ -96,30 +112,50 @@ StatusOr FloatingPointConvertAndCheck(From before) { } // namespace StatusOr DataPiece::ToInt32() const { - if (type_ == TYPE_STRING) { - return StringToNumber(safe_strto32); - } + if (type_ == TYPE_STRING) return StringToNumber(safe_strto32); + + if (type_ == TYPE_DOUBLE) + return FloatingPointToIntConvertAndCheck(double_); + + if (type_ == TYPE_FLOAT) + return FloatingPointToIntConvertAndCheck(float_); + return GenericConvert(); } StatusOr DataPiece::ToUint32() const { - if (type_ == TYPE_STRING) { - return StringToNumber(safe_strtou32); - } + if (type_ == TYPE_STRING) return StringToNumber(safe_strtou32); + + if (type_ == TYPE_DOUBLE) + return FloatingPointToIntConvertAndCheck(double_); + + if (type_ == TYPE_FLOAT) + return FloatingPointToIntConvertAndCheck(float_); + return GenericConvert(); } StatusOr DataPiece::ToInt64() const { - if (type_ == TYPE_STRING) { - return StringToNumber(safe_strto64); - } + if (type_ == TYPE_STRING) return StringToNumber(safe_strto64); + + if (type_ == TYPE_DOUBLE) + return FloatingPointToIntConvertAndCheck(double_); + + if (type_ == TYPE_FLOAT) + return FloatingPointToIntConvertAndCheck(float_); + return GenericConvert(); } StatusOr DataPiece::ToUint64() const { - if (type_ == TYPE_STRING) { - return StringToNumber(safe_strtou64); - } + if (type_ == TYPE_STRING) return StringToNumber(safe_strtou64); + + if (type_ == TYPE_DOUBLE) + return FloatingPointToIntConvertAndCheck(double_); + + if (type_ == TYPE_FLOAT) + return FloatingPointToIntConvertAndCheck(float_); + return GenericConvert(); } diff --git a/src/google/protobuf/util/internal/datapiece.h b/src/google/protobuf/util/internal/datapiece.h index 2ab3fa88..f22bfe70 100644 --- a/src/google/protobuf/util/internal/datapiece.h +++ b/src/google/protobuf/util/internal/datapiece.h @@ -98,7 +98,8 @@ class LIBPROTOBUF_EXPORT DataPiece { static DataPiece NullData() { return DataPiece(TYPE_NULL, 0); } - virtual ~DataPiece() {} + virtual ~DataPiece() { + } // Accessors Type type() const { return type_; } diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.cc b/src/google/protobuf/util/internal/default_value_objectwriter.cc index 97b248ff..a63e560d 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter.cc @@ -33,6 +33,7 @@ #include #include +#include #include namespace google { @@ -42,13 +43,25 @@ using util::Status; using util::StatusOr; namespace converter { +namespace { +// Helper function to convert string value to given data type by calling the +// passed converter function on the DataPiece created from "value" argument. +// If value is empty or if conversion fails, the default_value is returned. +template +T ConvertTo(StringPiece value, StatusOr (DataPiece::*converter_fn)() const, + T default_value) { + if (value.empty()) return default_value; + StatusOr result = (DataPiece(value).*converter_fn)(); + return result.ok() ? result.ValueOrDie() : default_value; +} +} // namespace + DefaultValueObjectWriter::DefaultValueObjectWriter( TypeResolver* type_resolver, const google::protobuf::Type& type, ObjectWriter* ow) : typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), own_typeinfo_(true), type_(type), - disable_normalize_(false), current_(NULL), root_(NULL), ow_(ow) {} @@ -165,12 +178,6 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderNull( return this; } -DefaultValueObjectWriter* -DefaultValueObjectWriter::DisableCaseNormalizationForNextKey() { - disable_normalize_ = true; - return this; -} - DefaultValueObjectWriter::Node::Node(const string& name, const google::protobuf::Type* type, NodeKind kind, const DataPiece& data, @@ -178,7 +185,6 @@ DefaultValueObjectWriter::Node::Node(const string& name, : name_(name), type_(type), kind_(kind), - disable_normalize_(false), is_any_(false), data_(data), is_placeholder_(is_placeholder) {} @@ -198,10 +204,6 @@ DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( } void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) { - if (disable_normalize_) { - ow->DisableCaseNormalizationForNextKey(); - } - if (kind_ == PRIMITIVE) { ObjectWriter::RenderDataPieceTo(data_, name_, ow); return; @@ -324,6 +326,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren( } } } + if (!is_map && field.cardinality() == google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) { @@ -336,11 +339,11 @@ void DefaultValueObjectWriter::Node::PopulateChildren( // If the child field is of primitive type, sets its data to the default // value of its type. - google::protobuf::scoped_ptr child( - new Node(field.json_name(), field_type, kind, - kind == PRIMITIVE ? CreateDefaultDataPieceForField(field) - : DataPiece::NullData(), - true)); + google::protobuf::scoped_ptr child(new Node( + field.json_name(), field_type, kind, + kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo) + : DataPiece::NullData(), + true)); new_children.push_back(child.release()); } // Adds all leftover nodes in children_ to the beginning of new_child. @@ -363,41 +366,68 @@ void DefaultValueObjectWriter::MaybePopulateChildrenOfAny(Node* node) { } } +DataPiece DefaultValueObjectWriter::FindEnumDefault( + const google::protobuf::Field& field, const TypeInfo* typeinfo) { + if (!field.default_value().empty()) return DataPiece(field.default_value()); + + const google::protobuf::Enum* enum_type = + typeinfo->GetEnumByTypeUrl(field.type_url()); + if (!enum_type) { + GOOGLE_LOG(WARNING) << "Could not find enum with type '" << field.type_url() + << "'"; + return DataPiece::NullData(); + } + // We treat the first value as the default if none is specified. + return enum_type->enumvalue_size() > 0 + ? DataPiece(enum_type->enumvalue(0).name()) + : DataPiece::NullData(); +} + DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( - const google::protobuf::Field& field) { + const google::protobuf::Field& field, const TypeInfo* typeinfo) { switch (field.kind()) { case google::protobuf::Field_Kind_TYPE_DOUBLE: { - return DataPiece(static_cast(0)); + return DataPiece(ConvertTo( + field.default_value(), &DataPiece::ToDouble, static_cast(0))); } case google::protobuf::Field_Kind_TYPE_FLOAT: { - return DataPiece(static_cast(0)); + return DataPiece(ConvertTo( + field.default_value(), &DataPiece::ToFloat, static_cast(0))); } case google::protobuf::Field_Kind_TYPE_INT64: case google::protobuf::Field_Kind_TYPE_SINT64: case google::protobuf::Field_Kind_TYPE_SFIXED64: { - return DataPiece(static_cast(0)); + return DataPiece(ConvertTo( + field.default_value(), &DataPiece::ToInt64, static_cast(0))); } case google::protobuf::Field_Kind_TYPE_UINT64: case google::protobuf::Field_Kind_TYPE_FIXED64: { - return DataPiece(static_cast(0)); + return DataPiece(ConvertTo( + field.default_value(), &DataPiece::ToUint64, static_cast(0))); } case google::protobuf::Field_Kind_TYPE_INT32: case google::protobuf::Field_Kind_TYPE_SINT32: case google::protobuf::Field_Kind_TYPE_SFIXED32: { - return DataPiece(static_cast(0)); + return DataPiece(ConvertTo( + field.default_value(), &DataPiece::ToInt32, static_cast(0))); } case google::protobuf::Field_Kind_TYPE_BOOL: { - return DataPiece(false); + return DataPiece( + ConvertTo(field.default_value(), &DataPiece::ToBool, false)); } case google::protobuf::Field_Kind_TYPE_STRING: { - return DataPiece(string()); + return DataPiece(field.default_value()); } case google::protobuf::Field_Kind_TYPE_BYTES: { - return DataPiece("", false); + return DataPiece(field.default_value(), false); } case google::protobuf::Field_Kind_TYPE_UINT32: case google::protobuf::Field_Kind_TYPE_FIXED32: { - return DataPiece(static_cast(0)); + return DataPiece(ConvertTo( + field.default_value(), &DataPiece::ToUint32, static_cast(0))); + } + case google::protobuf::Field_Kind_TYPE_ENUM: { + return FindEnumDefault(field, typeinfo); } default: { return DataPiece::NullData(); } } @@ -408,7 +438,6 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( if (current_ == NULL) { root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(), false)); - root_->set_disable_normalize(GetAndResetDisableNormalize()); root_->PopulateChildren(typeinfo_); current_ = root_.get(); return this; @@ -428,7 +457,6 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( } child->set_is_placeholder(false); - child->set_disable_normalize(GetAndResetDisableNormalize()); if (child->kind() == OBJECT && child->number_of_children() == 0) { child->PopulateChildren(typeinfo_); } @@ -454,21 +482,18 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( if (current_ == NULL) { root_.reset( new Node(name.ToString(), &type_, LIST, DataPiece::NullData(), false)); - root_->set_disable_normalize(GetAndResetDisableNormalize()); current_ = root_.get(); return this; } MaybePopulateChildrenOfAny(current_); Node* child = current_->FindChild(name); if (child == NULL || child->kind() != LIST) { - GOOGLE_LOG(WARNING) << "Cannot find field '" << name << "'."; google::protobuf::scoped_ptr node( new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false)); child = node.get(); current_->AddChild(node.release()); } child->set_is_placeholder(false); - child->set_disable_normalize(GetAndResetDisableNormalize()); stack_.push(current_); current_ = child; @@ -526,7 +551,6 @@ void DefaultValueObjectWriter::RenderDataPiece(StringPiece name, } else { child->set_data(data); } - child->set_disable_normalize(GetAndResetDisableNormalize()); } } // namespace converter diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h index bcb526e8..695b9dd8 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.h +++ b/src/google/protobuf/util/internal/default_value_objectwriter.h @@ -98,8 +98,6 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { virtual DefaultValueObjectWriter* RenderNull(StringPiece name); - virtual DefaultValueObjectWriter* DisableCaseNormalizationForNextKey(); - private: enum NodeKind { PRIMITIVE = 0, @@ -149,10 +147,6 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { void set_data(const DataPiece& data) { data_ = data; } - void set_disable_normalize(bool disable_normalize) { - disable_normalize_ = disable_normalize; - } - bool is_any() { return is_any_; } void set_is_any(bool is_any) { is_any_ = is_any; } @@ -176,8 +170,6 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { const google::protobuf::Type* type_; // The kind of this node. NodeKind kind_; - // Whether to disable case normalization of the name. - bool disable_normalize_; // Whether this is a node for "Any". bool is_any_; // The data of this node when it is a leaf node. @@ -201,16 +193,17 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Creates a DataPiece containing the default value of the type of the field. static DataPiece CreateDefaultDataPieceForField( - const google::protobuf::Field& field); - - // Returns disable_normalize_ and reset it to false. - bool GetAndResetDisableNormalize() { - return disable_normalize_ ? (disable_normalize_ = false, true) : false; - } + const google::protobuf::Field& field, const TypeInfo* typeinfo); // Adds or replaces the data_ of a primitive child node. void RenderDataPiece(StringPiece name, const DataPiece& data); + // Returns the default enum value as a DataPiece, or the first enum value if + // there is no default. For proto3, where we cannot specify an explicit + // default, a zero value will always be returned. + static DataPiece FindEnumDefault(const google::protobuf::Field& field, + const TypeInfo* typeinfo); + // Type information for all the types used in the descriptor. Used to find // google::protobuf::Type of nested messages/enums. const TypeInfo* typeinfo_; @@ -221,8 +214,6 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Holds copies of strings passed to RenderString. vector string_values_; - // Whether to disable case normalization of the next node. - bool disable_normalize_; // The current Node. Owned by its parents. Node* current_; // The root Node. diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index 237d0722..b7a537ab 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -43,21 +43,19 @@ namespace testing { using google::protobuf::testing::DefaultValueTest; -// Tests to cover some basic DefaultValueObjectWriter use cases. More tests are -// in the marshalling_test.cc and translator_integration_test.cc. -class DefaultValueObjectWriterTest +// Base class for setting up required state for running default values tests on +// different descriptors. +class BaseDefaultValueObjectWriterTest : public ::testing::TestWithParam { protected: - DefaultValueObjectWriterTest() + explicit BaseDefaultValueObjectWriterTest(const Descriptor* descriptor) : helper_(GetParam()), mock_(), expects_(&mock_) { - helper_.ResetTypeInfo(DefaultValueTest::descriptor()); + helper_.ResetTypeInfo(descriptor); testing_.reset(helper_.NewDefaultValueWriter( - string(kTypeServiceBaseUrl) + "/" + - DefaultValueTest::descriptor()->full_name(), - &mock_)); + string(kTypeServiceBaseUrl) + "/" + descriptor->full_name(), &mock_)); } - virtual ~DefaultValueObjectWriterTest() {} + ~BaseDefaultValueObjectWriterTest() override {} TypeInfoTestHelper helper_; MockObjectWriter mock_; @@ -65,6 +63,15 @@ class DefaultValueObjectWriterTest google::protobuf::scoped_ptr testing_; }; +// Tests to cover some basic DefaultValueObjectWriter use cases. More tests are +// in the marshalling_test.cc and translator_integration_test.cc. +class DefaultValueObjectWriterTest : public BaseDefaultValueObjectWriterTest { + protected: + DefaultValueObjectWriterTest() + : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) {} + ~DefaultValueObjectWriterTest() override {} +}; + INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, DefaultValueObjectWriterTest, ::testing::Values( @@ -74,6 +81,8 @@ TEST_P(DefaultValueObjectWriterTest, Empty) { // Set expectation expects_.StartObject("") ->RenderDouble("doubleValue", 0.0) + ->StartList("repeatedDouble") + ->EndList() ->RenderFloat("floatValue", 0.0) ->RenderInt64("int64Value", 0) ->RenderUint64("uint64Value", 0) @@ -82,6 +91,7 @@ TEST_P(DefaultValueObjectWriterTest, Empty) { ->RenderBool("boolValue", false) ->RenderString("stringValue", "") ->RenderBytes("bytesValue", "") + ->RenderString("enumValue", "ENUM_FIRST") ->EndObject(); // Actual testing @@ -92,6 +102,8 @@ TEST_P(DefaultValueObjectWriterTest, NonDefaultDouble) { // Set expectation expects_.StartObject("") ->RenderDouble("doubleValue", 1.0) + ->StartList("repeatedDouble") + ->EndList() ->RenderFloat("floatValue", 0.0) ->RenderInt64("int64Value", 0) ->RenderUint64("uint64Value", 0) @@ -99,6 +111,7 @@ TEST_P(DefaultValueObjectWriterTest, NonDefaultDouble) { ->RenderUint32("uint32Value", 0) ->RenderBool("boolValue", false) ->RenderString("stringValue", "") + ->RenderString("enumValue", "ENUM_FIRST") ->EndObject(); // Actual testing @@ -109,6 +122,8 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) { // Set expectation expects_.StartObject("") ->RenderDouble("doubleValue", 1.0) + ->StartList("repeatedDouble") + ->EndList() ->RenderFloat("floatValue", 0.0) ->RenderInt64("int64Value", 0) ->RenderUint64("uint64Value", 0) @@ -120,6 +135,7 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) { ->StartObject("unknownObject") ->RenderString("unknown", "def") ->EndObject() + ->RenderString("enumValue", "ENUM_FIRST") ->EndObject(); // Actual testing @@ -132,6 +148,7 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) { ->EndObject(); } + } // namespace testing } // namespace converter } // namespace util diff --git a/src/google/protobuf/util/internal/error_listener.h b/src/google/protobuf/util/internal/error_listener.h index 2699684d..3f063936 100644 --- a/src/google/protobuf/util/internal/error_listener.h +++ b/src/google/protobuf/util/internal/error_listener.h @@ -31,11 +31,13 @@ #ifndef GOOGLE_PROTOBUF_UTIL_CONVERTER_ERROR_LISTENER_H__ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_ERROR_LISTENER_H__ +#include #include #ifndef _SHARED_PTR_H #include #endif #include +#include #include #include diff --git a/src/google/protobuf/util/internal/json_objectwriter_test.cc b/src/google/protobuf/util/internal/json_objectwriter_test.cc index dcd60601..9d820162 100644 --- a/src/google/protobuf/util/internal/json_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/json_objectwriter_test.cc @@ -47,8 +47,7 @@ class JsonObjectWriterTest : public ::testing::Test { JsonObjectWriterTest() : str_stream_(new StringOutputStream(&output_)), out_stream_(new CodedOutputStream(str_stream_)), - ow_(NULL) { - } + ow_(NULL) {} virtual ~JsonObjectWriterTest() { delete ow_; @@ -64,36 +63,34 @@ class JsonObjectWriterTest : public ::testing::Test { TEST_F(JsonObjectWriterTest, EmptyRootObject) { ow_ = new JsonObjectWriter("", out_stream_); - ow_->StartObject("") - ->EndObject(); + ow_->StartObject("")->EndObject(); EXPECT_EQ("{}", output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, EmptyObject) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") - ->RenderString("test", "value") - ->StartObject("empty") - ->EndObject() - ->EndObject(); + ->RenderString("test", "value") + ->StartObject("empty") + ->EndObject() + ->EndObject(); EXPECT_EQ("{\"test\":\"value\",\"empty\":{}}", output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, EmptyRootList) { ow_ = new JsonObjectWriter("", out_stream_); - ow_->StartList("") - ->EndList(); + ow_->StartList("")->EndList(); EXPECT_EQ("[]", output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, EmptyList) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") - ->RenderString("test", "value") - ->StartList("empty") - ->EndList() - ->EndObject(); + ->RenderString("test", "value") + ->StartList("empty") + ->EndList() + ->EndObject(); EXPECT_EQ("{\"test\":\"value\",\"empty\":[]}", output_.substr(0, out_stream_->ByteCount())); } @@ -101,10 +98,10 @@ TEST_F(JsonObjectWriterTest, EmptyList) { TEST_F(JsonObjectWriterTest, ObjectInObject) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") - ->StartObject("nested") - ->RenderString("field", "value") - ->EndObject() - ->EndObject(); + ->StartObject("nested") + ->RenderString("field", "value") + ->EndObject() + ->EndObject(); EXPECT_EQ("{\"nested\":{\"field\":\"value\"}}", output_.substr(0, out_stream_->ByteCount())); } @@ -112,10 +109,10 @@ TEST_F(JsonObjectWriterTest, ObjectInObject) { TEST_F(JsonObjectWriterTest, ListInObject) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") - ->StartList("nested") - ->RenderString("", "value") - ->EndList() - ->EndObject(); + ->StartList("nested") + ->RenderString("", "value") + ->EndList() + ->EndObject(); EXPECT_EQ("{\"nested\":[\"value\"]}", output_.substr(0, out_stream_->ByteCount())); } @@ -123,10 +120,10 @@ TEST_F(JsonObjectWriterTest, ListInObject) { TEST_F(JsonObjectWriterTest, ObjectInList) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartList("") - ->StartObject("") - ->RenderString("field", "value") - ->EndObject() - ->EndList(); + ->StartObject("") + ->RenderString("field", "value") + ->EndObject() + ->EndList(); EXPECT_EQ("[{\"field\":\"value\"}]", output_.substr(0, out_stream_->ByteCount())); } @@ -134,10 +131,10 @@ TEST_F(JsonObjectWriterTest, ObjectInList) { TEST_F(JsonObjectWriterTest, ListInList) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartList("") - ->StartList("") - ->RenderString("", "value") - ->EndList() - ->EndList(); + ->StartList("") + ->RenderString("", "value") + ->EndList() + ->EndList(); EXPECT_EQ("[[\"value\"]]", output_.substr(0, out_stream_->ByteCount())); } @@ -156,14 +153,18 @@ TEST_F(JsonObjectWriterTest, RenderPrimitives) { ->EndObject(); EXPECT_EQ( "{\"bool\":true," - "\"double\":" + ValueAsString(1.7976931348623157e+308) + "," - "\"float\":" + ValueAsString(3.4028235e+38) + "," - "\"int\":-2147483648," - "\"long\":\"-9223372036854775808\"," - "\"bytes\":\"YWJyYWNhZGFicmE=\"," - "\"string\":\"string\"," - "\"emptybytes\":\"\"," - "\"emptystring\":\"\"}", + "\"double\":" + + ValueAsString(std::numeric_limits::max()) + + "," + "\"float\":" + + ValueAsString(std::numeric_limits::max()) + + "," + "\"int\":-2147483648," + "\"long\":\"-9223372036854775808\"," + "\"bytes\":\"YWJyYWNhZGFicmE=\"," + "\"string\":\"string\"," + "\"emptybytes\":\"\"," + "\"emptystring\":\"\"}", output_.substr(0, out_stream_->ByteCount())); } @@ -181,81 +182,83 @@ TEST_F(JsonObjectWriterTest, BytesEncodesAsNonWebSafeBase64) { TEST_F(JsonObjectWriterTest, PrettyPrintList) { ow_ = new JsonObjectWriter(" ", out_stream_); ow_->StartObject("") - ->StartList("items") - ->RenderString("", "item1") - ->RenderString("", "item2") - ->RenderString("", "item3") - ->EndList() - ->StartList("empty") - ->EndList() - ->EndObject(); - EXPECT_EQ("{\n" - " \"items\": [\n" - " \"item1\",\n" - " \"item2\",\n" - " \"item3\"\n" - " ],\n" - " \"empty\": []\n" - "}\n", - output_.substr(0, out_stream_->ByteCount())); + ->StartList("items") + ->RenderString("", "item1") + ->RenderString("", "item2") + ->RenderString("", "item3") + ->EndList() + ->StartList("empty") + ->EndList() + ->EndObject(); + EXPECT_EQ( + "{\n" + " \"items\": [\n" + " \"item1\",\n" + " \"item2\",\n" + " \"item3\"\n" + " ],\n" + " \"empty\": []\n" + "}\n", + output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, PrettyPrintObject) { ow_ = new JsonObjectWriter(" ", out_stream_); ow_->StartObject("") - ->StartObject("items") - ->RenderString("key1", "item1") - ->RenderString("key2", "item2") - ->RenderString("key3", "item3") - ->EndObject() - ->StartObject("empty") - ->EndObject() - ->EndObject(); - EXPECT_EQ("{\n" - " \"items\": {\n" - " \"key1\": \"item1\",\n" - " \"key2\": \"item2\",\n" - " \"key3\": \"item3\"\n" - " },\n" - " \"empty\": {}\n" - "}\n", - output_.substr(0, out_stream_->ByteCount())); + ->StartObject("items") + ->RenderString("key1", "item1") + ->RenderString("key2", "item2") + ->RenderString("key3", "item3") + ->EndObject() + ->StartObject("empty") + ->EndObject() + ->EndObject(); + EXPECT_EQ( + "{\n" + " \"items\": {\n" + " \"key1\": \"item1\",\n" + " \"key2\": \"item2\",\n" + " \"key3\": \"item3\"\n" + " },\n" + " \"empty\": {}\n" + "}\n", + output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, PrettyPrintEmptyObjectInEmptyList) { ow_ = new JsonObjectWriter(" ", out_stream_); ow_->StartObject("") - ->StartList("list") - ->StartObject("") - ->EndObject() - ->EndList() - ->EndObject(); - EXPECT_EQ("{\n" - " \"list\": [\n" - " {}\n" - " ]\n" - "}\n", - output_.substr(0, out_stream_->ByteCount())); + ->StartList("list") + ->StartObject("") + ->EndObject() + ->EndList() + ->EndObject(); + EXPECT_EQ( + "{\n" + " \"list\": [\n" + " {}\n" + " ]\n" + "}\n", + output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, PrettyPrintDoubleIndent) { ow_ = new JsonObjectWriter(" ", out_stream_); ow_->StartObject("") - ->RenderBool("bool", true) - ->RenderInt32("int", 42) - ->EndObject(); - EXPECT_EQ("{\n" - " \"bool\": true,\n" - " \"int\": 42\n" - "}\n", - output_.substr(0, out_stream_->ByteCount())); + ->RenderBool("bool", true) + ->RenderInt32("int", 42) + ->EndObject(); + EXPECT_EQ( + "{\n" + " \"bool\": true,\n" + " \"int\": 42\n" + "}\n", + output_.substr(0, out_stream_->ByteCount())); } TEST_F(JsonObjectWriterTest, StringsEscapedAndEnclosedInDoubleQuotes) { ow_ = new JsonObjectWriter("", out_stream_); - ow_->StartObject("") - ->RenderString("string", "'<>&\\\"\r\n") - ->EndObject(); + ow_->StartObject("")->RenderString("string", "'<>&\\\"\r\n")->EndObject(); EXPECT_EQ("{\"string\":\"'\\u003c\\u003e&\\\\\\\"\\r\\n\"}", output_.substr(0, out_stream_->ByteCount())); } @@ -263,13 +266,13 @@ TEST_F(JsonObjectWriterTest, StringsEscapedAndEnclosedInDoubleQuotes) { TEST_F(JsonObjectWriterTest, Stringification) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") - ->RenderDouble("double_nan", std::numeric_limits::quiet_NaN()) - ->RenderFloat("float_nan", std::numeric_limits::quiet_NaN()) - ->RenderDouble("double_pos", std::numeric_limits::infinity()) - ->RenderFloat("float_pos", std::numeric_limits::infinity()) - ->RenderDouble("double_neg", -std::numeric_limits::infinity()) - ->RenderFloat("float_neg", -std::numeric_limits::infinity()) - ->EndObject(); + ->RenderDouble("double_nan", std::numeric_limits::quiet_NaN()) + ->RenderFloat("float_nan", std::numeric_limits::quiet_NaN()) + ->RenderDouble("double_pos", std::numeric_limits::infinity()) + ->RenderFloat("float_pos", std::numeric_limits::infinity()) + ->RenderDouble("double_neg", -std::numeric_limits::infinity()) + ->RenderFloat("float_neg", -std::numeric_limits::infinity()) + ->EndObject(); EXPECT_EQ( "{\"double_nan\":\"NaN\"," "\"float_nan\":\"NaN\"," diff --git a/src/google/protobuf/util/internal/json_stream_parser.cc b/src/google/protobuf/util/internal/json_stream_parser.cc index a7ef7fe2..df916751 100644 --- a/src/google/protobuf/util/internal/json_stream_parser.cc +++ b/src/google/protobuf/util/internal/json_stream_parser.cc @@ -157,10 +157,10 @@ util::Status JsonStreamParser::FinishParse() { char* coerced = internal::UTF8CoerceToStructurallyValid(leftover_, utf8.get(), ' '); p_ = json_ = StringPiece(coerced, leftover_.size()); } else { + p_ = json_ = leftover_; if (!internal::IsStructurallyValidUTF8(leftover_)) { return ReportFailure("Encountered non UTF-8 code points."); } - p_ = json_ = leftover_; } // Parse the remainder in finishing mode, which reports errors for things like diff --git a/src/google/protobuf/util/internal/json_stream_parser_test.cc b/src/google/protobuf/util/internal/json_stream_parser_test.cc index c833ed1f..3414826e 100644 --- a/src/google/protobuf/util/internal/json_stream_parser_test.cc +++ b/src/google/protobuf/util/internal/json_stream_parser_test.cc @@ -348,6 +348,7 @@ TEST_F(JsonStreamParserTest, RejectNonUtf8WhenNotCoerced) { for (int i = 0; i <= json.length(); ++i) { DoErrorTest(json, i, "Encountered non UTF-8 code points."); } + DoErrorTest("\xFF{}", 0, "Encountered non UTF-8 code points."); } #ifndef _MSC_VER diff --git a/src/google/protobuf/util/internal/object_writer.h b/src/google/protobuf/util/internal/object_writer.h index 20bd3627..e695f45e 100644 --- a/src/google/protobuf/util/internal/object_writer.h +++ b/src/google/protobuf/util/internal/object_writer.h @@ -101,11 +101,6 @@ class LIBPROTOBUF_EXPORT ObjectWriter { // Renders a Null value. virtual ObjectWriter* RenderNull(StringPiece name) = 0; - // Disables case normalization. Any RenderTYPE call after calling this - // function will output the name field as-is. No normalization is attempted on - // it. This setting is reset immediately after the next RenderTYPE is called. - virtual ObjectWriter* DisableCaseNormalizationForNextKey() { return this; } - // Renders a DataPiece object to a ObjectWriter. static void RenderDataPieceTo(const DataPiece& data, StringPiece name, ObjectWriter* ow); diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc new file mode 100644 index 00000000..47e0009e --- /dev/null +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -0,0 +1,744 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace google { +namespace protobuf { +namespace util { +namespace converter { + +using google::protobuf::internal::WireFormatLite; +using google::protobuf::io::CodedOutputStream; +using util::error::INVALID_ARGUMENT; +using util::Status; +using util::StatusOr; + + +ProtoWriter::ProtoWriter(TypeResolver* type_resolver, + const google::protobuf::Type& type, + strings::ByteSink* output, ErrorListener* listener) + : master_type_(type), + typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), + own_typeinfo_(true), + done_(false), + element_(NULL), + size_insert_(), + output_(output), + buffer_(), + adapter_(&buffer_), + stream_(new CodedOutputStream(&adapter_)), + listener_(listener), + invalid_depth_(0), + tracker_(new ObjectLocationTracker()) {} + +ProtoWriter::ProtoWriter(const TypeInfo* typeinfo, + const google::protobuf::Type& type, + strings::ByteSink* output, ErrorListener* listener) + : master_type_(type), + typeinfo_(typeinfo), + own_typeinfo_(false), + done_(false), + element_(NULL), + size_insert_(), + output_(output), + buffer_(), + adapter_(&buffer_), + stream_(new CodedOutputStream(&adapter_)), + listener_(listener), + invalid_depth_(0), + tracker_(new ObjectLocationTracker()) {} + +ProtoWriter::~ProtoWriter() { + if (own_typeinfo_) { + delete typeinfo_; + } + if (element_ == NULL) return; + // Cleanup explicitly in order to avoid destructor stack overflow when input + // is deeply nested. + // Cast to BaseElement to avoid doing additional checks (like missing fields) + // during pop(). + google::protobuf::scoped_ptr element( + static_cast(element_.get())->pop()); + while (element != NULL) { + element.reset(element->pop()); + } +} + +namespace { + +// Writes an INT32 field, including tag to the stream. +inline Status WriteInt32(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr i32 = data.ToInt32(); + if (i32.ok()) { + WireFormatLite::WriteInt32(field_number, i32.ValueOrDie(), stream); + } + return i32.status(); +} + +// writes an SFIXED32 field, including tag, to the stream. +inline Status WriteSFixed32(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr i32 = data.ToInt32(); + if (i32.ok()) { + WireFormatLite::WriteSFixed32(field_number, i32.ValueOrDie(), stream); + } + return i32.status(); +} + +// Writes an SINT32 field, including tag, to the stream. +inline Status WriteSInt32(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr i32 = data.ToInt32(); + if (i32.ok()) { + WireFormatLite::WriteSInt32(field_number, i32.ValueOrDie(), stream); + } + return i32.status(); +} + +// Writes a FIXED32 field, including tag, to the stream. +inline Status WriteFixed32(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr u32 = data.ToUint32(); + if (u32.ok()) { + WireFormatLite::WriteFixed32(field_number, u32.ValueOrDie(), stream); + } + return u32.status(); +} + +// Writes a UINT32 field, including tag, to the stream. +inline Status WriteUInt32(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr u32 = data.ToUint32(); + if (u32.ok()) { + WireFormatLite::WriteUInt32(field_number, u32.ValueOrDie(), stream); + } + return u32.status(); +} + +// Writes an INT64 field, including tag, to the stream. +inline Status WriteInt64(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr i64 = data.ToInt64(); + if (i64.ok()) { + WireFormatLite::WriteInt64(field_number, i64.ValueOrDie(), stream); + } + return i64.status(); +} + +// Writes an SFIXED64 field, including tag, to the stream. +inline Status WriteSFixed64(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr i64 = data.ToInt64(); + if (i64.ok()) { + WireFormatLite::WriteSFixed64(field_number, i64.ValueOrDie(), stream); + } + return i64.status(); +} + +// Writes an SINT64 field, including tag, to the stream. +inline Status WriteSInt64(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr i64 = data.ToInt64(); + if (i64.ok()) { + WireFormatLite::WriteSInt64(field_number, i64.ValueOrDie(), stream); + } + return i64.status(); +} + +// Writes a FIXED64 field, including tag, to the stream. +inline Status WriteFixed64(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr u64 = data.ToUint64(); + if (u64.ok()) { + WireFormatLite::WriteFixed64(field_number, u64.ValueOrDie(), stream); + } + return u64.status(); +} + +// Writes a UINT64 field, including tag, to the stream. +inline Status WriteUInt64(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr u64 = data.ToUint64(); + if (u64.ok()) { + WireFormatLite::WriteUInt64(field_number, u64.ValueOrDie(), stream); + } + return u64.status(); +} + +// Writes a DOUBLE field, including tag, to the stream. +inline Status WriteDouble(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr d = data.ToDouble(); + if (d.ok()) { + WireFormatLite::WriteDouble(field_number, d.ValueOrDie(), stream); + } + return d.status(); +} + +// Writes a FLOAT field, including tag, to the stream. +inline Status WriteFloat(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr f = data.ToFloat(); + if (f.ok()) { + WireFormatLite::WriteFloat(field_number, f.ValueOrDie(), stream); + } + return f.status(); +} + +// Writes a BOOL field, including tag, to the stream. +inline Status WriteBool(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr b = data.ToBool(); + if (b.ok()) { + WireFormatLite::WriteBool(field_number, b.ValueOrDie(), stream); + } + return b.status(); +} + +// Writes a BYTES field, including tag, to the stream. +inline Status WriteBytes(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr c = data.ToBytes(); + if (c.ok()) { + WireFormatLite::WriteBytes(field_number, c.ValueOrDie(), stream); + } + return c.status(); +} + +// Writes a STRING field, including tag, to the stream. +inline Status WriteString(int field_number, const DataPiece& data, + CodedOutputStream* stream) { + StatusOr s = data.ToString(); + if (s.ok()) { + WireFormatLite::WriteString(field_number, s.ValueOrDie(), stream); + } + return s.status(); +} + +// Writes an ENUM field, including tag, to the stream. +inline Status WriteEnum(int field_number, const DataPiece& data, + const google::protobuf::Enum* enum_type, + CodedOutputStream* stream) { + StatusOr e = data.ToEnum(enum_type); + if (e.ok()) { + WireFormatLite::WriteEnum(field_number, e.ValueOrDie(), stream); + } + return e.status(); +} + +// Given a google::protobuf::Type, returns the set of all required fields. +std::set GetRequiredFields( + const google::protobuf::Type& type) { + std::set required; + for (int i = 0; i < type.fields_size(); i++) { + const google::protobuf::Field& field = type.fields(i); + if (field.cardinality() == + google::protobuf::Field_Cardinality_CARDINALITY_REQUIRED) { + required.insert(&field); + } + } + return required; +} + +} // namespace + +ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo, + const google::protobuf::Type& type, + ProtoWriter* enclosing) + : BaseElement(NULL), + ow_(enclosing), + parent_field_(NULL), + typeinfo_(typeinfo), + type_(type), + required_fields_(GetRequiredFields(type)), + size_index_(-1), + array_index_(-1) {} + +ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent, + const google::protobuf::Field* field, + const google::protobuf::Type& type, + bool is_list) + : BaseElement(parent), + ow_(this->parent()->ow_), + parent_field_(field), + typeinfo_(this->parent()->typeinfo_), + type_(type), + size_index_( + !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE + ? ow_->size_insert_.size() + : -1), + array_index_(is_list ? 0 : -1) { + if (!is_list) { + if (ow_->IsRepeated(*field)) { + // Update array_index_ if it is an explicit list. + if (this->parent()->array_index_ >= 0) this->parent()->array_index_++; + } else { + this->parent()->RegisterField(field); + } + + if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { + required_fields_ = GetRequiredFields(type_); + int start_pos = ow_->stream_->ByteCount(); + // length of serialized message is the final buffer position minus + // starting buffer position, plus length adjustments for size fields + // of any nested messages. We start with -start_pos here, so we only + // need to add the final buffer position to it at the end. + SizeInfo info = {start_pos, -start_pos}; + ow_->size_insert_.push_back(info); + } + } +} + +ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() { + // Calls the registered error listener for any required field(s) not yet + // seen. + for (set::iterator it = + required_fields_.begin(); + it != required_fields_.end(); ++it) { + ow_->MissingField((*it)->name()); + } + // Computes the total number of proto bytes used by a message, also adjusts + // the size of all parent messages by the length of this size field. + // If size_index_ < 0, this is not a message, so no size field is added. + if (size_index_ >= 0) { + // Add the final buffer position to compute the total length of this + // serialized message. The stored value (before this addition) already + // contains the total length of the size fields of all nested messages + // minus the initial buffer position. + ow_->size_insert_[size_index_].size += ow_->stream_->ByteCount(); + // Calculate the length required to serialize the size field of the + // message, and propagate this additional size information upward to + // all enclosing messages. + int size = ow_->size_insert_[size_index_].size; + int length = CodedOutputStream::VarintSize32(size); + for (ProtoElement* e = parent(); e != NULL; e = e->parent()) { + // Only nested messages have size field, lists do not have size field. + if (e->size_index_ >= 0) { + ow_->size_insert_[e->size_index_].size += length; + } + } + } + return BaseElement::pop(); +} + +void ProtoWriter::ProtoElement::RegisterField( + const google::protobuf::Field* field) { + if (!required_fields_.empty() && + field->cardinality() == + google::protobuf::Field_Cardinality_CARDINALITY_REQUIRED) { + required_fields_.erase(field); + } +} + +string ProtoWriter::ProtoElement::ToString() const { + if (parent() == NULL) return ""; + string loc = parent()->ToString(); + if (!ow_->IsRepeated(*parent_field_) || + parent()->parent_field_ != parent_field_) { + string name = parent_field_->name(); + int i = 0; + while (i < name.size() && (ascii_isalnum(name[i]) || name[i] == '_')) ++i; + if (i > 0 && i == name.size()) { // safe field name + if (loc.empty()) { + loc = name; + } else { + StrAppend(&loc, ".", name); + } + } else { + StrAppend(&loc, "[\"", CEscape(name), "\"]"); + } + } + if (ow_->IsRepeated(*parent_field_) && array_index_ > 0) { + StrAppend(&loc, "[", array_index_ - 1, "]"); + } + return loc.empty() ? "." : loc; +} + +bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) { + return ContainsKey(oneof_indices_, index); +} + +void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) { + InsertIfNotPresent(&oneof_indices_, index); +} + +void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) { + listener_->InvalidName(location(), ToSnakeCase(unknown_name), message); +} + +void ProtoWriter::InvalidValue(StringPiece type_name, StringPiece value) { + listener_->InvalidValue(location(), type_name, value); +} + +void ProtoWriter::MissingField(StringPiece missing_name) { + listener_->MissingField(location(), missing_name); +} + +ProtoWriter* ProtoWriter::StartObject(StringPiece name) { + // Starting the root message. Create the root ProtoElement and return. + if (element_ == NULL) { + if (!name.empty()) { + InvalidName(name, "Root element should not be named."); + } + element_.reset(new ProtoElement(typeinfo_, master_type_, this)); + return this; + } + + const google::protobuf::Field* field = NULL; + field = BeginNamed(name, false); + if (field == NULL) return this; + + // Check to see if this field is a oneof and that no oneof in that group has + // already been set. + if (!ValidOneof(*field, name)) { + ++invalid_depth_; + return this; + } + + const google::protobuf::Type* type = LookupType(field); + if (type == NULL) { + ++invalid_depth_; + InvalidName(name, + StrCat("Missing descriptor for field: ", field->type_url())); + return this; + } + + WriteTag(*field); + element_.reset(new ProtoElement(element_.release(), field, *type, false)); + return this; +} + +ProtoWriter* ProtoWriter::EndObject() { + if (invalid_depth_ > 0) { + --invalid_depth_; + return this; + } + + if (element_ != NULL) { + element_.reset(element_->pop()); + } + + + // If ending the root element, + // then serialize the full message with calculated sizes. + if (element_ == NULL) { + WriteRootMessage(); + } + return this; +} + +ProtoWriter* ProtoWriter::StartList(StringPiece name) { + const google::protobuf::Field* field = BeginNamed(name, true); + if (field == NULL) return this; + + if (!ValidOneof(*field, name)) { + ++invalid_depth_; + return this; + } + + const google::protobuf::Type* type = LookupType(field); + if (type == NULL) { + ++invalid_depth_; + InvalidName(name, + StrCat("Missing descriptor for field: ", field->type_url())); + return this; + } + + element_.reset(new ProtoElement(element_.release(), field, *type, true)); + return this; +} + +ProtoWriter* ProtoWriter::EndList() { + if (invalid_depth_ > 0) { + --invalid_depth_; + } else if (element_ != NULL) { + element_.reset(element_->pop()); + } + return this; +} + +ProtoWriter* ProtoWriter::RenderDataPiece(StringPiece name, + const DataPiece& data) { + Status status; + if (invalid_depth_ > 0) return this; + + const google::protobuf::Field* field = Lookup(name); + if (field == NULL) return this; + + if (!ValidOneof(*field, name)) return this; + + const google::protobuf::Type* type = LookupType(field); + if (type == NULL) { + InvalidName(name, + StrCat("Missing descriptor for field: ", field->type_url())); + return this; + } + + // Pushing a ProtoElement and then pop it off at the end for 2 purposes: + // error location reporting and required field accounting. + element_.reset(new ProtoElement(element_.release(), field, *type, false)); + + if (field->kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN || + field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { + InvalidValue(field->type_url().empty() + ? google::protobuf::Field_Kind_Name(field->kind()) + : field->type_url(), + data.ValueAsStringOrDefault("")); + element_.reset(element()->pop()); + return this; + } + + switch (field->kind()) { + case google::protobuf::Field_Kind_TYPE_INT32: { + status = WriteInt32(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_SFIXED32: { + status = WriteSFixed32(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_SINT32: { + status = WriteSInt32(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_FIXED32: { + status = WriteFixed32(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_UINT32: { + status = WriteUInt32(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_INT64: { + status = WriteInt64(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_SFIXED64: { + status = WriteSFixed64(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_SINT64: { + status = WriteSInt64(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_FIXED64: { + status = WriteFixed64(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_UINT64: { + status = WriteUInt64(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_DOUBLE: { + status = WriteDouble(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_FLOAT: { + status = WriteFloat(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_BOOL: { + status = WriteBool(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_BYTES: { + status = WriteBytes(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_STRING: { + status = WriteString(field->number(), data, stream_.get()); + break; + } + case google::protobuf::Field_Kind_TYPE_ENUM: { + status = WriteEnum(field->number(), data, + typeinfo_->GetEnumByTypeUrl(field->type_url()), + stream_.get()); + break; + } + default: // TYPE_GROUP or TYPE_MESSAGE + status = Status(INVALID_ARGUMENT, data.ToString().ValueOrDie()); + } + + if (!status.ok()) { + InvalidValue(google::protobuf::Field_Kind_Name(field->kind()), + status.error_message()); + } + + element_.reset(element()->pop()); + return this; +} + +bool ProtoWriter::ValidOneof(const google::protobuf::Field& field, + StringPiece unnormalized_name) { + if (element_ == NULL) return true; + + if (field.oneof_index() > 0) { + if (element_->IsOneofIndexTaken(field.oneof_index())) { + InvalidValue( + "oneof", + StrCat("oneof field '", + element_->type().oneofs(field.oneof_index() - 1), + "' is already set. Cannot set '", unnormalized_name, "'")); + return false; + } + element_->TakeOneofIndex(field.oneof_index()); + } + return true; +} + +bool ProtoWriter::IsRepeated(const google::protobuf::Field& field) { + return field.cardinality() == + google::protobuf::Field_Cardinality_CARDINALITY_REPEATED; +} + +const google::protobuf::Field* ProtoWriter::BeginNamed(StringPiece name, + bool is_list) { + if (invalid_depth_ > 0) { + ++invalid_depth_; + return NULL; + } + const google::protobuf::Field* field = Lookup(name); + if (field == NULL) { + ++invalid_depth_; + // InvalidName() already called in Lookup(). + return NULL; + } + if (is_list && !IsRepeated(*field)) { + ++invalid_depth_; + InvalidName(name, "Proto field is not repeating, cannot start list."); + return NULL; + } + return field; +} + +const google::protobuf::Field* ProtoWriter::Lookup( + StringPiece unnormalized_name) { + ProtoElement* e = element(); + if (e == NULL) { + InvalidName(unnormalized_name, "Root element must be a message."); + return NULL; + } + if (unnormalized_name.empty()) { + // Objects in repeated field inherit the same field descriptor. + if (e->parent_field() == NULL) { + InvalidName(unnormalized_name, "Proto fields must have a name."); + } else if (!IsRepeated(*e->parent_field())) { + InvalidName(unnormalized_name, "Proto fields must have a name."); + return NULL; + } + return e->parent_field(); + } + const google::protobuf::Field* field = + typeinfo_->FindField(&e->type(), unnormalized_name); + if (field == NULL) InvalidName(unnormalized_name, "Cannot find field."); + return field; +} + +const google::protobuf::Type* ProtoWriter::LookupType( + const google::protobuf::Field* field) { + return ((field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE || + field->kind() == google::protobuf::Field_Kind_TYPE_GROUP) + ? typeinfo_->GetTypeByTypeUrl(field->type_url()) + : &element_->type()); +} + +void ProtoWriter::WriteRootMessage() { + GOOGLE_DCHECK(!done_); + int curr_pos = 0; + // Calls the destructor of CodedOutputStream to remove any uninitialized + // memory from the Cord before we read it. + stream_.reset(NULL); + const void* data; + int length; + google::protobuf::io::ArrayInputStream input_stream(buffer_.data(), buffer_.size()); + while (input_stream.Next(&data, &length)) { + if (length == 0) continue; + int num_bytes = length; + // Write up to where we need to insert the size field. + // The number of bytes we may write is the smaller of: + // - the current fragment size + // - the distance to the next position where a size field needs to be + // inserted. + if (!size_insert_.empty() && + size_insert_.front().pos - curr_pos < num_bytes) { + num_bytes = size_insert_.front().pos - curr_pos; + } + output_->Append(static_cast(data), num_bytes); + if (num_bytes < length) { + input_stream.BackUp(length - num_bytes); + } + curr_pos += num_bytes; + // Insert the size field. + // size_insert_.front(): the next pair to be written. + // size_insert_.front().pos: position of the size field. + // size_insert_.front().size: the size (integer) to be inserted. + if (!size_insert_.empty() && curr_pos == size_insert_.front().pos) { + // Varint32 occupies at most 10 bytes. + uint8 insert_buffer[10]; + uint8* insert_buffer_pos = CodedOutputStream::WriteVarint32ToArray( + size_insert_.front().size, insert_buffer); + output_->Append(reinterpret_cast(insert_buffer), + insert_buffer_pos - insert_buffer); + size_insert_.pop_front(); + } + } + output_->Flush(); + stream_.reset(new CodedOutputStream(&adapter_)); + done_ = true; +} + +void ProtoWriter::WriteTag(const google::protobuf::Field& field) { + WireFormatLite::WireType wire_type = WireFormatLite::WireTypeForFieldType( + static_cast(field.kind())); + stream_->WriteTag(WireFormatLite::MakeTag(field.number(), wire_type)); +} + + +} // namespace converter +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h new file mode 100644 index 00000000..e631e56f --- /dev/null +++ b/src/google/protobuf/util/internal/proto_writer.h @@ -0,0 +1,315 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__ +#define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace io { +class CodedOutputStream; +} // namespace io +} // namespace protobuf + + +namespace protobuf { +class Type; +class Field; +} // namespace protobuf + + +namespace protobuf { +namespace util { +namespace converter { + +class ObjectLocationTracker; + +// An ObjectWriter that can write protobuf bytes directly from writer events. +// This class does not support special types like Struct or Map. However, since +// this class supports raw protobuf, it can be used to provide support for +// special types by inheriting from it or by wrapping it. +// +// It also supports streaming. +class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { + public: +// Constructor. Does not take ownership of any parameter passed in. + ProtoWriter(TypeResolver* type_resolver, const google::protobuf::Type& type, + strings::ByteSink* output, ErrorListener* listener); + virtual ~ProtoWriter(); + + // ObjectWriter methods. + virtual ProtoWriter* StartObject(StringPiece name); + virtual ProtoWriter* EndObject(); + virtual ProtoWriter* StartList(StringPiece name); + virtual ProtoWriter* EndList(); + virtual ProtoWriter* RenderBool(StringPiece name, bool value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderInt32(StringPiece name, int32 value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderUint32(StringPiece name, uint32 value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderInt64(StringPiece name, int64 value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderUint64(StringPiece name, uint64 value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderDouble(StringPiece name, double value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderFloat(StringPiece name, float value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderString(StringPiece name, StringPiece value) { + return RenderDataPiece(name, DataPiece(value)); + } + virtual ProtoWriter* RenderBytes(StringPiece name, StringPiece value) { + return RenderDataPiece(name, DataPiece(value, false)); + } + virtual ProtoWriter* RenderNull(StringPiece name) { + return RenderDataPiece(name, DataPiece::NullData()); + } + + // Renders a DataPiece 'value' into a field whose wire type is determined + // from the given field 'name'. + virtual ProtoWriter* RenderDataPiece(StringPiece name, + const DataPiece& value); + + // Returns the location tracker to use for tracking locations for errors. + const LocationTrackerInterface& location() { + return element_ != NULL ? *element_ : *tracker_; + } + + // When true, we finished writing to output a complete message. + bool done() const { return done_; } + + // Returns the proto stream object. + google::protobuf::io::CodedOutputStream* stream() { return stream_.get(); } + + // Getters and mutators of invalid_depth_. + void IncrementInvalidDepth() { ++invalid_depth_; } + void DecrementInvalidDepth() { --invalid_depth_; } + int invalid_depth() { return invalid_depth_; } + + ErrorListener* listener() { return listener_; } + + const TypeInfo* typeinfo() { return typeinfo_; } + + protected: + class LIBPROTOBUF_EXPORT ProtoElement : public BaseElement, public LocationTrackerInterface { + public: + // Constructor for the root element. No parent nor field. + ProtoElement(const TypeInfo* typeinfo, const google::protobuf::Type& type, + ProtoWriter* enclosing); + + // Constructor for a field of an element. + ProtoElement(ProtoElement* parent, const google::protobuf::Field* field, + const google::protobuf::Type& type, bool is_list); + + virtual ~ProtoElement() {} + + // Called just before the destructor for clean up: + // - reports any missing required fields + // - computes the space needed by the size field, and augment the + // length of all parent messages by this additional space. + // - releases and returns the parent pointer. + ProtoElement* pop(); + + // Accessors + // parent_field() may be NULL if we are at root. + const google::protobuf::Field* parent_field() const { + return parent_field_; + } + const google::protobuf::Type& type() const { return type_; } + + // Registers field for accounting required fields. + void RegisterField(const google::protobuf::Field* field); + + // To report location on error messages. + virtual string ToString() const; + + virtual ProtoElement* parent() const { + return static_cast(BaseElement::parent()); + } + + // Returns true if the index is already taken by a preceeding oneof input. + bool IsOneofIndexTaken(int32 index); + + // Marks the oneof 'index' as taken. Future inputs to this oneof will + // generate an error. + void TakeOneofIndex(int32 index); + + private: + // Used for access to variables of the enclosing instance of + // ProtoWriter. + ProtoWriter* ow_; + + // Describes the element as a field in the parent message. + // parent_field_ is NULL if and only if this element is the root element. + const google::protobuf::Field* parent_field_; + + // TypeInfo to lookup types. + const TypeInfo* typeinfo_; + + // Additional variables if this element is a message: + // (Root element is always a message). + // type_ : the type of this element. + // required_fields_ : set of required fields. + // size_index_ : index into ProtoWriter::size_insert_ + // for later insertion of serialized message length. + const google::protobuf::Type& type_; + std::set required_fields_; + const int size_index_; + + // Tracks position in repeated fields, needed for LocationTrackerInterface. + int array_index_; + + // Set of oneof indices already seen for the type_. Used to validate + // incoming messages so no more than one oneof is set. + hash_set oneof_indices_; + + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); + }; + + // Container for inserting 'size' information at the 'pos' position. + struct SizeInfo { + const int pos; + int size; + }; + + ProtoWriter(const TypeInfo* typeinfo, const google::protobuf::Type& type, + strings::ByteSink* output, ErrorListener* listener); + + virtual ProtoElement* element() { return element_.get(); } + + // Helper methods for calling ErrorListener. See error_listener.h. + void InvalidName(StringPiece unknown_name, StringPiece message); + void InvalidValue(StringPiece type_name, StringPiece value); + void MissingField(StringPiece missing_name); + + // Common code for BeginObject() and BeginList() that does invalid_depth_ + // bookkeeping associated with name lookup. + const google::protobuf::Field* BeginNamed(StringPiece name, bool is_list); + + // Lookup the field in the current element. Looks in the base descriptor + // and in any extension. This will report an error if the field cannot be + // found or if multiple matching extensions are found. + const google::protobuf::Field* Lookup(StringPiece name); + + // Lookup the field type in the type descriptor. Returns NULL if the type + // is not known. + const google::protobuf::Type* LookupType( + const google::protobuf::Field* field); + + // Write serialized output to the final output ByteSink, inserting all + // the size information for nested messages that are missing from the + // intermediate Cord buffer. + void WriteRootMessage(); + + // Helper method to write proto tags based on the given field. + void WriteTag(const google::protobuf::Field& field); + + + // Returns true if the field for type_ can be set as a oneof. If field is not + // a oneof type, this function does nothing and returns true. + // If another field for this oneof is already set, this function returns + // false. It also calls the appropriate error callback. + // unnormalized_name is used for error string. + bool ValidOneof(const google::protobuf::Field& field, + StringPiece unnormalized_name); + + // Returns true if the field is repeated. + bool IsRepeated(const google::protobuf::Field& field); + + private: + // Variables for describing the structure of the input tree: + // master_type_: descriptor for the whole protobuf message. + // typeinfo_ : the TypeInfo object to lookup types. + const google::protobuf::Type& master_type_; + const TypeInfo* typeinfo_; + // Whether we own the typeinfo_ object. + bool own_typeinfo_; + + // Indicates whether we finished writing root message completely. + bool done_; + + // Variable for internal state processing: + // element_ : the current element. + // size_insert_: sizes of nested messages. + // pos - position to insert the size field. + // size - size value to be inserted. + google::protobuf::scoped_ptr element_; + std::deque size_insert_; + + // Variables for output generation: + // output_ : pointer to an external ByteSink for final user-visible output. + // buffer_ : buffer holding partial message before being ready for output_. + // adapter_ : internal adapter between CodedOutputStream and buffer_. + // stream_ : wrapper for writing tags and other encodings in wire format. + strings::ByteSink* output_; + string buffer_; + google::protobuf::io::StringOutputStream adapter_; + google::protobuf::scoped_ptr stream_; + + // Variables for error tracking and reporting: + // listener_ : a place to report any errors found. + // invalid_depth_: number of enclosing invalid nested messages. + // tracker_ : the root location tracker interface. + ErrorListener* listener_; + int invalid_depth_; + google::protobuf::scoped_ptr tracker_; + + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoWriter); +}; + +} // namespace converter +} // namespace util +} // namespace protobuf + +} // namespace google +#endif // GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__ diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc index 2bbb1597..034d616f 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource.cc @@ -140,10 +140,10 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, bool include_start_and_end, ObjectWriter* ow) const { - const TypeRenderer* type_renderer = FindTypeRenderer(type.name()); - if (type_renderer != NULL) { - return (*type_renderer)(this, type, name, ow); - } + const TypeRenderer* type_renderer = FindTypeRenderer(type.name()); + if (type_renderer != NULL) { + return (*type_renderer)(this, type, name, ow); + } const google::protobuf::Field* field = NULL; string field_name; @@ -171,7 +171,9 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, if (field->cardinality() == google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) { - if (IsMap(*field)) { + bool check_maps = true; + + if (check_maps && IsMap(*field)) { ow->StartObject(field_name); ASSIGN_OR_RETURN(tag, RenderMap(field, field_name, tag, ow)); ow->EndObject(); @@ -218,48 +220,32 @@ StatusOr ProtoStreamObjectSource::RenderMap( const google::protobuf::Type* field_type = typeinfo_->GetTypeByTypeUrl(field->type_url()); uint32 tag_to_return = 0; - if (IsPackable(*field) && - list_tag == - WireFormatLite::MakeTag(field->number(), - WireFormatLite::WIRETYPE_LENGTH_DELIMITED)) { - RETURN_IF_ERROR(RenderPackedMapEntry(field_type, ow)); - tag_to_return = stream_->ReadTag(); - } else { - do { - RETURN_IF_ERROR(RenderMapEntry(field_type, ow)); - } while ((tag_to_return = stream_->ReadTag()) == list_tag); - } - return tag_to_return; -} - -Status ProtoStreamObjectSource::RenderMapEntry( - const google::protobuf::Type* type, ObjectWriter* ow) const { - uint32 buffer32; - stream_->ReadVarint32(&buffer32); // message length - int old_limit = stream_->PushLimit(buffer32); - string map_key; - for (uint32 tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { - const google::protobuf::Field* field = FindAndVerifyField(*type, tag); - if (field == NULL) { - WireFormat::SkipField(stream_, tag, NULL); - continue; - } - // Map field numbers are key = 1 and value = 2 - if (field->number() == 1) { - map_key = ReadFieldValueAsString(*field); - } else if (field->number() == 2) { - if (map_key.empty()) { - return Status(util::error::INTERNAL, "Map key must be non-empty"); + do { + // Render map entry message type. + uint32 buffer32; + stream_->ReadVarint32(&buffer32); // message length + int old_limit = stream_->PushLimit(buffer32); + string map_key; + for (uint32 tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { + const google::protobuf::Field* field = + FindAndVerifyField(*field_type, tag); + if (field == NULL) { + WireFormat::SkipField(stream_, tag, NULL); + continue; + } + // Map field numbers are key = 1 and value = 2 + if (field->number() == 1) { + map_key = ReadFieldValueAsString(*field); + } else if (field->number() == 2) { + if (map_key.empty()) { + return Status(util::error::INTERNAL, "Map key must be non-empty"); + } + RETURN_IF_ERROR(RenderField(field, map_key, ow)); } - // Disable case normalization for map keys as they are just data. We - // retain them intact. - ow->DisableCaseNormalizationForNextKey(); - RETURN_IF_ERROR(RenderField(field, map_key, ow)); } - } - stream_->PopLimit(old_limit); - - return Status::OK; + stream_->PopLimit(old_limit); + } while ((tag_to_return = stream_->ReadTag()) == list_tag); + return tag_to_return; } Status ProtoStreamObjectSource::RenderPacked( @@ -274,18 +260,6 @@ Status ProtoStreamObjectSource::RenderPacked( return Status::OK; } -Status ProtoStreamObjectSource::RenderPackedMapEntry( - const google::protobuf::Type* type, ObjectWriter* ow) const { - uint32 length; - stream_->ReadVarint32(&length); - int old_limit = stream_->PushLimit(length); - while (stream_->BytesUntilLimit() > 0) { - RETURN_IF_ERROR(RenderMapEntry(type, ow)); - } - stream_->PopLimit(old_limit); - return Status::OK; -} - Status ProtoStreamObjectSource::RenderTimestamp( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { @@ -601,10 +575,10 @@ Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, // nested_type cannot be null at this time. const google::protobuf::Type* nested_type = resolved_type.ValueOrDie(); - // We know the type so we can render it. Recursively parse the nested stream - // using a nested ProtoStreamObjectSource using our nested type information. google::protobuf::io::ArrayInputStream zero_copy_stream(value.data(), value.size()); google::protobuf::io::CodedInputStream in_stream(&zero_copy_stream); + // We know the type so we can render it. Recursively parse the nested stream + // using a nested ProtoStreamObjectSource using our nested type information. ProtoStreamObjectSource nested_os(&in_stream, os->typeinfo_, *nested_type); // We manually call start and end object here so we can inject the @type. @@ -676,8 +650,7 @@ void ProtoStreamObjectSource::InitRendererMap() { &ProtoStreamObjectSource::RenderString; (*renderers_)["google.protobuf.BytesValue"] = &ProtoStreamObjectSource::RenderBytes; - (*renderers_)["google.protobuf.Any"] = - &ProtoStreamObjectSource::RenderAny; + (*renderers_)["google.protobuf.Any"] = &ProtoStreamObjectSource::RenderAny; (*renderers_)["google.protobuf.Struct"] = &ProtoStreamObjectSource::RenderStruct; (*renderers_)["google.protobuf.Value"] = @@ -704,87 +677,118 @@ ProtoStreamObjectSource::FindTypeRenderer(const string& type_url) { Status ProtoStreamObjectSource::RenderField( const google::protobuf::Field* field, StringPiece field_name, ObjectWriter* ow) const { + // Short-circuit message types as it tends to call WriteMessage recursively + // and ends up using a lot of stack space. Keep the stack usage of this + // message small in order to preserve stack space and not crash. + if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { + uint32 buffer32; + stream_->ReadVarint32(&buffer32); // message length + int old_limit = stream_->PushLimit(buffer32); + // Get the nested message type for this field. + const google::protobuf::Type* type = + typeinfo_->GetTypeByTypeUrl(field->type_url()); + if (type == NULL) { + return Status(util::error::INTERNAL, + StrCat("Invalid configuration. Could not find the type: ", + field->type_url())); + } + + // Short-circuit any special type rendering to save call-stack space. + const TypeRenderer* type_renderer = FindTypeRenderer(type->name()); + + bool use_type_renderer = type_renderer != NULL; + + if (use_type_renderer) { + RETURN_IF_ERROR((*type_renderer)(this, *type, field_name, ow)); + } else { + RETURN_IF_ERROR(WriteMessage(*type, field_name, 0, true, ow)); + } + if (!stream_->ConsumedEntireMessage()) { + return Status(util::error::INVALID_ARGUMENT, + "Nested protocol message not parsed in its entirety."); + } + stream_->PopLimit(old_limit); + } else { + // Render all other non-message types. + return RenderNonMessageField(field, field_name, ow); + } + return Status::OK; +} + +Status ProtoStreamObjectSource::RenderNonMessageField( + const google::protobuf::Field* field, StringPiece field_name, + ObjectWriter* ow) const { + // Temporary buffers of different types. + uint32 buffer32; + uint64 buffer64; + string strbuffer; switch (field->kind()) { case google::protobuf::Field_Kind_TYPE_BOOL: { - uint64 buffer64; stream_->ReadVarint64(&buffer64); ow->RenderBool(field_name, buffer64 != 0); break; } case google::protobuf::Field_Kind_TYPE_INT32: { - uint32 buffer32; stream_->ReadVarint32(&buffer32); ow->RenderInt32(field_name, bit_cast(buffer32)); break; } case google::protobuf::Field_Kind_TYPE_INT64: { - uint64 buffer64; stream_->ReadVarint64(&buffer64); ow->RenderInt64(field_name, bit_cast(buffer64)); break; } case google::protobuf::Field_Kind_TYPE_UINT32: { - uint32 buffer32; stream_->ReadVarint32(&buffer32); ow->RenderUint32(field_name, bit_cast(buffer32)); break; } case google::protobuf::Field_Kind_TYPE_UINT64: { - uint64 buffer64; stream_->ReadVarint64(&buffer64); ow->RenderUint64(field_name, bit_cast(buffer64)); break; } case google::protobuf::Field_Kind_TYPE_SINT32: { - uint32 buffer32; stream_->ReadVarint32(&buffer32); ow->RenderInt32(field_name, WireFormatLite::ZigZagDecode32(buffer32)); break; } case google::protobuf::Field_Kind_TYPE_SINT64: { - uint64 buffer64; stream_->ReadVarint64(&buffer64); ow->RenderInt64(field_name, WireFormatLite::ZigZagDecode64(buffer64)); break; } case google::protobuf::Field_Kind_TYPE_SFIXED32: { - uint32 buffer32; stream_->ReadLittleEndian32(&buffer32); ow->RenderInt32(field_name, bit_cast(buffer32)); break; } case google::protobuf::Field_Kind_TYPE_SFIXED64: { - uint64 buffer64; stream_->ReadLittleEndian64(&buffer64); ow->RenderInt64(field_name, bit_cast(buffer64)); break; } case google::protobuf::Field_Kind_TYPE_FIXED32: { - uint32 buffer32; stream_->ReadLittleEndian32(&buffer32); ow->RenderUint32(field_name, bit_cast(buffer32)); break; } case google::protobuf::Field_Kind_TYPE_FIXED64: { - uint64 buffer64; stream_->ReadLittleEndian64(&buffer64); ow->RenderUint64(field_name, bit_cast(buffer64)); break; } case google::protobuf::Field_Kind_TYPE_FLOAT: { - uint32 buffer32; stream_->ReadLittleEndian32(&buffer32); ow->RenderFloat(field_name, bit_cast(buffer32)); break; } case google::protobuf::Field_Kind_TYPE_DOUBLE: { - uint64 buffer64; stream_->ReadLittleEndian64(&buffer64); ow->RenderDouble(field_name, bit_cast(buffer64)); break; } case google::protobuf::Field_Kind_TYPE_ENUM: { - uint32 buffer32; stream_->ReadVarint32(&buffer32); // If the field represents an explicit NULL value, render null. @@ -811,40 +815,15 @@ Status ProtoStreamObjectSource::RenderField( break; } case google::protobuf::Field_Kind_TYPE_STRING: { - uint32 buffer32; - string str; stream_->ReadVarint32(&buffer32); // string size. - stream_->ReadString(&str, buffer32); - ow->RenderString(field_name, str); + stream_->ReadString(&strbuffer, buffer32); + ow->RenderString(field_name, strbuffer); break; } case google::protobuf::Field_Kind_TYPE_BYTES: { - uint32 buffer32; stream_->ReadVarint32(&buffer32); // bytes size. - string value; - stream_->ReadString(&value, buffer32); - ow->RenderBytes(field_name, value); - break; - } - case google::protobuf::Field_Kind_TYPE_MESSAGE: { - uint32 buffer32; - stream_->ReadVarint32(&buffer32); // message length - int old_limit = stream_->PushLimit(buffer32); - // Get the nested message type for this field. - const google::protobuf::Type* type = - typeinfo_->GetTypeByTypeUrl(field->type_url()); - if (type == NULL) { - return Status(util::error::INTERNAL, - StrCat("Invalid configuration. Could not find the type: ", - field->type_url())); - } - - RETURN_IF_ERROR(WriteMessage(*type, field_name, 0, true, ow)); - if (!stream_->ConsumedEntireMessage()) { - return Status(util::error::INVALID_ARGUMENT, - "Nested protocol message not parsed in its entirety."); - } - stream_->PopLimit(old_limit); + stream_->ReadString(&strbuffer, buffer32); + ow->RenderBytes(field_name, strbuffer); break; } default: diff --git a/src/google/protobuf/util/internal/protostream_objectsource.h b/src/google/protobuf/util/internal/protostream_objectsource.h index 3cd37aa1..78defa1d 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.h +++ b/src/google/protobuf/util/internal/protostream_objectsource.h @@ -122,20 +122,12 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { StringPiece name, uint32 list_tag, ObjectWriter* ow) const; - // Renders an entry in a map, advancing stream pointers appropriately. - util::Status RenderMapEntry(const google::protobuf::Type* type, - ObjectWriter* ow) const; - // Renders a packed repeating field. A packed field is stored as: // {tag length item1 item2 item3} instead of the less efficient // {tag item1 tag item2 tag item3}. util::Status RenderPacked(const google::protobuf::Field* field, ObjectWriter* ow) const; - // Equivalent of RenderPacked, but for map entries. - util::Status RenderPackedMapEntry(const google::protobuf::Type* type, - ObjectWriter* ow) const; - // Renders a google.protobuf.Timestamp value to ObjectWriter static util::Status RenderTimestamp(const ProtoStreamObjectSource* os, const google::protobuf::Type& type, @@ -210,6 +202,12 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { util::Status RenderField(const google::protobuf::Field* field, StringPiece field_name, ObjectWriter* ow) const; + // Same as above but renders all non-message field types. Callers don't call + // this function directly. They just use RenderField. + util::Status RenderNonMessageField(const google::protobuf::Field* field, + StringPiece field_name, + ObjectWriter* ow) const; + // Reads field value according to Field spec in 'field' and returns the read // value as string. This only works for primitive datatypes (no message @@ -238,6 +236,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { // google::protobuf::Type of the message source. const google::protobuf::Type& type_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoStreamObjectSource); }; diff --git a/src/google/protobuf/util/internal/protostream_objectsource_test.cc b/src/google/protobuf/util/internal/protostream_objectsource_test.cc index f6e5ee7a..561f6763 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource_test.cc @@ -327,9 +327,16 @@ TEST_P(ProtostreamObjectSourceTest, RepeatingPrimitives) { DoTest(primitive, Primitive::descriptor()); } +TEST_P(ProtostreamObjectSourceTest, CustomJsonName) { + Author author; + author.set_id(12345); + + ow_.StartObject("")->RenderUint64("@id", 12345)->EndObject(); + DoTest(author, Author::descriptor()); +} + TEST_P(ProtostreamObjectSourceTest, NestedMessage) { Author* author = new Author(); - author->set_id(101L); author->set_name("Tolstoy"); Book book; book.set_title("My Book"); @@ -338,7 +345,6 @@ TEST_P(ProtostreamObjectSourceTest, NestedMessage) { ow_.StartObject("") ->RenderString("title", "My Book") ->StartObject("author") - ->RenderUint64("id", bit_cast(101LL)) ->RenderString("name", "Tolstoy") ->EndObject() ->EndObject(); diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc index 0958997c..786bf0be 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc @@ -51,7 +51,6 @@ namespace util { namespace converter { using google::protobuf::internal::WireFormatLite; -using google::protobuf::io::CodedOutputStream; using util::error::INVALID_ARGUMENT; using util::Status; using util::StatusOr; @@ -60,230 +59,31 @@ using util::StatusOr; ProtoStreamObjectWriter::ProtoStreamObjectWriter( TypeResolver* type_resolver, const google::protobuf::Type& type, strings::ByteSink* output, ErrorListener* listener) - : master_type_(type), - typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), - own_typeinfo_(true), - done_(false), - element_(NULL), - size_insert_(), - output_(output), - buffer_(), - adapter_(&buffer_), - stream_(new CodedOutputStream(&adapter_)), - listener_(listener), - invalid_depth_(0), - tracker_(new ObjectLocationTracker()) {} + : ProtoWriter(type_resolver, type, output, listener), + master_type_(type), + current_(NULL) {} ProtoStreamObjectWriter::ProtoStreamObjectWriter( const TypeInfo* typeinfo, const google::protobuf::Type& type, strings::ByteSink* output, ErrorListener* listener) - : master_type_(type), - typeinfo_(typeinfo), - own_typeinfo_(false), - done_(false), - element_(NULL), - size_insert_(), - output_(output), - buffer_(), - adapter_(&buffer_), - stream_(new CodedOutputStream(&adapter_)), - listener_(listener), - invalid_depth_(0), - tracker_(new ObjectLocationTracker()) {} + : ProtoWriter(typeinfo, type, output, listener), + master_type_(type), + current_(NULL) {} ProtoStreamObjectWriter::~ProtoStreamObjectWriter() { - if (own_typeinfo_) { - delete typeinfo_; - } - if (element_ == NULL) return; + if (current_ == NULL) return; // Cleanup explicitly in order to avoid destructor stack overflow when input // is deeply nested. // Cast to BaseElement to avoid doing additional checks (like missing fields) // during pop(). google::protobuf::scoped_ptr element( - static_cast(element_.get())->pop()); + static_cast(current_.get())->pop()); while (element != NULL) { element.reset(element->pop()); } } namespace { - -// Writes an INT32 field, including tag to the stream. -inline Status WriteInt32(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr i32 = data.ToInt32(); - if (i32.ok()) { - WireFormatLite::WriteInt32(field_number, i32.ValueOrDie(), stream); - } - return i32.status(); -} - -// writes an SFIXED32 field, including tag, to the stream. -inline Status WriteSFixed32(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr i32 = data.ToInt32(); - if (i32.ok()) { - WireFormatLite::WriteSFixed32(field_number, i32.ValueOrDie(), stream); - } - return i32.status(); -} - -// Writes an SINT32 field, including tag, to the stream. -inline Status WriteSInt32(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr i32 = data.ToInt32(); - if (i32.ok()) { - WireFormatLite::WriteSInt32(field_number, i32.ValueOrDie(), stream); - } - return i32.status(); -} - -// Writes a FIXED32 field, including tag, to the stream. -inline Status WriteFixed32(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr u32 = data.ToUint32(); - if (u32.ok()) { - WireFormatLite::WriteFixed32(field_number, u32.ValueOrDie(), stream); - } - return u32.status(); -} - -// Writes a UINT32 field, including tag, to the stream. -inline Status WriteUInt32(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr u32 = data.ToUint32(); - if (u32.ok()) { - WireFormatLite::WriteUInt32(field_number, u32.ValueOrDie(), stream); - } - return u32.status(); -} - -// Writes an INT64 field, including tag, to the stream. -inline Status WriteInt64(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr i64 = data.ToInt64(); - if (i64.ok()) { - WireFormatLite::WriteInt64(field_number, i64.ValueOrDie(), stream); - } - return i64.status(); -} - -// Writes an SFIXED64 field, including tag, to the stream. -inline Status WriteSFixed64(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr i64 = data.ToInt64(); - if (i64.ok()) { - WireFormatLite::WriteSFixed64(field_number, i64.ValueOrDie(), stream); - } - return i64.status(); -} - -// Writes an SINT64 field, including tag, to the stream. -inline Status WriteSInt64(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr i64 = data.ToInt64(); - if (i64.ok()) { - WireFormatLite::WriteSInt64(field_number, i64.ValueOrDie(), stream); - } - return i64.status(); -} - -// Writes a FIXED64 field, including tag, to the stream. -inline Status WriteFixed64(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr u64 = data.ToUint64(); - if (u64.ok()) { - WireFormatLite::WriteFixed64(field_number, u64.ValueOrDie(), stream); - } - return u64.status(); -} - -// Writes a UINT64 field, including tag, to the stream. -inline Status WriteUInt64(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr u64 = data.ToUint64(); - if (u64.ok()) { - WireFormatLite::WriteUInt64(field_number, u64.ValueOrDie(), stream); - } - return u64.status(); -} - -// Writes a DOUBLE field, including tag, to the stream. -inline Status WriteDouble(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr d = data.ToDouble(); - if (d.ok()) { - WireFormatLite::WriteDouble(field_number, d.ValueOrDie(), stream); - } - return d.status(); -} - -// Writes a FLOAT field, including tag, to the stream. -inline Status WriteFloat(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr f = data.ToFloat(); - if (f.ok()) { - WireFormatLite::WriteFloat(field_number, f.ValueOrDie(), stream); - } - return f.status(); -} - -// Writes a BOOL field, including tag, to the stream. -inline Status WriteBool(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr b = data.ToBool(); - if (b.ok()) { - WireFormatLite::WriteBool(field_number, b.ValueOrDie(), stream); - } - return b.status(); -} - -// Writes a BYTES field, including tag, to the stream. -inline Status WriteBytes(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr c = data.ToBytes(); - if (c.ok()) { - WireFormatLite::WriteBytes(field_number, c.ValueOrDie(), stream); - } - return c.status(); -} - -// Writes a STRING field, including tag, to the stream. -inline Status WriteString(int field_number, const DataPiece& data, - CodedOutputStream* stream) { - StatusOr s = data.ToString(); - if (s.ok()) { - WireFormatLite::WriteString(field_number, s.ValueOrDie(), stream); - } - return s.status(); -} - -// Writes an ENUM field, including tag, to the stream. -inline Status WriteEnum(int field_number, const DataPiece& data, - const google::protobuf::Enum* enum_type, - CodedOutputStream* stream) { - StatusOr e = data.ToEnum(enum_type); - if (e.ok()) { - WireFormatLite::WriteEnum(field_number, e.ValueOrDie(), stream); - } - return e.status(); -} - -// Given a google::protobuf::Type, returns the set of all required fields. -std::set GetRequiredFields( - const google::protobuf::Type& type) { - std::set required; - for (int i = 0; i < type.fields_size(); i++) { - const google::protobuf::Field& field = type.fields(i); - if (field.cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REQUIRED) { - required.insert(&field); - } - } - return required; -} - // Utility method to split a string representation of Timestamp or Duration and // return the parts. void SplitSecondsAndNanos(StringPiece input, StringPiece* seconds, @@ -298,6 +98,78 @@ void SplitSecondsAndNanos(StringPiece input, StringPiece* seconds, } } +Status GetNanosFromStringPiece(StringPiece s_nanos, + const char* parse_failure_message, + const char* exceeded_limit_message, + int32* nanos) { + *nanos = 0; + + // Count the number of leading 0s and consume them. + int num_leading_zeros = 0; + while (s_nanos.Consume("0")) { + num_leading_zeros++; + } + int32 i_nanos = 0; + // 's_nanos' contains fractional seconds -- i.e. 'nanos' is equal to + // "0." + s_nanos.ToString() seconds. An int32 is used for the + // conversion to 'nanos', rather than a double, so that there is no + // loss of precision. + if (!s_nanos.empty() && !safe_strto32(s_nanos.ToString(), &i_nanos)) { + return Status(INVALID_ARGUMENT, parse_failure_message); + } + if (i_nanos > kNanosPerSecond || i_nanos < 0) { + return Status(INVALID_ARGUMENT, exceeded_limit_message); + } + // s_nanos should only have digits. No whitespace. + if (s_nanos.find_first_not_of("0123456789") != StringPiece::npos) { + return Status(INVALID_ARGUMENT, parse_failure_message); + } + + if (i_nanos > 0) { + // 'scale' is the number of digits to the right of the decimal + // point in "0." + s_nanos.ToString() + int32 scale = num_leading_zeros + s_nanos.size(); + // 'conversion' converts i_nanos into nanoseconds. + // conversion = kNanosPerSecond / static_cast(std::pow(10, scale)) + // For efficiency, we precompute the conversion factor. + int32 conversion = 0; + switch (scale) { + case 1: + conversion = 100000000; + break; + case 2: + conversion = 10000000; + break; + case 3: + conversion = 1000000; + break; + case 4: + conversion = 100000; + break; + case 5: + conversion = 10000; + break; + case 6: + conversion = 1000; + break; + case 7: + conversion = 100; + break; + case 8: + conversion = 10; + break; + case 9: + conversion = 1; + break; + default: + return Status(INVALID_ARGUMENT, exceeded_limit_message); + } + *nanos = i_nanos * conversion; + } + + return Status::OK; +} + } // namespace ProtoStreamObjectWriter::AnyWriter::AnyWriter(ProtoStreamObjectWriter* parent) @@ -421,7 +293,7 @@ void ProtoStreamObjectWriter::AnyWriter::StartAny(const DataPiece& value) { } // Resolve the type url, and report an error if we failed to resolve it. StatusOr resolved_type = - parent_->typeinfo_->ResolveTypeUrl(type_url_); + parent_->typeinfo()->ResolveTypeUrl(type_url_); if (!resolved_type.ok()) { parent_->InvalidValue("Any", resolved_type.status().error_message()); invalid_ = true; @@ -440,8 +312,8 @@ void ProtoStreamObjectWriter::AnyWriter::StartAny(const DataPiece& value) { // Create our object writer and initialize it with the first StartObject // call. - ow_.reset(new ProtoStreamObjectWriter(parent_->typeinfo_, *type, &output_, - parent_->listener_)); + ow_.reset(new ProtoStreamObjectWriter(parent_->typeinfo(), *type, &output_, + parent_->listener())); ow_->StartObject(""); } @@ -453,604 +325,431 @@ void ProtoStreamObjectWriter::AnyWriter::WriteAny() { } // Render the type_url and value fields directly to the stream. // type_url has tag 1 and value has tag 2. - WireFormatLite::WriteString(1, type_url_, parent_->stream_.get()); + WireFormatLite::WriteString(1, type_url_, parent_->stream()); if (!data_.empty()) { - WireFormatLite::WriteBytes(2, data_, parent_->stream_.get()); + WireFormatLite::WriteBytes(2, data_, parent_->stream()); } } -ProtoStreamObjectWriter::ProtoElement::ProtoElement( - const TypeInfo* typeinfo, const google::protobuf::Type& type, - ProtoStreamObjectWriter* enclosing) +ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter* enclosing, + ItemType item_type, bool is_placeholder, + bool is_list) : BaseElement(NULL), ow_(enclosing), any_(), - field_(NULL), - typeinfo_(typeinfo), - type_(type), - required_fields_(GetRequiredFields(type)), - is_repeated_type_(false), - size_index_(-1), - array_index_(-1), - element_type_(GetElementType(type_)) { - if (element_type_ == ANY) { + item_type_(item_type), + is_placeholder_(is_placeholder), + is_list_(is_list) { + if (item_type_ == ANY) { any_.reset(new AnyWriter(ow_)); } } -ProtoStreamObjectWriter::ProtoElement::ProtoElement( - ProtoStreamObjectWriter::ProtoElement* parent, - const google::protobuf::Field* field, const google::protobuf::Type& type, - ElementType element_type) +ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter::Item* parent, + ItemType item_type, bool is_placeholder, + bool is_list) : BaseElement(parent), ow_(this->parent()->ow_), any_(), - field_(field), - typeinfo_(this->parent()->typeinfo_), - type_(type), - is_repeated_type_(element_type == ProtoElement::LIST || - element_type == ProtoElement::STRUCT_LIST || - element_type == ProtoElement::MAP || - element_type == ProtoElement::STRUCT_MAP), - size_index_(!is_repeated_type_ && - field->kind() == - google::protobuf::Field_Kind_TYPE_MESSAGE - ? ow_->size_insert_.size() - : -1), - array_index_(is_repeated_type_ ? 0 : -1), - element_type_(element_type) { - if (!is_repeated_type_) { - if (field->cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) { - // Update array_index_ if it is an explicit list. - if (this->parent()->array_index_ >= 0) this->parent()->array_index_++; - } else { - this->parent()->RegisterField(field); - } - if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { - required_fields_ = GetRequiredFields(type_); - int start_pos = ow_->stream_->ByteCount(); - // length of serialized message is the final buffer position minus - // starting buffer position, plus length adjustments for size fields - // of any nested messages. We start with -start_pos here, so we only - // need to add the final buffer position to it at the end. - SizeInfo info = {start_pos, -start_pos}; - ow_->size_insert_.push_back(info); - } - } - if (element_type == ANY) { + item_type_(item_type), + is_placeholder_(is_placeholder), + is_list_(is_list) { + if (item_type == ANY) { any_.reset(new AnyWriter(ow_)); } } -ProtoStreamObjectWriter::ProtoElement* -ProtoStreamObjectWriter::ProtoElement::pop() { - // Calls the registered error listener for any required field(s) not yet - // seen. - for (set::iterator it = - required_fields_.begin(); - it != required_fields_.end(); ++it) { - ow_->MissingField((*it)->name()); - } - // Computes the total number of proto bytes used by a message, also adjusts - // the size of all parent messages by the length of this size field. - // If size_index_ < 0, this is not a message, so no size field is added. - if (size_index_ >= 0) { - // Add the final buffer position to compute the total length of this - // serialized message. The stored value (before this addition) already - // contains the total length of the size fields of all nested messages - // minus the initial buffer position. - ow_->size_insert_[size_index_].size += ow_->stream_->ByteCount(); - // Calculate the length required to serialize the size field of the - // message, and propagate this additional size information upward to - // all enclosing messages. - int size = ow_->size_insert_[size_index_].size; - int length = CodedOutputStream::VarintSize32(size); - for (ProtoElement* e = parent(); e != NULL; e = e->parent()) { - // Only nested messages have size field, lists do not have size field. - if (e->size_index_ >= 0) { - ow_->size_insert_[e->size_index_].size += length; - } - } - } - return BaseElement::pop(); -} - -void ProtoStreamObjectWriter::ProtoElement::RegisterField( - const google::protobuf::Field* field) { - if (!required_fields_.empty() && - field->cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REQUIRED) { - required_fields_.erase(field); - } -} - -string ProtoStreamObjectWriter::ProtoElement::ToString() const { - if (parent() == NULL) return ""; - string loc = parent()->ToString(); - if (field_->cardinality() != - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED || - parent()->field_ != field_) { - string name = field_->name(); - int i = 0; - while (i < name.size() && (ascii_isalnum(name[i]) || name[i] == '_')) ++i; - if (i > 0 && i == name.size()) { // safe field name - if (loc.empty()) { - loc = name; - } else { - StrAppend(&loc, ".", name); - } - } else { - StrAppend(&loc, "[\"", CEscape(name), "\"]"); - } - } - if (field_->cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED && - array_index_ > 0) { - StrAppend(&loc, "[", array_index_ - 1, "]"); - } - return loc.empty() ? "." : loc; -} - -bool ProtoStreamObjectWriter::ProtoElement::OneofIndexTaken(int32 index) { - return ContainsKey(oneof_indices_, index); -} - -void ProtoStreamObjectWriter::ProtoElement::TakeOneofIndex(int32 index) { - InsertIfNotPresent(&oneof_indices_, index); -} - -bool ProtoStreamObjectWriter::ProtoElement::InsertMapKeyIfNotPresent( +bool ProtoStreamObjectWriter::Item::InsertMapKeyIfNotPresent( StringPiece map_key) { return InsertIfNotPresent(&map_keys_, map_key.ToString()); } -inline void ProtoStreamObjectWriter::InvalidName(StringPiece unknown_name, - StringPiece message) { - listener_->InvalidName(location(), ToSnakeCase(unknown_name), message); -} - -inline void ProtoStreamObjectWriter::InvalidValue(StringPiece type_name, - StringPiece value) { - listener_->InvalidValue(location(), type_name, value); -} - -inline void ProtoStreamObjectWriter::MissingField(StringPiece missing_name) { - listener_->MissingField(location(), missing_name); -} - ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( StringPiece name) { - // Starting the root message. Create the root ProtoElement and return. - if (element_ == NULL) { - if (!name.empty()) { - InvalidName(name, "Root element should not be named."); - } - element_.reset(new ProtoElement(typeinfo_, master_type_, this)); + if (invalid_depth() > 0) { + IncrementInvalidDepth(); + return this; + } + + // Starting the root message. Create the root Item and return. + // ANY message type does not need special handling, just set the ItemType + // to ANY. + if (current_ == NULL) { + ProtoWriter::StartObject(name); + current_.reset(new Item( + this, master_type_.name() == kAnyType ? Item::ANY : Item::MESSAGE, + false, false)); // If master type is a special type that needs extra values to be written to // stream, we write those values. if (master_type_.name() == kStructType) { - StartStruct(NULL); - } else if (master_type_.name() == kStructValueType) { - // We got a StartObject call with google.protobuf.Value field. This means - // we are starting an object within google.protobuf.Value type. The only - // object within that type is a struct type. So start a struct. - const google::protobuf::Field* field = StartStructValueInStruct(NULL); - StartStruct(field); + // Struct has a map field called "fields". + // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto + // "fields": [ + Push("fields", Item::MAP, true, true); + return this; } - return this; - } - const google::protobuf::Field* field = NULL; - if (element_ != NULL && element_->IsAny()) { - element_->any()->StartObject(name); - return this; - } else if (element_ != NULL && - (element_->IsMap() || element_->IsStructMap())) { - if (!ValidMapKey(name)) { - ++invalid_depth_; + if (master_type_.name() == kStructValueType) { + // We got a StartObject call with google.protobuf.Value field. The only + // object within that type is a struct type. So start a struct. + // + // The struct field in Value type is named "struct_value" + // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto + // Also start the map field "fields" within the struct. + // "struct_value": { + // "fields": [ + Push("struct_value", Item::MESSAGE, true, false); + Push("fields", Item::MAP, true, true); return this; } - field = StartMapEntry(name); - if (element_->IsStructMapEntry()) { - // If the top element is a map entry, this means we are starting another - // struct within a struct. - field = StartStructValueInStruct(field); + if (master_type_.name() == kStructListValueType) { + InvalidValue(kStructListValueType, + "Cannot start root message with ListValue."); } - } else if (element_ != NULL && element_->IsStructList()) { - // If the top element is a list, then we are starting a list field within a - // struct. - field = Lookup(name); - field = StartStructValueInStruct(field); - } else { - field = BeginNamed(name, false); - } - if (field == NULL) { - return this; - } - const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { - ++invalid_depth_; - InvalidName(name, - StrCat("Missing descriptor for field: ", field->type_url())); return this; } - // Check to see if this field is a oneof and that no oneof in that group has - // already been set. - if (!ValidOneof(*field, name)) { - ++invalid_depth_; + // Send all ANY events to AnyWriter. + if (current_->IsAny()) { + current_->any()->StartObject(name); return this; } - if (field->type_url() == GetFullTypeWithUrl(kStructType)) { - // Start a struct object. - StartStruct(field); - } else if (field->type_url() == GetFullTypeWithUrl(kStructValueType)) { - // We got a StartObject call with google.protobuf.Value field. This means we - // are starting an object within google.protobuf.Value type. The only object - // within that type is a struct type. So start a struct. - field = StartStructValueInStruct(field); - StartStruct(field); - } else if (field->type_url() == GetFullTypeWithUrl(kAnyType)) { - // Begin an Any. We can't do the real work till we get the @type field. - WriteTag(*field); - element_.reset( - new ProtoElement(element_.release(), field, *type, ProtoElement::ANY)); - } else if (IsMap(*field)) { - // Begin a map. - // A map is triggered by a StartObject() call if the current field has a map - // type. Map values are written to proto in a manner detailed in comments - // above StartMapEntry() function. - element_.reset( - new ProtoElement(element_.release(), field, *type, ProtoElement::MAP)); - } else { - WriteTag(*field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::MESSAGE)); - } - return this; -} - -// Proto3 maps are represented on the wire as a message with -// "key" and a "value". -// -// For example, the syntax: -// map map_field = N; -// -// is represented as: -// message MapFieldEntry { -// option map_entry = true; // marks the map construct in the descriptor -// -// key_type key = 1; -// value_type value = 2; -// } -// repeated MapFieldEntry map_field = N; -// -// See go/proto3-maps for more information. -const google::protobuf::Field* ProtoStreamObjectWriter::StartMapEntry( - StringPiece name) { - // top of stack is already a map field - const google::protobuf::Field* field = element_->field(); - const google::protobuf::Type& type = element_->type(); - // If we come from a regular map, use MAP_ENTRY or if we come from a struct, - // use STRUCT_MAP_ENTRY. These values are used later in StartObject/StartList - // or RenderDataPiece for making appropriate decisions. - ProtoElement::ElementType element_type = element_->IsStructMap() - ? ProtoElement::STRUCT_MAP_ENTRY - : ProtoElement::MAP_ENTRY; - WriteTag(*field); - element_.reset( - new ProtoElement(element_.release(), field, type, element_type)); - RenderDataPiece("key", DataPiece(name)); - return BeginNamed("value", false); -} + // If we are within a map, we render name as keys and send StartObject to the + // value field. + if (current_->IsMap()) { + if (!ValidMapKey(name)) { + IncrementInvalidDepth(); + return this; + } -// Starts a google.protobuf.Struct. -// 'field' represents a field in a message of type google.protobuf.Struct. A -// struct contains a map with name 'fields'. This function starts this map as -// well. -// When 'field' is NULL, it means that the top level message is of struct -// type. -void ProtoStreamObjectWriter::StartStruct( - const google::protobuf::Field* field) { - const google::protobuf::Type* type = NULL; - if (field) { - type = LookupType(field); - WriteTag(*field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::STRUCT)); - } - const google::protobuf::Field* struct_field = BeginNamed("fields", false); + // Map is a repeated field of message type with a "key" and a "value" field. + // https://developers.google.com/protocol-buffers/docs/proto3?hl=en#maps + // message MapFieldEntry { + // key_type key = 1; + // value_type value = 2; + // } + // + // repeated MapFieldEntry map_field = N; + // + // That means, we render the following element within a list (hence no + // name): + // { "key": "", "value": { + Push("", Item::MESSAGE, false, false); + ProtoWriter::RenderDataPiece("key", DataPiece(name)); + Push("value", Item::MESSAGE, true, false); + + // Make sure we are valid so far after starting map fields. + if (invalid_depth() > 0) return this; + + // If top of stack is g.p.Struct type, start the struct the map field within + // it. + if (element() != NULL && IsStruct(*element()->parent_field())) { + // Render "fields": [ + Push("fields", Item::MAP, true, true); + return this; + } - if (!struct_field) { - // It is a programmatic error if this happens. Log an error. - GOOGLE_LOG(ERROR) << "Invalid internal state. Cannot find 'fields' within " - << (field ? field->type_url() : "google.protobuf.Struct"); - return; + // If top of stack is g.p.Value type, start the Struct within it. + if (element() != NULL && IsStructValue(*element()->parent_field())) { + // Render + // "struct_value": { + // "fields": [ + Push("struct_value", Item::MESSAGE, true, false); + Push("fields", Item::MAP, true, true); + } + return this; } - type = LookupType(struct_field); - element_.reset(new ProtoElement(element_.release(), struct_field, *type, - ProtoElement::STRUCT_MAP)); -} + const google::protobuf::Field* field = BeginNamed(name, false); + if (field == NULL) return this; -// Starts a "struct_value" within struct.proto's google.protobuf.Value type. -// 'field' should be of the type google.protobuf.Value. -// Returns the field identifying "struct_value" within the given field. -// -// If field is NULL, then we are starting struct_value at the top-level, in -// this case skip writing any tag information for the passed field. -const google::protobuf::Field* -ProtoStreamObjectWriter::StartStructValueInStruct( - const google::protobuf::Field* field) { - if (field) { - const google::protobuf::Type* type = LookupType(field); - WriteTag(*field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::STRUCT_VALUE)); - } - return BeginNamed("struct_value", false); -} - -// Starts a "list_value" within struct.proto's google.protobuf.Value type. -// 'field' should be of the type google.protobuf.Value. -// Returns the field identifying "list_value" within the given field. -// -// If field is NULL, then we are starting list_value at the top-level, in -// this case skip writing any tag information for the passed field. -const google::protobuf::Field* ProtoStreamObjectWriter::StartListValueInStruct( - const google::protobuf::Field* field) { - if (field) { - const google::protobuf::Type* type = LookupType(field); - WriteTag(*field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::STRUCT_VALUE)); + if (IsStruct(*field)) { + // Start a struct object. + // Render + // "": { + // "fields": { + Push(name, Item::MESSAGE, false, false); + Push("fields", Item::MAP, true, true); + return this; } - const google::protobuf::Field* list_value = BeginNamed("list_value", false); - if (!list_value) { - // It is a programmatic error if this happens. Log an error. - GOOGLE_LOG(ERROR) << "Invalid internal state. Cannot find 'list_value' within " - << (field ? field->type_url() : "google.protobuf.Value"); - return field; + if (IsStructValue(*field)) { + // We got a StartObject call with google.protobuf.Value field. The only + // object within that type is a struct type. So start a struct. + // Render + // "": { + // "struct_value": { + // "fields": { + Push(name, Item::MESSAGE, false, false); + Push("struct_value", Item::MESSAGE, true, false); + Push("fields", Item::MAP, true, true); + return this; } - return StartRepeatedValuesInListValue(list_value); -} - -// Starts the repeated "values" field in struct.proto's -// google.protobuf.ListValue type. 'field' should be of type -// google.protobuf.ListValue. -// -// If field is NULL, then we are starting ListValue at the top-level, in -// this case skip writing any tag information for the passed field. -const google::protobuf::Field* -ProtoStreamObjectWriter::StartRepeatedValuesInListValue( - const google::protobuf::Field* field) { - if (field) { - const google::protobuf::Type* type = LookupType(field); - WriteTag(*field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::STRUCT_LIST_VALUE)); + if (IsMap(*field)) { + // Begin a map. A map is triggered by a StartObject() call if the current + // field has a map type. + // A map type is always repeated, hence set is_list to true. + // Render + // "": [ + Push(name, Item::MAP, false, true); + return this; } - return BeginNamed("values", true); -} -void ProtoStreamObjectWriter::SkipElements() { - if (element_ == NULL) return; - - ProtoElement::ElementType element_type = element_->element_type(); - while (element_type == ProtoElement::STRUCT || - element_type == ProtoElement::STRUCT_LIST_VALUE || - element_type == ProtoElement::STRUCT_VALUE || - element_type == ProtoElement::STRUCT_MAP_ENTRY || - element_type == ProtoElement::MAP_ENTRY) { - element_.reset(element_->pop()); - element_type = - element_ != NULL ? element_->element_type() : ProtoElement::MESSAGE; - } + // A regular message type. Pass it directly to ProtoWriter. + // Render + // "": { + Push(name, IsAny(*field) ? Item::ANY : Item::MESSAGE, false, false); + return this; } ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndObject() { - if (invalid_depth_ > 0) { - --invalid_depth_; + if (invalid_depth() > 0) { + DecrementInvalidDepth(); return this; } - if (element_ != NULL && element_->IsAny()) { - if (element_->any()->EndObject()) { - return this; - } - } - if (element_ != NULL) { - element_.reset(element_->pop()); - } - // Skip sentinel elements added to keep track of new proto3 types - map, - // struct. - SkipElements(); + if (current_ == NULL) return this; - - // If ending the root element, - // then serialize the full message with calculated sizes. - if (element_ == NULL) { - WriteRootMessage(); + if (current_->IsAny()) { + if (current_->any()->EndObject()) return this; } + + Pop(); + return this; } ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { - const google::protobuf::Field* field = NULL; + if (invalid_depth() > 0) { + IncrementInvalidDepth(); + return this; + } + // Since we cannot have a top-level repeated item in protobuf, the only way - // element_ can be null when here is when we start a top-level list - // google.protobuf.ListValue. - if (element_ == NULL) { + // this is valid is if we start a special type google.protobuf.ListValue or + // google.protobuf.Value. + if (current_ == NULL) { if (!name.empty()) { InvalidName(name, "Root element should not be named."); + IncrementInvalidDepth(); + return this; } - element_.reset(new ProtoElement(typeinfo_, master_type_, this)); // If master type is a special type that needs extra values to be written to // stream, we write those values. if (master_type_.name() == kStructValueType) { // We got a StartList with google.protobuf.Value master type. This means // we have to start the "list_value" within google.protobuf.Value. - field = StartListValueInStruct(NULL); - } else if (master_type_.name() == kStructListValueType) { + // + // See + // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto + // + // Render + // "": { + // "list_value": { + // "values": [ // Start this list. + ProtoWriter::StartObject(name); + current_.reset(new Item(this, Item::MESSAGE, false, false)); + Push("list_value", Item::MESSAGE, true, false); + Push("values", Item::MESSAGE, true, true); + return this; + } + + if (master_type_.name() == kStructListValueType) { // We got a StartList with google.protobuf.ListValue master type. This // means we have to start the "values" within google.protobuf.ListValue. - field = StartRepeatedValuesInListValue(NULL); + // + // Render + // "": { + // "values": [ // Start this list. + ProtoWriter::StartObject(name); + current_.reset(new Item(this, Item::MESSAGE, false, false)); + Push("values", Item::MESSAGE, true, true); + return this; } - // field is NULL when master_type_ is anything other than - // google.protobuf.Value or google.protobuf.ListValue. - if (field) { - const google::protobuf::Type* type = LookupType(field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::STRUCT_LIST)); - } + // Send the event to ProtoWriter so proper errors can be reported. + // + // Render a regular list: + // "": [ + ProtoWriter::StartList(name); + current_.reset(new Item(this, Item::MESSAGE, false, true)); return this; } - if (element_->IsAny()) { - element_->any()->StartList(name); + if (current_->IsAny()) { + current_->any()->StartList(name); return this; } - // The type of element we push to stack. - ProtoElement::ElementType element_type = ProtoElement::LIST; - // Check if we need to start a map. This can heppen when there is either a map - // or a struct type within a list. - if (element_->IsMap() || element_->IsStructMap()) { + // If the top of stack is a map, we are starting a list value within a map. + // Since map does not allow repeated values, this can only happen when the map + // value is of a special type that renders a list in JSON. These can be one + // of 3 cases: + // i. We are rendering a list value within google.protobuf.Struct + // ii. We are rendering a list value within google.protobuf.Value + // iii. We are rendering a list value with type google.protobuf.ListValue. + if (current_->IsMap()) { if (!ValidMapKey(name)) { - ++invalid_depth_; + IncrementInvalidDepth(); return this; } - field = StartMapEntry(name); - if (field == NULL) return this; + // Start the repeated map entry object. + // Render + // { "key": "", "value": { + Push("", Item::MESSAGE, false, false); + ProtoWriter::RenderDataPiece("key", DataPiece(name)); + Push("value", Item::MESSAGE, true, false); + + // Make sure we are valid after pushing all above items. + if (invalid_depth() > 0) return this; + + // case i and ii above. Start "list_value" field within g.p.Value + if (element() != NULL && element()->parent_field() != NULL) { + // Render + // "list_value": { + // "values": [ // Start this list + if (IsStructValue(*element()->parent_field())) { + Push("list_value", Item::MESSAGE, true, false); + Push("values", Item::MESSAGE, true, true); + return this; + } - if (element_->IsStructMapEntry()) { - // If the top element is a map entry, this means we are starting a list - // within a struct or a map. - // An example sequence of calls would be - // StartObject -> StartList - field = StartListValueInStruct(field); - if (field == NULL) return this; + // Render + // "values": [ + if (IsStructListValue(*element()->parent_field())) { + // case iii above. Bind directly to g.p.ListValue + Push("values", Item::MESSAGE, true, true); + return this; + } } - element_type = ProtoElement::STRUCT_LIST; - } else if (element_->IsStructList()) { - // If the top element is a STRUCT_LIST, this means we are starting a list - // within the current list (inside a struct). - // An example call sequence would be - // StartObject -> StartList -> StartList - // with StartObject starting a struct. + // Report an error. + InvalidValue("Map", StrCat("Cannot have repeated items ('", name, + "') within a map.")); + return this; + } - // Lookup the last list type in element stack as we are adding an element of - // the same type. - field = Lookup(name); - if (field == NULL) return this; + // When name is empty and stack is not empty, we are rendering an item within + // a list. + if (name.empty()) { + if (element() != NULL && element()->parent_field() != NULL) { + if (IsStructValue(*element()->parent_field())) { + // Since it is g.p.Value, we bind directly to the list_value. + // Render + // { // g.p.Value item within the list + // "list_value": { + // "values": [ + Push("", Item::MESSAGE, false, false); + Push("list_value", Item::MESSAGE, true, false); + Push("values", Item::MESSAGE, true, true); + return this; + } - field = StartListValueInStruct(field); - if (field == NULL) return this; + if (IsStructListValue(*element()->parent_field())) { + // Since it is g.p.ListValue, we bind to it directly. + // Render + // { // g.p.ListValue item within the list + // "values": [ + Push("", Item::MESSAGE, false, false); + Push("values", Item::MESSAGE, true, true); + return this; + } + } - element_type = ProtoElement::STRUCT_LIST; - } else { - // Lookup field corresponding to 'name'. If it is a google.protobuf.Value - // or google.protobuf.ListValue type, then StartList is a valid call, start - // this list. - // We cannot use Lookup() here as it will produce InvalidName() error if the - // field is not found. We do not want to error here as it would cause us to - // report errors twice, once here and again later in BeginNamed() call. - // Also we ignore if the field is not found here as it is caught later. - field = typeinfo_->FindField(&element_->type(), name); - - // Only check for oneof collisions on the first StartList call. We identify - // the first call with !name.empty() check. Subsequent list element calls - // will not have the name filled. - if (!name.empty() && field && !ValidOneof(*field, name)) { - ++invalid_depth_; + // Pass the event to underlying ProtoWriter. + Push(name, Item::MESSAGE, false, true); + return this; + } + + // name is not empty + const google::protobuf::Field* field = Lookup(name); + if (field == NULL) { + IncrementInvalidDepth(); + return this; + } + + if (IsStructValue(*field)) { + // If g.p.Value is repeated, start that list. Otherwise, start the + // "list_value" within it. + if (IsRepeated(*field)) { + // Render it just like a regular repeated field. + // "": [ + Push(name, Item::MESSAGE, false, true); return this; } - // It is an error to try to bind to map, which behind the scenes is a list. - if (field && IsMap(*field)) { - // Push field to stack for error location tracking & reporting. - element_.reset(new ProtoElement(element_.release(), field, - *LookupType(field), - ProtoElement::MESSAGE)); - InvalidValue("Map", "Cannot bind a list to map."); - ++invalid_depth_; - element_.reset(element_->pop()); + // Start the "list_value" field. + // Render + // "": { + // "list_value": { + // "values": [ + Push(name, Item::MESSAGE, false, false); + Push("list_value", Item::MESSAGE, true, false); + Push("values", Item::MESSAGE, true, true); + return this; + } + + if (IsStructListValue(*field)) { + // If g.p.ListValue is repeated, start that list. Otherwise, start the + // "values" within it. + if (IsRepeated(*field)) { + // Render it just like a regular repeated field. + // "": [ + Push(name, Item::MESSAGE, false, true); return this; } - if (field && field->type_url() == GetFullTypeWithUrl(kStructValueType)) { - // There are 2 cases possible: - // a. g.p.Value is repeated - // b. g.p.Value is not repeated - // - // For case (a), the StartList should bind to the repeated g.p.Value. - // For case (b), the StartList should bind to g.p.ListValue within the - // g.p.Value. - // - // This means, for case (a), we treat it just like any other repeated - // message, except we would apply an appropriate element_type so future - // Start or Render calls are routed appropriately. - if (field->cardinality() != - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) { - field = StartListValueInStruct(field); - } - element_type = ProtoElement::STRUCT_LIST; - } else if (field && - field->type_url() == GetFullTypeWithUrl(kStructListValueType)) { - // We got a StartList with google.protobuf.ListValue master type. This - // means we have to start the "values" within google.protobuf.ListValue. - field = StartRepeatedValuesInListValue(field); - } else { - // If no special types are to be bound, fall back to normal processing of - // StartList. - field = BeginNamed(name, true); - } - if (field == NULL) return this; + // Start the "values" field within g.p.ListValue. + // Render + // "": { + // "values": [ + Push(name, Item::MESSAGE, false, false); + Push("values", Item::MESSAGE, true, true); + return this; } - const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { - ++invalid_depth_; - InvalidName(name, - StrCat("Missing descriptor for field: ", field->type_url())); + // If we are here, the field should be repeated. Report an error otherwise. + if (!IsRepeated(*field)) { + IncrementInvalidDepth(); + InvalidName(name, "Proto field is not repeating, cannot start list."); return this; } - element_.reset( - new ProtoElement(element_.release(), field, *type, element_type)); + if (IsMap(*field)) { + InvalidValue("Map", + StrCat("Cannot bind a list to map for field '", name, "'.")); + IncrementInvalidDepth(); + return this; + } + + // Pass the event to ProtoWriter. + // Render + // "": [ + Push(name, Item::MESSAGE, false, true); return this; } ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndList() { - if (invalid_depth_ > 0) { - --invalid_depth_; - } else if (element_ != NULL) { - if (element_->IsAny()) { - element_->any()->EndList(); - } else { - element_.reset(element_->pop()); - // Skip sentinel elements added to keep track of new proto3 types - map, - // struct. - SkipElements(); - } + if (invalid_depth() > 0) { + DecrementInvalidDepth(); + return this; } - // When element_ is NULL, we have reached the root message type. Write out - // the bytes. - if (element_ == NULL) { - WriteRootMessage(); + if (current_ == NULL) return this; + + if (current_->IsAny()) { + current_->any()->EndList(); + return this; } + + Pop(); return this; } @@ -1083,7 +782,7 @@ Status ProtoStreamObjectWriter::RenderStructValue(ProtoStreamObjectWriter* ow, "null values are supported."); } } - ow->RenderDataPiece(struct_field_name, data); + ow->ProtoWriter::RenderDataPiece(struct_field_name, data); return Status::OK; } @@ -1105,15 +804,15 @@ Status ProtoStreamObjectWriter::RenderTimestamp(ProtoStreamObjectWriter* ow, } - ow->RenderDataPiece("seconds", DataPiece(seconds)); - ow->RenderDataPiece("nanos", DataPiece(nanos)); + ow->ProtoWriter::RenderDataPiece("seconds", DataPiece(seconds)); + ow->ProtoWriter::RenderDataPiece("nanos", DataPiece(nanos)); return Status::OK; } static inline util::Status RenderOneFieldPath(ProtoStreamObjectWriter* ow, StringPiece path) { - ow->RenderDataPiece("paths", - DataPiece(ConvertFieldMaskPath(path, &ToSnakeCase))); + ow->ProtoWriter::RenderDataPiece( + "paths", DataPiece(ConvertFieldMaskPath(path, &ToSnakeCase))); return Status::OK; } @@ -1162,224 +861,133 @@ Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, "Invalid duration format, failed to parse seconds"); } - double d_nanos = 0; - if (!safe_strtod("0." + s_nanos.ToString(), &d_nanos)) { - return Status(INVALID_ARGUMENT, - "Invalid duration format, failed to parse nanos seconds"); + int32 nanos = 0; + Status nanos_status = GetNanosFromStringPiece( + s_nanos, "Invalid duration format, failed to parse nano seconds", + "Duration value exceeds limits", &nanos); + if (!nanos_status.ok()) { + return nanos_status; } + nanos = sign * nanos; - int32 nanos = sign * static_cast(d_nanos * kNanosPerSecond); int64 seconds = sign * unsigned_seconds; - if (seconds > kMaxSeconds || seconds < kMinSeconds || nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { return Status(INVALID_ARGUMENT, "Duration value exceeds limits"); } - ow->RenderDataPiece("seconds", DataPiece(seconds)); - ow->RenderDataPiece("nanos", DataPiece(nanos)); + ow->ProtoWriter::RenderDataPiece("seconds", DataPiece(seconds)); + ow->ProtoWriter::RenderDataPiece("nanos", DataPiece(nanos)); return Status::OK; } Status ProtoStreamObjectWriter::RenderWrapperType(ProtoStreamObjectWriter* ow, const DataPiece& data) { - ow->RenderDataPiece("value", data); + ow->ProtoWriter::RenderDataPiece("value", data); return Status::OK; } ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( StringPiece name, const DataPiece& data) { Status status; - if (invalid_depth_ > 0) return this; - if (element_ != NULL && element_->IsAny()) { - element_->any()->RenderDataPiece(name, data); + if (invalid_depth() > 0) return this; + + if (current_ == NULL) { + const TypeRenderer* type_renderer = + FindTypeRenderer(GetFullTypeWithUrl(master_type_.name())); + if (type_renderer == NULL) { + InvalidName(name, "Root element must be a message."); + return this; + } + // Render the special type. + // "": { + // ... Render special type ... + // } + ProtoWriter::StartObject(name); + status = (*type_renderer)(this, data); + if (!status.ok()) { + InvalidValue(master_type_.name(), + StrCat("Field '", name, "', ", status.error_message())); + } + ProtoWriter::EndObject(); + return this; + } + + if (current_->IsAny()) { + current_->any()->RenderDataPiece(name, data); return this; } const google::protobuf::Field* field = NULL; - string type_url; - bool is_map_entry = false; - // We are at the root when element_ == NULL. - if (element_ == NULL) { - type_url = GetFullTypeWithUrl(master_type_.name()); - } else { - if (element_->IsMap() || element_->IsStructMap()) { - if (!ValidMapKey(name)) return this; - is_map_entry = true; - field = StartMapEntry(name); - } else { - field = Lookup(name); - } + if (current_->IsMap()) { + if (!ValidMapKey(name)) return this; + + // Render an item in repeated map list. + // { "key": "", "value": + Push("", Item::MESSAGE, false, false); + ProtoWriter::RenderDataPiece("key", DataPiece(name)); + field = Lookup("value"); if (field == NULL) { + GOOGLE_LOG(DFATAL) << "Map does not have a value field."; + return this; + } + + const TypeRenderer* type_renderer = FindTypeRenderer(field->type_url()); + if (type_renderer != NULL) { + // Map's value type is a special type. Render it like a message: + // "value": { + // ... Render special type ... + // } + Push("value", Item::MESSAGE, true, false); + status = (*type_renderer)(this, data); + if (!status.ok()) { + InvalidValue(field->type_url(), + StrCat("Field '", name, "', ", status.error_message())); + } + Pop(); return this; } - // Check to see if this field is a oneof and that no oneof in that group has - // already been set. - if (!ValidOneof(*field, name)) return this; + // If we are rendering explicit null values and the backend proto field is + // not of the google.protobuf.NullType type, we do nothing. + if (data.type() == DataPiece::TYPE_NULL && + field->type_url() != kStructNullValueTypeUrl) { + return this; + } - type_url = field->type_url(); + // Render the map value as a primitive type. + ProtoWriter::RenderDataPiece("value", data); + Pop(); + return this; } - // Check if there are any well known type renderers available for type_url. - const TypeRenderer* type_renderer = FindTypeRenderer(type_url); + field = Lookup(name); + if (field == NULL) return this; + + // Check if the field is of special type. Render it accordingly if so. + const TypeRenderer* type_renderer = FindTypeRenderer(field->type_url()); if (type_renderer != NULL) { - // Push the current element to stack so lookups in type_renderer will - // find the fields. We do an EndObject soon after, which pops this. This is - // safe because all well-known types are messages. - if (element_ == NULL) { - element_.reset(new ProtoElement(typeinfo_, master_type_, this)); - } else { - if (field) { - WriteTag(*field); - const google::protobuf::Type* type = LookupType(field); - element_.reset(new ProtoElement(element_.release(), field, *type, - ProtoElement::MESSAGE)); - } - } + Push(name, Item::MESSAGE, false, false); status = (*type_renderer)(this, data); if (!status.ok()) { - InvalidValue(type_url, + InvalidValue(field->type_url(), StrCat("Field '", name, "', ", status.error_message())); } - EndObject(); - return this; - } else if (element_ == NULL) { // no message type found at root - element_.reset(new ProtoElement(typeinfo_, master_type_, this)); - InvalidName(name, "Root element must be a message."); + Pop(); return this; } - if (field == NULL) { - return this; - } - const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { - InvalidName(name, - StrCat("Missing descriptor for field: ", field->type_url())); + // If we are rendering explicit null values and the backend proto field is + // not of the google.protobuf.NullType type, we do nothing. + if (data.type() == DataPiece::TYPE_NULL && + field->type_url() != kStructNullValueTypeUrl) { return this; } - // Whether we should pop at the end. Set to true if the data field is a - // message type, which can happen in case of struct values. - bool should_pop = false; - - RenderSimpleDataPiece(*field, *type, data); - - if (should_pop && element_ != NULL) { - element_.reset(element_->pop()); - } - - if (is_map_entry) { - // Ending map is the same as ending an object. - EndObject(); - } + ProtoWriter::RenderDataPiece(name, data); return this; } -void ProtoStreamObjectWriter::RenderSimpleDataPiece( - const google::protobuf::Field& field, const google::protobuf::Type& type, - const DataPiece& data) { - // If we are rendering explicit null values and the backend proto field is not - // of the google.protobuf.NullType type, we do nothing. - if (data.type() == DataPiece::TYPE_NULL && - field.type_url() != kStructNullValueTypeUrl) { - return; - } - - // Pushing a ProtoElement and then pop it off at the end for 2 purposes: - // error location reporting and required field accounting. - element_.reset(new ProtoElement(element_.release(), &field, type, - ProtoElement::MESSAGE)); - - // Make sure that field represents a simple data type. - if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN || - field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { - InvalidValue(field.type_url().empty() - ? google::protobuf::Field_Kind_Name(field.kind()) - : field.type_url(), - data.ValueAsStringOrDefault("")); - return; - } - - Status status; - switch (field.kind()) { - case google::protobuf::Field_Kind_TYPE_INT32: { - status = WriteInt32(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_SFIXED32: { - status = WriteSFixed32(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_SINT32: { - status = WriteSInt32(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_FIXED32: { - status = WriteFixed32(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_UINT32: { - status = WriteUInt32(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_INT64: { - status = WriteInt64(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_SFIXED64: { - status = WriteSFixed64(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_SINT64: { - status = WriteSInt64(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_FIXED64: { - status = WriteFixed64(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_UINT64: { - status = WriteUInt64(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_DOUBLE: { - status = WriteDouble(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_FLOAT: { - status = WriteFloat(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_BOOL: { - status = WriteBool(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_BYTES: { - status = WriteBytes(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_STRING: { - status = WriteString(field.number(), data, stream_.get()); - break; - } - case google::protobuf::Field_Kind_TYPE_ENUM: { - status = WriteEnum(field.number(), data, - typeinfo_->GetEnumByTypeUrl(field.type_url()), - stream_.get()); - break; - } - default: // TYPE_GROUP or TYPE_MESSAGE - status = Status(INVALID_ARGUMENT, data.ToString().ValueOrDie()); - } - if (!status.ok()) { - InvalidValue(google::protobuf::Field_Kind_Name(field.kind()), - status.error_message()); - } - element_.reset(element_->pop()); -} - // Map of functions that are responsible for rendering well known type // represented by the key. hash_map* @@ -1446,45 +1054,12 @@ ProtoStreamObjectWriter::FindTypeRenderer(const string& type_url) { return FindOrNull(*renderers_, type_url); } -ProtoStreamObjectWriter::ProtoElement::ElementType -ProtoStreamObjectWriter::GetElementType(const google::protobuf::Type& type) { - if (type.name() == kAnyType) { - return ProtoElement::ANY; - } else if (type.name() == kStructType) { - return ProtoElement::STRUCT; - } else if (type.name() == kStructValueType) { - return ProtoElement::STRUCT_VALUE; - } else if (type.name() == kStructListValueType) { - return ProtoElement::STRUCT_LIST_VALUE; - } else { - return ProtoElement::MESSAGE; - } -} - -bool ProtoStreamObjectWriter::ValidOneof(const google::protobuf::Field& field, - StringPiece unnormalized_name) { - if (element_ == NULL) return true; - - if (field.oneof_index() > 0) { - if (element_->OneofIndexTaken(field.oneof_index())) { - InvalidValue( - "oneof", - StrCat("oneof field '", - element_->type().oneofs(field.oneof_index() - 1), - "' is already set. Cannot set '", unnormalized_name, "'")); - return false; - } - element_->TakeOneofIndex(field.oneof_index()); - } - return true; -} - bool ProtoStreamObjectWriter::ValidMapKey(StringPiece unnormalized_name) { - if (element_ == NULL) return true; + if (current_ == NULL) return true; - if (!element_->InsertMapKeyIfNotPresent(unnormalized_name)) { - InvalidName( - unnormalized_name, + if (!current_->InsertMapKeyIfNotPresent(unnormalized_name)) { + listener()->InvalidName( + location(), unnormalized_name, StrCat("Repeated map key: '", unnormalized_name, "' is already set.")); return false; } @@ -1492,134 +1067,30 @@ bool ProtoStreamObjectWriter::ValidMapKey(StringPiece unnormalized_name) { return true; } -const google::protobuf::Field* ProtoStreamObjectWriter::BeginNamed( - StringPiece name, bool is_list) { - if (invalid_depth_ > 0) { - ++invalid_depth_; - return NULL; - } - const google::protobuf::Field* field = Lookup(name); - if (field == NULL) { - ++invalid_depth_; - // InvalidName() already called in Lookup(). - return NULL; - } - if (is_list && - field->cardinality() != - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) { - ++invalid_depth_; - InvalidName(name, "Proto field is not repeating, cannot start list."); - return NULL; - } - return field; -} +void ProtoStreamObjectWriter::Push(StringPiece name, Item::ItemType item_type, + bool is_placeholder, bool is_list) { + is_list ? ProtoWriter::StartList(name) : ProtoWriter::StartObject(name); -const google::protobuf::Field* ProtoStreamObjectWriter::Lookup( - StringPiece unnormalized_name) { - ProtoElement* e = element(); - if (e == NULL) { - InvalidName(unnormalized_name, "Root element must be a message."); - return NULL; - } - if (unnormalized_name.empty()) { - // Objects in repeated field inherit the same field descriptor. - if (e->field() == NULL) { - InvalidName(unnormalized_name, "Proto fields must have a name."); - } else if (e->field()->cardinality() != - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED) { - InvalidName(unnormalized_name, "Proto fields must have a name."); - return NULL; - } - return e->field(); - } - const google::protobuf::Field* field = - typeinfo_->FindField(&e->type(), unnormalized_name); - if (field == NULL) InvalidName(unnormalized_name, "Cannot find field."); - return field; -} - -const google::protobuf::Type* ProtoStreamObjectWriter::LookupType( - const google::protobuf::Field* field) { - return (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE - ? typeinfo_->GetTypeByTypeUrl(field->type_url()) - : &element_->type()); + // invalid_depth == 0 means it is a successful StartObject or StartList. + if (invalid_depth() == 0) + current_.reset( + new Item(current_.release(), item_type, is_placeholder, is_list)); } -// Looks up the oneof struct field based on the data type. -StatusOr -ProtoStreamObjectWriter::LookupStructField(DataPiece::Type type) { - const google::protobuf::Field* field = NULL; - switch (type) { - // Our JSON parser parses numbers as either int64, uint64, or double. - case DataPiece::TYPE_INT64: - case DataPiece::TYPE_UINT64: - case DataPiece::TYPE_DOUBLE: { - field = Lookup("number_value"); - break; - } - case DataPiece::TYPE_STRING: { - field = Lookup("string_value"); - break; - } - case DataPiece::TYPE_BOOL: { - field = Lookup("bool_value"); - break; - } - case DataPiece::TYPE_NULL: { - field = Lookup("null_value"); - break; - } - default: { return Status(INVALID_ARGUMENT, "Invalid struct data type"); } +void ProtoStreamObjectWriter::Pop() { + // Pop all placeholder items sending StartObject or StartList events to + // ProtoWriter according to is_list value. + while (current_ != NULL && current_->is_placeholder()) { + PopOneElement(); } - if (field == NULL) { - return Status(INVALID_ARGUMENT, "Could not lookup struct field"); + if (current_ != NULL) { + PopOneElement(); } - return field; } -void ProtoStreamObjectWriter::WriteRootMessage() { - GOOGLE_DCHECK(!done_); - int curr_pos = 0; - // Calls the destructor of CodedOutputStream to remove any uninitialized - // memory from the Cord before we read it. - stream_.reset(NULL); - const void* data; - int length; - google::protobuf::io::ArrayInputStream input_stream(buffer_.data(), buffer_.size()); - while (input_stream.Next(&data, &length)) { - if (length == 0) continue; - int num_bytes = length; - // Write up to where we need to insert the size field. - // The number of bytes we may write is the smaller of: - // - the current fragment size - // - the distance to the next position where a size field needs to be - // inserted. - if (!size_insert_.empty() && - size_insert_.front().pos - curr_pos < num_bytes) { - num_bytes = size_insert_.front().pos - curr_pos; - } - output_->Append(static_cast(data), num_bytes); - if (num_bytes < length) { - input_stream.BackUp(length - num_bytes); - } - curr_pos += num_bytes; - // Insert the size field. - // size_insert_.front(): the next pair to be written. - // size_insert_.front().pos: position of the size field. - // size_insert_.front().size: the size (integer) to be inserted. - if (!size_insert_.empty() && curr_pos == size_insert_.front().pos) { - // Varint32 occupies at most 10 bytes. - uint8 insert_buffer[10]; - uint8* insert_buffer_pos = CodedOutputStream::WriteVarint32ToArray( - size_insert_.front().size, insert_buffer); - output_->Append(reinterpret_cast(insert_buffer), - insert_buffer_pos - insert_buffer); - size_insert_.pop_front(); - } - } - output_->Flush(); - stream_.reset(new CodedOutputStream(&adapter_)); - done_ = true; +void ProtoStreamObjectWriter::PopOneElement() { + current_->is_list() ? ProtoWriter::EndList() : ProtoWriter::EndObject(); + current_.reset(current_->pop()); } bool ProtoStreamObjectWriter::IsMap(const google::protobuf::Field& field) { @@ -1630,7 +1101,7 @@ bool ProtoStreamObjectWriter::IsMap(const google::protobuf::Field& field) { return false; } const google::protobuf::Type* field_type = - typeinfo_->GetTypeByTypeUrl(field.type_url()); + typeinfo()->GetTypeByTypeUrl(field.type_url()); // TODO(xiaofeng): Unify option names. return GetBoolOptionOrDefault(field_type->options(), @@ -1638,12 +1109,23 @@ bool ProtoStreamObjectWriter::IsMap(const google::protobuf::Field& field) { GetBoolOptionOrDefault(field_type->options(), "map_entry", false); } -void ProtoStreamObjectWriter::WriteTag(const google::protobuf::Field& field) { - WireFormatLite::WireType wire_type = WireFormatLite::WireTypeForFieldType( - static_cast(field.kind())); - stream_->WriteTag(WireFormatLite::MakeTag(field.number(), wire_type)); +bool ProtoStreamObjectWriter::IsAny(const google::protobuf::Field& field) { + return GetTypeWithoutUrl(field.type_url()) == kAnyType; } +bool ProtoStreamObjectWriter::IsStruct(const google::protobuf::Field& field) { + return GetTypeWithoutUrl(field.type_url()) == kStructType; +} + +bool ProtoStreamObjectWriter::IsStructValue( + const google::protobuf::Field& field) { + return GetTypeWithoutUrl(field.type_url()) == kStructValueType; +} + +bool ProtoStreamObjectWriter::IsStructListValue( + const google::protobuf::Field& field) { + return GetTypeWithoutUrl(field.type_url()) == kStructListValueType; +} } // namespace converter } // namespace util diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.h b/src/google/protobuf/util/internal/protostream_objectwriter.h index 6e133679..08ac6e33 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.h +++ b/src/google/protobuf/util/internal/protostream_objectwriter.h @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -67,9 +68,11 @@ namespace converter { class ObjectLocationTracker; // An ObjectWriter that can write protobuf bytes directly from writer events. +// This class supports all special types like Struct and Map. It uses +// the ProtoWriter class to write raw proto bytes. // // It also supports streaming. -class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter { +class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { public: // Constructor. Does not take ownership of any parameter passed in. ProtoStreamObjectWriter(TypeResolver* type_resolver, @@ -82,56 +85,13 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter virtual ProtoStreamObjectWriter* EndObject(); virtual ProtoStreamObjectWriter* StartList(StringPiece name); virtual ProtoStreamObjectWriter* EndList(); - virtual ProtoStreamObjectWriter* RenderBool(StringPiece name, bool value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderInt32(StringPiece name, int32 value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderUint32(StringPiece name, - uint32 value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderInt64(StringPiece name, int64 value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderUint64(StringPiece name, - uint64 value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderDouble(StringPiece name, - double value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderFloat(StringPiece name, float value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderString(StringPiece name, - StringPiece value) { - return RenderDataPiece(name, DataPiece(value)); - } - virtual ProtoStreamObjectWriter* RenderBytes(StringPiece name, - StringPiece value) { - return RenderDataPiece(name, DataPiece(value, false)); - } - virtual ProtoStreamObjectWriter* RenderNull(StringPiece name) { - return RenderDataPiece(name, DataPiece::NullData()); - } // Renders a DataPiece 'value' into a field whose wire type is determined // from the given field 'name'. - ProtoStreamObjectWriter* RenderDataPiece(StringPiece name, - const DataPiece& value); + virtual ProtoStreamObjectWriter* RenderDataPiece(StringPiece name, + const DataPiece& value); - // Returns the location tracker to use for tracking locations for errors. - const LocationTrackerInterface& location() { - return element_ != NULL ? *element_ : *tracker_; - } - - // When true, we finished writing to output a complete message. - bool done() const { return done_; } - - private: + protected: // Function that renders a well known type with modified behavior. typedef util::Status (*TypeRenderer)(ProtoStreamObjectWriter*, const DataPiece&); @@ -192,73 +152,37 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter bool has_injected_value_message_; }; - class LIBPROTOBUF_EXPORT ProtoElement : public BaseElement, public LocationTrackerInterface { + // Represents an item in a stack of items used to keep state between + // ObjectWrier events. + class LIBPROTOBUF_EXPORT Item : public BaseElement { public: - // Indicates the type of element. Special types like LIST, MAP, MAP_ENTRY, - // STRUCT etc. are used to deduce other information based on their position - // on the stack of elements. - enum ElementType { - MESSAGE, // Simple message - LIST, // List/repeated element - MAP, // Proto3 map type - MAP_ENTRY, // Proto3 map message type, with 'key' and 'value' fields - ANY, // Proto3 Any type - STRUCT, // Proto3 struct type - STRUCT_VALUE, // Struct's Value message type - STRUCT_LIST, // List type indicator within a struct - STRUCT_LIST_VALUE, // Struct Value's ListValue message type - STRUCT_MAP, // Struct within a struct type - STRUCT_MAP_ENTRY // Struct map's entry type with 'key' and 'value' - // fields + // Indicates the type of item. + enum ItemType { + MESSAGE, // Simple message + MAP, // Proto3 map type + ANY, // Proto3 Any type }; - // Constructor for the root element. No parent nor field. - ProtoElement(const TypeInfo* typeinfo, const google::protobuf::Type& type, - ProtoStreamObjectWriter* enclosing); - - // Constructor for a field of an element. - ProtoElement(ProtoElement* parent, const google::protobuf::Field* field, - const google::protobuf::Type& type, ElementType element_type); + // Constructor for the root item. + Item(ProtoStreamObjectWriter* enclosing, ItemType item_type, + bool is_placeholder, bool is_list); - virtual ~ProtoElement() {} + // Constructor for a field of a message. + Item(Item* parent, ItemType item_type, bool is_placeholder, bool is_list); - // Called just before the destructor for clean up: - // - reports any missing required fields - // - computes the space needed by the size field, and augment the - // length of all parent messages by this additional space. - // - releases and returns the parent pointer. - ProtoElement* pop(); - - // Accessors - const google::protobuf::Field* field() const { return field_; } - const google::protobuf::Type& type() const { return type_; } + virtual ~Item() {} // These functions return true if the element type is corresponding to the // type in function name. - bool IsMap() { return element_type_ == MAP; } - bool IsStructMap() { return element_type_ == STRUCT_MAP; } - bool IsStructMapEntry() { return element_type_ == STRUCT_MAP_ENTRY; } - bool IsStructList() { return element_type_ == STRUCT_LIST; } - bool IsAny() { return element_type_ == ANY; } - - ElementType element_type() { return element_type_; } - - void RegisterField(const google::protobuf::Field* field); - virtual string ToString() const; + bool IsMap() { return item_type_ == MAP; } + bool IsAny() { return item_type_ == ANY; } AnyWriter* any() const { return any_.get(); } - virtual ProtoElement* parent() const { - return static_cast(BaseElement::parent()); + virtual Item* parent() const { + return static_cast(BaseElement::parent()); } - // Returns true if the index is already taken by a preceeding oneof input. - bool OneofIndexTaken(int32 index); - - // Marks the oneof 'index' as taken. Future inputs to this oneof will - // generate an error. - void TakeOneofIndex(int32 index); - // Inserts map key into hash set if and only if the key did NOT already // exist in hash set. // The hash set (map_keys_) is ONLY used to keep track of map keys. @@ -266,6 +190,9 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter // already present. bool InsertMapKeyIfNotPresent(StringPiece map_key); + bool is_placeholder() const { return is_placeholder_; } + bool is_list() const { return is_list_; } + private: // Used for access to variables of the enclosing instance of // ProtoStreamObjectWriter. @@ -274,126 +201,42 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter // A writer for Any objects, handles all Any-related nonsense. google::protobuf::scoped_ptr any_; - // Describes the element as a field in the parent message. - // field_ is NULL if and only if this element is the root element. - const google::protobuf::Field* field_; - - // TypeInfo to lookup types. - const TypeInfo* typeinfo_; - - // Additional variables if this element is a message: - // (Root element is always a message). - // descriptor_ : describes allowed fields in the message. - // required_fields_: set of required fields. - // is_repeated_type_ : true if the element is of type list or map. - // size_index_ : index into ProtoStreamObjectWriter::size_insert_ - // for later insertion of serialized message length. - const google::protobuf::Type& type_; - std::set required_fields_; - const bool is_repeated_type_; - const int size_index_; - - // Tracks position in repeated fields, needed for LocationTrackerInterface. - int array_index_; - // The type of this element, see enum for permissible types. - ElementType element_type_; - - // Set of oneof indices already seen for the type_. Used to validate - // incoming messages so no more than one oneof is set. - hash_set oneof_indices_; + ItemType item_type_; // Set of map keys already seen for the type_. Used to validate incoming // messages so no map key appears more than once. hash_set map_keys_; - GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); - }; + // Conveys whether this Item is a placeholder or not. Placeholder items are + // pushed to stack to account for special types. + bool is_placeholder_; - // Container for inserting 'size' information at the 'pos' position. - struct SizeInfo { - const int pos; - int size; + // Conveys whether this Item is a list or not. This is used to send + // StartList or EndList calls to underlying ObjectWriter. + bool is_list_; + + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(Item); }; ProtoStreamObjectWriter(const TypeInfo* typeinfo, const google::protobuf::Type& type, strings::ByteSink* output, ErrorListener* listener); - ProtoElement* element() { return element_.get(); } - - // Helper methods for calling ErrorListener. See error_listener.h. - void InvalidName(StringPiece unknown_name, StringPiece message); - void InvalidValue(StringPiece type_name, StringPiece value); - void MissingField(StringPiece missing_name); - - // Common code for BeginObject() and BeginList() that does invalid_depth_ - // bookkeeping associated with name lookup. - const google::protobuf::Field* BeginNamed(StringPiece name, bool is_list); - - // Lookup the field in the current element. Looks in the base descriptor - // and in any extension. This will report an error if the field cannot be - // found or if multiple matching extensions are found. - const google::protobuf::Field* Lookup(StringPiece name); - - // Lookup the field type in the type descriptor. Returns NULL if the type - // is not known. - const google::protobuf::Type* LookupType( - const google::protobuf::Field* field); - - // Looks up the oneof struct Value field depending on the type. - // On failure to find, it returns an appropriate error. - util::StatusOr LookupStructField( - DataPiece::Type type); - - // Starts an entry in map. This will be called after placing map element at - // the top of the stack. Uses this information to write map entries. - const google::protobuf::Field* StartMapEntry(StringPiece name); - - // Starts a google.protobuf.Struct. - // 'field' is of type google.protobuf.Struct. - // If field is NULL, it indicates that the top-level message is a struct - // type. - void StartStruct(const google::protobuf::Field* field); - - // Starts another struct within a struct. - // 'field' is of type google.protobuf.Value (see struct.proto). - const google::protobuf::Field* StartStructValueInStruct( - const google::protobuf::Field* field); - - // Starts a list within a struct. - // 'field' is of type google.protobuf.ListValue (see struct.proto). - const google::protobuf::Field* StartListValueInStruct( - const google::protobuf::Field* field); - - // Starts the repeated "values" field in struct.proto's - // google.protobuf.ListValue type. 'field' should be of type - // google.protobuf.ListValue. - const google::protobuf::Field* StartRepeatedValuesInListValue( - const google::protobuf::Field* field); - - // Pops sentinel elements off the stack. - void SkipElements(); - - // Write serialized output to the final output ByteSink, inserting all - // the size information for nested messages that are missing from the - // intermediate Cord buffer. - void WriteRootMessage(); - // Returns true if the field is a map. bool IsMap(const google::protobuf::Field& field); // Returns true if the field is an any. bool IsAny(const google::protobuf::Field& field); - // Helper method to write proto tags based on the given field. - void WriteTag(const google::protobuf::Field& field); + // Returns true if the field is google.protobuf.Struct. + bool IsStruct(const google::protobuf::Field& field); + // Returns true if the field is google.protobuf.Value. + bool IsStructValue(const google::protobuf::Field& field); - // Helper function to render primitive data types in DataPiece. - void RenderSimpleDataPiece(const google::protobuf::Field& field, - const google::protobuf::Type& type, - const DataPiece& data); + // Returns true if the field is google.protobuf.ListValue. + bool IsStructListValue(const google::protobuf::Field& field); // Renders google.protobuf.Value in struct.proto. It picks the right oneof // type based on value's type. @@ -417,69 +260,46 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public StructuredObjectWriter static util::Status RenderWrapperType(ProtoStreamObjectWriter* ow, const DataPiece& value); - // Helper functions to create the map and find functions responsible for - // rendering well known types, keyed by type URL. - static hash_map* renderers_; static void InitRendererMap(); static void DeleteRendererMap(); static TypeRenderer* FindTypeRenderer(const string& type_url); - // Returns the ProtoElement::ElementType for the given Type. - static ProtoElement::ElementType GetElementType( - const google::protobuf::Type& type); - - // Returns true if the field for type_ can be set as a oneof. If field is not - // a oneof type, this function does nothing and returns true. - // If another field for this oneof is already set, this function returns - // false. It also calls the appropriate error callback. - // unnormalized_name is used for error string. - bool ValidOneof(const google::protobuf::Field& field, - StringPiece unnormalized_name); - // Returns true if the map key for type_ is not duplicated key. // If map key is duplicated key, this function returns false. - // Note that caller should make sure that the current proto element (element_) + // Note that caller should make sure that the current proto element (current_) // is of element type MAP or STRUCT_MAP. // It also calls the appropriate error callback and unnormalzied_name is used // for error string. bool ValidMapKey(StringPiece unnormalized_name); + // Pushes an item on to the stack. Also calls either StartObject or StartList + // on the underlying ObjectWriter depending on whether is_list is false or + // not. + // is_placeholder conveys whether the item is a placeholder item or not. + // Placeholder items are pushed when adding auxillary types' StartObject or + // StartList calls. + void Push(StringPiece name, Item::ItemType item_type, bool is_placeholder, + bool is_list); + + // Pops items from the stack. All placeholder items are popped until a + // non-placeholder item is found. + void Pop(); + + // Pops one element from the stack. Calls EndObject() or EndList() on the + // underlying ObjectWriter depending on the value of is_list_. + void PopOneElement(); + + private: + // Helper functions to create the map and find functions responsible for + // rendering well known types, keyed by type URL. + static hash_map* renderers_; + // Variables for describing the structure of the input tree: // master_type_: descriptor for the whole protobuf message. - // typeinfo_ : the TypeInfo object to lookup types. const google::protobuf::Type& master_type_; - const TypeInfo* typeinfo_; - // Whether we own the typeinfo_ object. - bool own_typeinfo_; - - // Indicates whether we finished writing root message completely. - bool done_; - - // Variable for internal state processing: - // element_ : the current element. - // size_insert_: sizes of nested messages. - // pos - position to insert the size field. - // size - size value to be inserted. - google::protobuf::scoped_ptr element_; - std::deque size_insert_; - - // Variables for output generation: - // output_ : pointer to an external ByteSink for final user-visible output. - // buffer_ : buffer holding partial message before being ready for output_. - // adapter_ : internal adapter between CodedOutputStream and Cord buffer_. - // stream_ : wrapper for writing tags and other encodings in wire format. - strings::ByteSink* output_; - string buffer_; - google::protobuf::io::StringOutputStream adapter_; - google::protobuf::scoped_ptr stream_; - - // Variables for error tracking and reporting: - // listener_ : a place to report any errors found. - // invalid_depth_: number of enclosing invalid nested messages. - // tracker_ : the root location tracker interface. - ErrorListener* listener_; - int invalid_depth_; - google::protobuf::scoped_ptr tracker_; + + // The current element, variable for internal state processing. + google::protobuf::scoped_ptr current_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoStreamObjectWriter); }; diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index cb9f97f7..c7667c21 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -241,6 +241,21 @@ TEST_P(ProtoStreamObjectWriterTest, SimpleMessage) { CheckOutput(book); } +TEST_P(ProtoStreamObjectWriterTest, CustomJsonName) { + Book book; + Author* robert = book.mutable_author(); + robert->set_id(12345); + robert->set_name("robert"); + + ow_->StartObject("") + ->StartObject("author") + ->RenderUint64("@id", 12345) + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + TEST_P(ProtoStreamObjectWriterTest, PrimitiveFromStringConversion) { Primitive full; full.set_fix32(101); @@ -796,10 +811,6 @@ TEST_P(ProtoStreamObjectWriterTest, RootNamedList) { InvalidName(_, StringPiece("oops"), StringPiece("Root element should not be named."))) .With(Args<0>(HasObjectLocation(""))); - EXPECT_CALL(listener_, - InvalidName(_, StringPiece(""), - StringPiece("Proto fields must have a name."))) - .With(Args<0>(HasObjectLocation(""))); ow_->StartList("oops")->RenderString("", "item")->EndList(); CheckOutput(empty, 0); } @@ -864,6 +875,18 @@ INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, ::testing::Values( testing::USE_TYPE_RESOLVER)); +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, ParseTimestamp) { + TimestampDuration timestamp; + google::protobuf::Timestamp* ts = timestamp.mutable_ts(); + ts->set_seconds(1448249855); + ts->set_nanos(33155000); + + ow_->StartObject("") + ->RenderString("ts", "2015-11-23T03:37:35.033155Z") + ->EndObject(); + CheckOutput(timestamp); +} + TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError1) { TimestampDuration timestamp; @@ -922,9 +945,66 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError4) { CheckOutput(timestamp); } -// TODO(skarvaje): Write a test for nanos that exceed limit. Currently, it is -// not possible to construct a test case where nanos exceed limit because of -// floating point arithmetic. +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError5) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2015-11-23T03:37:35.033155 Z"))); + + ow_->StartObject("") + // Whitespace in the Timestamp nanos is not allowed. + ->RenderString("ts", "2015-11-23T03:37:35.033155 Z") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError6) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2015-11-23T03:37:35.033155 1234Z"))); + + ow_->StartObject("") + // Whitespace in the Timestamp nanos is not allowed. + ->RenderString("ts", "2015-11-23T03:37:35.033155 1234Z") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError7) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2015-11-23T03:37:35.033abc155Z"))); + + ow_->StartObject("") + // Non-numeric characters in the Timestamp nanos is not allowed. + ->RenderString("ts", "2015-11-23T03:37:35.033abc155Z") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, ParseDuration) { + TimestampDuration duration; + google::protobuf::Duration* dur = duration.mutable_dur(); + dur->set_seconds(1448216930); + dur->set_nanos(132262000); + + ow_->StartObject("")->RenderString("dur", "1448216930.132262s")->EndObject(); + CheckOutput(duration); +} TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidDurationError1) { TimestampDuration duration; @@ -962,7 +1042,7 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidDurationError3) { InvalidValue( _, StringPiece("type.googleapis.com/google.protobuf.Duration"), StringPiece("Field 'dur', Invalid duration format, failed to " - "parse nanos seconds"))); + "parse nano seconds"))); ow_->StartObject("")->RenderString("dur", "123.DEFs")->EndObject(); CheckOutput(duration); @@ -981,6 +1061,19 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidDurationError4) { CheckOutput(duration); } +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidDurationError5) { + TimestampDuration duration; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Duration"), + StringPiece("Field 'dur', Duration value exceeds limits"))); + + ow_->StartObject("")->RenderString("dur", "0.1000000001s")->EndObject(); + CheckOutput(duration); +} + TEST_P(ProtoStreamObjectWriterTimestampDurationTest, MismatchedTimestampTypeInput) { TimestampDuration timestamp; @@ -1066,12 +1159,12 @@ TEST_P(ProtoStreamObjectWriterStructTest, StructInvalidInputFailure) { TEST_P(ProtoStreamObjectWriterStructTest, SimpleRepeatedStructMapKeyTest) { EXPECT_CALL( listener_, - InvalidName(_, StringPiece("k1"), - StringPiece("Repeated map key: 'k1' is already set."))); + InvalidName(_, StringPiece("gBike"), + StringPiece("Repeated map key: 'gBike' is already set."))); ow_->StartObject("") ->StartObject("object") - ->RenderString("k1", "v1") - ->RenderString("k1", "v2") + ->RenderString("gBike", "v1") + ->RenderString("gBike", "v2") ->EndObject() ->EndObject(); } @@ -1121,10 +1214,11 @@ INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, TEST_P(ProtoStreamObjectWriterMapTest, MapShouldNotAcceptList) { MapIn mm; - EXPECT_CALL(listener_, - InvalidValue(_, StringPiece("Map"), - StringPiece("Cannot bind a list to map."))) - .With(Args<0>(HasObjectLocation("map_input"))); + EXPECT_CALL( + listener_, + InvalidValue( + _, StringPiece("Map"), + StringPiece("Cannot bind a list to map for field 'map_input'."))); ow_->StartObject("") ->StartList("map_input") ->RenderString("a", "b") diff --git a/src/google/protobuf/util/internal/testdata/books.proto b/src/google/protobuf/util/internal/testdata/books.proto index 6e2f109b..82b81760 100644 --- a/src/google/protobuf/util/internal/testdata/books.proto +++ b/src/google/protobuf/util/internal/testdata/books.proto @@ -66,7 +66,7 @@ message Publisher { // An author of a book message Author { - optional uint64 id = 1; + optional uint64 id = 1 [json_name = "@id"]; optional string name = 2; repeated string pseudonym = 3; optional bool alive = 4; diff --git a/src/google/protobuf/util/internal/testdata/default_value.proto b/src/google/protobuf/util/internal/testdata/default_value.proto index ebbdf6ab..cccc741c 100644 --- a/src/google/protobuf/util/internal/testdata/default_value.proto +++ b/src/google/protobuf/util/internal/testdata/default_value.proto @@ -75,9 +75,10 @@ message DefaultValueTestCases { IntToStringMap int_to_string = 403; MixedMap mixed1 = 404; MixedMap2 mixed2 = 405; - MessageMap map_of_objects = 406; - MixedMap mixed_empty = 407; - MessageMap message_map_empty = 408; + MixedMap2 empty_mixed2 = 406; + MessageMap map_of_objects = 407; + MixedMap mixed_empty = 408; + MessageMap message_map_empty = 409; DoubleValueMessage double_value = 501; DoubleValueMessage double_value_default = 502; } diff --git a/src/google/protobuf/util/internal/testdata/default_value_test.proto b/src/google/protobuf/util/internal/testdata/default_value_test.proto index 21b85e6d..93288341 100644 --- a/src/google/protobuf/util/internal/testdata/default_value_test.proto +++ b/src/google/protobuf/util/internal/testdata/default_value_test.proto @@ -43,4 +43,11 @@ message DefaultValueTest { bool bool_value = 13; string string_value = 15; bytes bytes_value = 17 [ctype = CORD]; + + enum EnumDefault { + ENUM_FIRST = 0; + ENUM_SECOND = 1; + ENUM_THIRD = 2; + } + EnumDefault enum_value = 18; } diff --git a/src/google/protobuf/util/internal/type_info.cc b/src/google/protobuf/util/internal/type_info.cc index a45a76e3..00a8ee7a 100644 --- a/src/google/protobuf/util/internal/type_info.cc +++ b/src/google/protobuf/util/internal/type_info.cc @@ -136,8 +136,7 @@ class TypeInfoForTypeResolver : public TypeInfo { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); StringPiece name = field.name(); - StringPiece camel_case_name = - *string_storage_.insert(ToCamelCase(name)).first; + StringPiece camel_case_name = field.json_name(); const StringPiece* existing = InsertOrReturnExisting( &camel_case_name_table_, camel_case_name, name); if (existing && *existing != name) { diff --git a/src/google/protobuf/util/internal/utility.cc b/src/google/protobuf/util/internal/utility.cc index 61899c24..1ddf2487 100644 --- a/src/google/protobuf/util/internal/utility.cc +++ b/src/google/protobuf/util/internal/utility.cc @@ -49,7 +49,8 @@ namespace converter { namespace { const StringPiece SkipWhiteSpace(StringPiece str) { StringPiece::size_type i; - for (i = 0; i < str.size() && isspace(str[i]); ++i) {} + for (i = 0; i < str.size() && isspace(str[i]); ++i) { + } GOOGLE_DCHECK(i == str.size() || !isspace(str[i])); return StringPiece(str, i); } @@ -160,6 +161,19 @@ const google::protobuf::Field* FindFieldInTypeOrNull( return NULL; } +const google::protobuf::Field* FindJsonFieldInTypeOrNull( + const google::protobuf::Type* type, StringPiece json_name) { + if (type != NULL) { + for (int i = 0; i < type->fields_size(); ++i) { + const google::protobuf::Field& field = type->fields(i); + if (field.json_name() == json_name) { + return &field; + } + } + } + return NULL; +} + const google::protobuf::EnumValue* FindEnumValueByNameOrNull( const google::protobuf::Enum* enum_type, StringPiece enum_name) { if (enum_type != NULL) { @@ -316,16 +330,23 @@ string FloatAsString(float value) { return DoubleAsString(value); } -bool SafeStrToFloat(StringPiece str, float *value) { +bool SafeStrToFloat(StringPiece str, float* value) { double double_value; if (!safe_strtod(str, &double_value)) { return false; } - *value = static_cast(double_value); - if (MathLimits::IsInf(*value)) { + if (MathLimits::IsInf(double_value) || + MathLimits::IsNaN(double_value)) + return false; + + // Fail if the value is not representable in float. + if (double_value > std::numeric_limits::max() || + double_value < -std::numeric_limits::max()) { return false; } + + *value = static_cast(double_value); return true; } diff --git a/src/google/protobuf/util/internal/utility.h b/src/google/protobuf/util/internal/utility.h index 5ba97bd2..33df8eda 100644 --- a/src/google/protobuf/util/internal/utility.h +++ b/src/google/protobuf/util/internal/utility.h @@ -127,6 +127,11 @@ const google::protobuf::Option* FindOptionOrNull( const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece field_name); +// Similar to FindFieldInTypeOrNull, but this looks up fields with given +// json_name. +const google::protobuf::Field* FindJsonFieldInTypeOrNull( + const google::protobuf::Type* type, StringPiece json_name); + // Finds and returns the EnumValue identified by enum_name in the passed tech // Enum object. Returns NULL if none found. const google::protobuf::EnumValue* FindEnumValueByNameOrNull( diff --git a/src/google/protobuf/util/json_format_proto3.proto b/src/google/protobuf/util/json_format_proto3.proto index e8137677..a1e24c18 100644 --- a/src/google/protobuf/util/json_format_proto3.proto +++ b/src/google/protobuf/util/json_format_proto3.proto @@ -165,3 +165,12 @@ message TestListValue { google.protobuf.ListValue value = 1; repeated google.protobuf.ListValue repeated_value = 2; } + +message TestBoolValue { + bool bool_value = 1; + map bool_map = 2; +} + +message TestCustomJsonName { + int32 value = 1 [json_name = "@value"]; +} diff --git a/src/google/protobuf/util/message_differencer.cc b/src/google/protobuf/util/message_differencer.cc index 47237e5a..0f879dc7 100644 --- a/src/google/protobuf/util/message_differencer.cc +++ b/src/google/protobuf/util/message_differencer.cc @@ -238,9 +238,25 @@ void MessageDifferencer::TreatAsSet(const FieldDescriptor* field) { GOOGLE_CHECK(key_comparator == NULL) << "Cannot treat this repeated field as both Map and Set for" << " comparison. Field name is: " << field->full_name(); + GOOGLE_CHECK(list_fields_.find(field) == list_fields_.end()) + << "Cannot treat the same field as both SET and LIST. Field name is: " + << field->full_name(); set_fields_.insert(field); } +void MessageDifferencer::TreatAsList(const FieldDescriptor* field) { + GOOGLE_CHECK(field->is_repeated()) << "Field must be repeated: " + << field->full_name(); + const MapKeyComparator* key_comparator = GetMapKeyComparator(field); + GOOGLE_CHECK(key_comparator == NULL) + << "Cannot treat this repeated field as both Map and Set for" + << " comparison. Field name is: " << field->full_name(); + GOOGLE_CHECK(set_fields_.find(field) == set_fields_.end()) + << "Cannot treat the same field as both SET and LIST. Field name is: " + << field->full_name(); + list_fields_.insert(field); +} + void MessageDifferencer::TreatAsMap(const FieldDescriptor* field, const FieldDescriptor* key) { GOOGLE_CHECK(field->is_repeated()) << "Field must be repeated: " @@ -255,6 +271,9 @@ void MessageDifferencer::TreatAsMap(const FieldDescriptor* field, GOOGLE_CHECK(set_fields_.find(field) == set_fields_.end()) << "Cannot treat this repeated field as both Map and Set for " << "comparison."; + GOOGLE_CHECK(list_fields_.find(field) == list_fields_.end()) + << "Cannot treat this repeated field as both Map and List for " + << "comparison."; MapKeyComparator* key_comparator = new MultipleFieldsMapKeyComparator(this, key); owned_key_comparators_.push_back(key_comparator); @@ -920,7 +939,8 @@ bool MessageDifferencer::CheckPathChanged( bool MessageDifferencer::IsTreatedAsSet(const FieldDescriptor* field) { if (!field->is_repeated()) return false; if (field->is_map()) return true; - if (repeated_field_comparison_ == AS_SET) return true; + if (repeated_field_comparison_ == AS_SET) + return list_fields_.find(field) == list_fields_.end(); return (set_fields_.find(field) != set_fields_.end()); } diff --git a/src/google/protobuf/util/message_differencer.h b/src/google/protobuf/util/message_differencer.h index 34c173db..3ea74e67 100644 --- a/src/google/protobuf/util/message_differencer.h +++ b/src/google/protobuf/util/message_differencer.h @@ -397,9 +397,16 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // + n^3) in which n^3 is the time complexity of the maximum matching // algorithm. // - // REQUIRES: field->is_repeated() + // REQUIRES: field->is_repeated() and field not registered with TreatAsList void TreatAsSet(const FieldDescriptor* field); + // The elements of the given repeated field will be treated as a list for + // diffing purposes, so different orderings of the same elements will NOT be + // considered equal. + // + // REQUIRED: field->is_repeated() and field not registered with TreatAsSet + void TreatAsList(const FieldDescriptor* field); + // The elements of the given repeated field will be treated as a map for // diffing purposes, with |key| being the map key. Thus, elements with the // same key will be compared even if they do not appear at the same index. @@ -791,6 +798,7 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { RepeatedFieldComparison repeated_field_comparison_; FieldSet set_fields_; + FieldSet list_fields_; // Keeps track of MapKeyComparators that are created within // MessageDifferencer. These MapKeyComparators should be deleted // before MessageDifferencer is destroyed. diff --git a/src/google/protobuf/util/message_differencer_unittest.cc b/src/google/protobuf/util/message_differencer_unittest.cc index 701b94ae..4e9ca348 100755 --- a/src/google/protobuf/util/message_differencer_unittest.cc +++ b/src/google/protobuf/util/message_differencer_unittest.cc @@ -1103,12 +1103,19 @@ TEST(MessageDifferencerTest, RepeatedFieldSetTest_Combination) { msg1.add_rw("change"); msg2.add_rw("change"); // Compare - util::MessageDifferencer differencer; - differencer.TreatAsMap(msg1.GetDescriptor()->FindFieldByName("item"), - item->GetDescriptor()->FindFieldByName("a")); - differencer.TreatAsSet(msg1.GetDescriptor()->FindFieldByName("rv")); - differencer.TreatAsSet(item->GetDescriptor()->FindFieldByName("ra")); - EXPECT_TRUE(differencer.Compare(msg1, msg2)); + util::MessageDifferencer differencer1; + differencer1.TreatAsMap(msg1.GetDescriptor()->FindFieldByName("item"), + item->GetDescriptor()->FindFieldByName("a")); + differencer1.TreatAsSet(msg1.GetDescriptor()->FindFieldByName("rv")); + differencer1.TreatAsSet(item->GetDescriptor()->FindFieldByName("ra")); + EXPECT_TRUE(differencer1.Compare(msg1, msg2)); + + util::MessageDifferencer differencer2; + differencer2.TreatAsMap(msg1.GetDescriptor()->FindFieldByName("item"), + item->GetDescriptor()->FindFieldByName("a")); + differencer2.set_repeated_field_comparison(util::MessageDifferencer::AS_SET); + differencer2.TreatAsList(msg1.GetDescriptor()->FindFieldByName("rw")); + EXPECT_TRUE(differencer2.Compare(msg1, msg2)); } TEST(MessageDifferencerTest, RepeatedFieldMapTest_Partial) { @@ -1168,6 +1175,11 @@ TEST(MessageDifferencerTest, RepeatedFieldSetTest_Duplicates) { differencer.TreatAsSet(GetFieldDescriptor(a, "rv")); EXPECT_TRUE(differencer.Compare(b, a)); EXPECT_FALSE(differencer.Compare(c, a)); + + util::MessageDifferencer differencer1; + differencer1.set_repeated_field_comparison(util::MessageDifferencer::AS_SET); + EXPECT_TRUE(differencer1.Compare(b, a)); + EXPECT_FALSE(differencer1.Compare(c, a)); } TEST(MessageDifferencerTest, RepeatedFieldSetTest_PartialSimple) { diff --git a/src/google/protobuf/util/type_resolver_util.cc b/src/google/protobuf/util/type_resolver_util.cc index a0996954..96393903 100644 --- a/src/google/protobuf/util/type_resolver_util.cc +++ b/src/google/protobuf/util/type_resolver_util.cc @@ -159,7 +159,10 @@ class DescriptorPoolTypeResolver : public TypeResolver { } field->set_number(descriptor->number()); field->set_name(descriptor->name()); - field->set_json_name(converter::ToCamelCase(descriptor->name())); + field->set_json_name(descriptor->json_name()); + if (descriptor->has_default_value()) { + field->set_default_value(DefaultValueAsString(descriptor)); + } if (descriptor->type() == FieldDescriptor::TYPE_MESSAGE) { field->set_type_url(GetTypeUrl(descriptor->message_type())); } else if (descriptor->type() == FieldDescriptor::TYPE_ENUM) { @@ -200,6 +203,46 @@ class DescriptorPoolTypeResolver : public TypeResolver { return url_prefix_ + "/" + descriptor->full_name(); } + string DefaultValueAsString(const FieldDescriptor* descriptor) { + switch (descriptor->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: + return SimpleItoa(descriptor->default_value_int32()); + break; + case FieldDescriptor::CPPTYPE_INT64: + return SimpleItoa(descriptor->default_value_int64()); + break; + case FieldDescriptor::CPPTYPE_UINT32: + return SimpleItoa(descriptor->default_value_uint32()); + break; + case FieldDescriptor::CPPTYPE_UINT64: + return SimpleItoa(descriptor->default_value_uint64()); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + return SimpleFtoa(descriptor->default_value_float()); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + return SimpleDtoa(descriptor->default_value_double()); + break; + case FieldDescriptor::CPPTYPE_BOOL: + return descriptor->default_value_bool() ? "true" : "false"; + break; + case FieldDescriptor::CPPTYPE_STRING: + if (descriptor->type() == FieldDescriptor::TYPE_BYTES) { + return CEscape(descriptor->default_value_string()); + } else { + return descriptor->default_value_string(); + } + break; + case FieldDescriptor::CPPTYPE_ENUM: + return descriptor->default_value_enum()->name(); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + GOOGLE_LOG(DFATAL) << "Messages can't have default values!"; + break; + } + return ""; + } + string url_prefix_; const DescriptorPool* pool_; }; diff --git a/src/google/protobuf/util/type_resolver_util_test.cc b/src/google/protobuf/util/type_resolver_util_test.cc index 74b2d0da..8a0bf652 100644 --- a/src/google/protobuf/util/type_resolver_util_test.cc +++ b/src/google/protobuf/util/type_resolver_util_test.cc @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -332,6 +333,19 @@ TEST_F(DescriptorPoolTypeResolverTest, TestEnum) { EnumHasValue(type, "NEG", -1); } +TEST_F(DescriptorPoolTypeResolverTest, TestJsonName) { + Type type; + ASSERT_TRUE(resolver_->ResolveMessageType( + GetTypeUrl(), &type) + .ok()); + EXPECT_EQ("optionalInt32", FindField(type, "optional_int32")->json_name()); + + ASSERT_TRUE(resolver_->ResolveMessageType( + GetTypeUrl(), &type) + .ok()); + EXPECT_EQ("@value", FindField(type, "value")->json_name()); +} + } // namespace } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/wrappers.pb.cc b/src/google/protobuf/wrappers.pb.cc index e153687b..212dd219 100644 --- a/src/google/protobuf/wrappers.pb.cc +++ b/src/google/protobuf/wrappers.pb.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -262,10 +263,10 @@ void protobuf_AddDesc_google_2fprotobuf_2fwrappers_2eproto() { "e\030\001 \001(\004\"\033\n\nInt32Value\022\r\n\005value\030\001 \001(\005\"\034\n\013" "UInt32Value\022\r\n\005value\030\001 \001(\r\"\032\n\tBoolValue\022" "\r\n\005value\030\001 \001(\010\"\034\n\013StringValue\022\r\n\005value\030\001" - " \001(\t\"\033\n\nBytesValue\022\r\n\005value\030\001 \001(\014BP\n\023com" - ".google.protobufB\rWrappersProtoP\001\240\001\001\242\002\003G" - "PB\252\002\036Google.Protobuf.WellKnownTypesb\006pro" - "to3", 403); + " \001(\t\"\033\n\nBytesValue\022\r\n\005value\030\001 \001(\014BS\n\023com" + ".google.protobufB\rWrappersProtoP\001\240\001\001\370\001\001\242" + "\002\003GPB\252\002\036Google.Protobuf.WellKnownTypesb\006" + "proto3", 406); ::google::protobuf::MessageFactory::InternalRegisterGeneratedFile( "google/protobuf/wrappers.proto", &protobuf_RegisterTypes); DoubleValue::default_instance_ = new DoubleValue(); @@ -318,6 +319,14 @@ DoubleValue::DoubleValue() // @@protoc_insertion_point(constructor:google.protobuf.DoubleValue) } +DoubleValue::DoubleValue(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.DoubleValue) +} + void DoubleValue::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -342,10 +351,20 @@ DoubleValue::~DoubleValue() { } void DoubleValue::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void DoubleValue::ArenaDtor(void* object) { + DoubleValue* _this = reinterpret_cast< DoubleValue* >(object); + (void)_this; +} +void DoubleValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void DoubleValue::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -364,11 +383,7 @@ const DoubleValue& DoubleValue::default_instance() { DoubleValue* DoubleValue::default_instance_ = NULL; DoubleValue* DoubleValue::New(::google::protobuf::Arena* arena) const { - DoubleValue* n = new DoubleValue; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void DoubleValue::Clear() { @@ -495,6 +510,18 @@ bool DoubleValue::IsInitialized() const { void DoubleValue::Swap(DoubleValue* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + DoubleValue temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void DoubleValue::UnsafeArenaSwap(DoubleValue* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void DoubleValue::InternalSwap(DoubleValue* other) { @@ -542,6 +569,14 @@ FloatValue::FloatValue() // @@protoc_insertion_point(constructor:google.protobuf.FloatValue) } +FloatValue::FloatValue(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.FloatValue) +} + void FloatValue::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -566,10 +601,20 @@ FloatValue::~FloatValue() { } void FloatValue::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void FloatValue::ArenaDtor(void* object) { + FloatValue* _this = reinterpret_cast< FloatValue* >(object); + (void)_this; +} +void FloatValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void FloatValue::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -588,11 +633,7 @@ const FloatValue& FloatValue::default_instance() { FloatValue* FloatValue::default_instance_ = NULL; FloatValue* FloatValue::New(::google::protobuf::Arena* arena) const { - FloatValue* n = new FloatValue; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void FloatValue::Clear() { @@ -719,6 +760,18 @@ bool FloatValue::IsInitialized() const { void FloatValue::Swap(FloatValue* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + FloatValue temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void FloatValue::UnsafeArenaSwap(FloatValue* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void FloatValue::InternalSwap(FloatValue* other) { @@ -766,6 +819,14 @@ Int64Value::Int64Value() // @@protoc_insertion_point(constructor:google.protobuf.Int64Value) } +Int64Value::Int64Value(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.Int64Value) +} + void Int64Value::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -790,10 +851,20 @@ Int64Value::~Int64Value() { } void Int64Value::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void Int64Value::ArenaDtor(void* object) { + Int64Value* _this = reinterpret_cast< Int64Value* >(object); + (void)_this; +} +void Int64Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void Int64Value::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -812,11 +883,7 @@ const Int64Value& Int64Value::default_instance() { Int64Value* Int64Value::default_instance_ = NULL; Int64Value* Int64Value::New(::google::protobuf::Arena* arena) const { - Int64Value* n = new Int64Value; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void Int64Value::Clear() { @@ -945,6 +1012,18 @@ bool Int64Value::IsInitialized() const { void Int64Value::Swap(Int64Value* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + Int64Value temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void Int64Value::UnsafeArenaSwap(Int64Value* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void Int64Value::InternalSwap(Int64Value* other) { @@ -992,6 +1071,14 @@ UInt64Value::UInt64Value() // @@protoc_insertion_point(constructor:google.protobuf.UInt64Value) } +UInt64Value::UInt64Value(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.UInt64Value) +} + void UInt64Value::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -1016,10 +1103,20 @@ UInt64Value::~UInt64Value() { } void UInt64Value::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void UInt64Value::ArenaDtor(void* object) { + UInt64Value* _this = reinterpret_cast< UInt64Value* >(object); + (void)_this; +} +void UInt64Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void UInt64Value::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -1038,11 +1135,7 @@ const UInt64Value& UInt64Value::default_instance() { UInt64Value* UInt64Value::default_instance_ = NULL; UInt64Value* UInt64Value::New(::google::protobuf::Arena* arena) const { - UInt64Value* n = new UInt64Value; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void UInt64Value::Clear() { @@ -1171,6 +1264,18 @@ bool UInt64Value::IsInitialized() const { void UInt64Value::Swap(UInt64Value* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + UInt64Value temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void UInt64Value::UnsafeArenaSwap(UInt64Value* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void UInt64Value::InternalSwap(UInt64Value* other) { @@ -1218,6 +1323,14 @@ Int32Value::Int32Value() // @@protoc_insertion_point(constructor:google.protobuf.Int32Value) } +Int32Value::Int32Value(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.Int32Value) +} + void Int32Value::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -1242,10 +1355,20 @@ Int32Value::~Int32Value() { } void Int32Value::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void Int32Value::ArenaDtor(void* object) { + Int32Value* _this = reinterpret_cast< Int32Value* >(object); + (void)_this; +} +void Int32Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void Int32Value::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -1264,11 +1387,7 @@ const Int32Value& Int32Value::default_instance() { Int32Value* Int32Value::default_instance_ = NULL; Int32Value* Int32Value::New(::google::protobuf::Arena* arena) const { - Int32Value* n = new Int32Value; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void Int32Value::Clear() { @@ -1397,6 +1516,18 @@ bool Int32Value::IsInitialized() const { void Int32Value::Swap(Int32Value* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + Int32Value temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void Int32Value::UnsafeArenaSwap(Int32Value* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void Int32Value::InternalSwap(Int32Value* other) { @@ -1444,6 +1575,14 @@ UInt32Value::UInt32Value() // @@protoc_insertion_point(constructor:google.protobuf.UInt32Value) } +UInt32Value::UInt32Value(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.UInt32Value) +} + void UInt32Value::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -1468,10 +1607,20 @@ UInt32Value::~UInt32Value() { } void UInt32Value::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void UInt32Value::ArenaDtor(void* object) { + UInt32Value* _this = reinterpret_cast< UInt32Value* >(object); + (void)_this; +} +void UInt32Value::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void UInt32Value::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -1490,11 +1639,7 @@ const UInt32Value& UInt32Value::default_instance() { UInt32Value* UInt32Value::default_instance_ = NULL; UInt32Value* UInt32Value::New(::google::protobuf::Arena* arena) const { - UInt32Value* n = new UInt32Value; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void UInt32Value::Clear() { @@ -1623,6 +1768,18 @@ bool UInt32Value::IsInitialized() const { void UInt32Value::Swap(UInt32Value* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + UInt32Value temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void UInt32Value::UnsafeArenaSwap(UInt32Value* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void UInt32Value::InternalSwap(UInt32Value* other) { @@ -1670,6 +1827,14 @@ BoolValue::BoolValue() // @@protoc_insertion_point(constructor:google.protobuf.BoolValue) } +BoolValue::BoolValue(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.BoolValue) +} + void BoolValue::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -1694,10 +1859,20 @@ BoolValue::~BoolValue() { } void BoolValue::SharedDtor() { + if (GetArenaNoVirtual() != NULL) { + return; + } + if (this != default_instance_) { } } +void BoolValue::ArenaDtor(void* object) { + BoolValue* _this = reinterpret_cast< BoolValue* >(object); + (void)_this; +} +void BoolValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void BoolValue::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -1716,11 +1891,7 @@ const BoolValue& BoolValue::default_instance() { BoolValue* BoolValue::default_instance_ = NULL; BoolValue* BoolValue::New(::google::protobuf::Arena* arena) const { - BoolValue* n = new BoolValue; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void BoolValue::Clear() { @@ -1847,6 +2018,18 @@ bool BoolValue::IsInitialized() const { void BoolValue::Swap(BoolValue* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + BoolValue temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void BoolValue::UnsafeArenaSwap(BoolValue* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void BoolValue::InternalSwap(BoolValue* other) { @@ -1894,6 +2077,14 @@ StringValue::StringValue() // @@protoc_insertion_point(constructor:google.protobuf.StringValue) } +StringValue::StringValue(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.StringValue) +} + void StringValue::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -1919,11 +2110,21 @@ StringValue::~StringValue() { } void StringValue::SharedDtor() { - value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (GetArenaNoVirtual() != NULL) { + return; + } + + value_.Destroy(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); if (this != default_instance_) { } } +void StringValue::ArenaDtor(void* object) { + StringValue* _this = reinterpret_cast< StringValue* >(object); + (void)_this; +} +void StringValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void StringValue::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -1942,15 +2143,11 @@ const StringValue& StringValue::default_instance() { StringValue* StringValue::default_instance_ = NULL; StringValue* StringValue::New(::google::protobuf::Arena* arena) const { - StringValue* n = new StringValue; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void StringValue::Clear() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } bool StringValue::MergePartialFromCodedStream( @@ -2065,8 +2262,7 @@ void StringValue::MergeFrom(const ::google::protobuf::Message& from) { void StringValue::MergeFrom(const StringValue& from) { if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__); if (from.value().size() > 0) { - - value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_); + set_value(from.value()); } } @@ -2089,6 +2285,18 @@ bool StringValue::IsInitialized() const { void StringValue::Swap(StringValue* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + StringValue temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void StringValue::UnsafeArenaSwap(StringValue* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void StringValue::InternalSwap(StringValue* other) { @@ -2110,36 +2318,44 @@ void StringValue::InternalSwap(StringValue* other) { // optional string value = 1; void StringValue::clear_value() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } const ::std::string& StringValue::value() const { // @@protoc_insertion_point(field_get:google.protobuf.StringValue.value) - return value_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Get(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } void StringValue::set_value(const ::std::string& value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set:google.protobuf.StringValue.value) } void StringValue::set_value(const char* value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_char:google.protobuf.StringValue.value) } - void StringValue::set_value(const char* value, size_t size) { + void StringValue::set_value(const char* value, + size_t size) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_pointer:google.protobuf.StringValue.value) } ::std::string* StringValue::mutable_value() { // @@protoc_insertion_point(field_mutable:google.protobuf.StringValue.value) - return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } ::std::string* StringValue::release_value() { - return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Release(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); +} + ::std::string* StringValue::unsafe_arena_release_value() { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + + return value_.UnsafeArenaRelease(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); } void StringValue::set_allocated_value(::std::string* value) { if (value != NULL) { @@ -2147,7 +2363,20 @@ void StringValue::clear_value() { } else { } - value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.SetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, + GetArenaNoVirtual()); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.StringValue.value) +} + void StringValue::unsafe_arena_set_allocated_value( + ::std::string* value) { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + if (value != NULL) { + + } else { + + } + value_.UnsafeArenaSetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_allocated:google.protobuf.StringValue.value) } @@ -2165,6 +2394,14 @@ BytesValue::BytesValue() // @@protoc_insertion_point(constructor:google.protobuf.BytesValue) } +BytesValue::BytesValue(::google::protobuf::Arena* arena) + : ::google::protobuf::Message(), + _internal_metadata_(arena) { + SharedCtor(); + RegisterArenaDtor(arena); + // @@protoc_insertion_point(arena_constructor:google.protobuf.BytesValue) +} + void BytesValue::InitAsDefaultInstance() { _is_default_instance_ = true; } @@ -2190,11 +2427,21 @@ BytesValue::~BytesValue() { } void BytesValue::SharedDtor() { - value_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (GetArenaNoVirtual() != NULL) { + return; + } + + value_.Destroy(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); if (this != default_instance_) { } } +void BytesValue::ArenaDtor(void* object) { + BytesValue* _this = reinterpret_cast< BytesValue* >(object); + (void)_this; +} +void BytesValue::RegisterArenaDtor(::google::protobuf::Arena* arena) { +} void BytesValue::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; @@ -2213,15 +2460,11 @@ const BytesValue& BytesValue::default_instance() { BytesValue* BytesValue::default_instance_ = NULL; BytesValue* BytesValue::New(::google::protobuf::Arena* arena) const { - BytesValue* n = new BytesValue; - if (arena != NULL) { - arena->Own(n); - } - return n; + return ::google::protobuf::Arena::CreateMessage(arena); } void BytesValue::Clear() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } bool BytesValue::MergePartialFromCodedStream( @@ -2324,8 +2567,7 @@ void BytesValue::MergeFrom(const ::google::protobuf::Message& from) { void BytesValue::MergeFrom(const BytesValue& from) { if (GOOGLE_PREDICT_FALSE(&from == this)) MergeFromFail(__LINE__); if (from.value().size() > 0) { - - value_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.value_); + set_value(from.value()); } } @@ -2348,6 +2590,18 @@ bool BytesValue::IsInitialized() const { void BytesValue::Swap(BytesValue* other) { if (other == this) return; + if (GetArenaNoVirtual() == other->GetArenaNoVirtual()) { + InternalSwap(other); + } else { + BytesValue temp; + temp.MergeFrom(*this); + CopyFrom(*other); + other->CopyFrom(temp); + } +} +void BytesValue::UnsafeArenaSwap(BytesValue* other) { + if (other == this) return; + GOOGLE_DCHECK(GetArenaNoVirtual() == other->GetArenaNoVirtual()); InternalSwap(other); } void BytesValue::InternalSwap(BytesValue* other) { @@ -2369,36 +2623,44 @@ void BytesValue::InternalSwap(BytesValue* other) { // optional bytes value = 1; void BytesValue::clear_value() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } const ::std::string& BytesValue::value() const { // @@protoc_insertion_point(field_get:google.protobuf.BytesValue.value) - return value_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Get(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } void BytesValue::set_value(const ::std::string& value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set:google.protobuf.BytesValue.value) } void BytesValue::set_value(const char* value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_char:google.protobuf.BytesValue.value) } - void BytesValue::set_value(const void* value, size_t size) { + void BytesValue::set_value(const void* value, + size_t size) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_pointer:google.protobuf.BytesValue.value) } ::std::string* BytesValue::mutable_value() { // @@protoc_insertion_point(field_mutable:google.protobuf.BytesValue.value) - return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } ::std::string* BytesValue::release_value() { - return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Release(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); +} + ::std::string* BytesValue::unsafe_arena_release_value() { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + + return value_.UnsafeArenaRelease(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); } void BytesValue::set_allocated_value(::std::string* value) { if (value != NULL) { @@ -2406,7 +2668,20 @@ void BytesValue::clear_value() { } else { } - value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.SetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, + GetArenaNoVirtual()); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.BytesValue.value) +} + void BytesValue::unsafe_arena_set_allocated_value( + ::std::string* value) { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + if (value != NULL) { + + } else { + + } + value_.UnsafeArenaSetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_allocated:google.protobuf.BytesValue.value) } diff --git a/src/google/protobuf/wrappers.pb.h b/src/google/protobuf/wrappers.pb.h index 15bcc7a2..7dca938c 100644 --- a/src/google/protobuf/wrappers.pb.h +++ b/src/google/protobuf/wrappers.pb.h @@ -61,9 +61,14 @@ class LIBPROTOBUF_EXPORT DoubleValue : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const DoubleValue& default_instance(); + void UnsafeArenaSwap(DoubleValue* other); void Swap(DoubleValue* other); // implements Message ---------------------------------------------- @@ -90,6 +95,11 @@ class LIBPROTOBUF_EXPORT DoubleValue : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(DoubleValue* other); + protected: + explicit DoubleValue(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -115,6 +125,9 @@ class LIBPROTOBUF_EXPORT DoubleValue : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; double value_; mutable int _cached_size_; @@ -139,9 +152,14 @@ class LIBPROTOBUF_EXPORT FloatValue : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const FloatValue& default_instance(); + void UnsafeArenaSwap(FloatValue* other); void Swap(FloatValue* other); // implements Message ---------------------------------------------- @@ -168,6 +186,11 @@ class LIBPROTOBUF_EXPORT FloatValue : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(FloatValue* other); + protected: + explicit FloatValue(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -193,6 +216,9 @@ class LIBPROTOBUF_EXPORT FloatValue : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; float value_; mutable int _cached_size_; @@ -217,9 +243,14 @@ class LIBPROTOBUF_EXPORT Int64Value : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const Int64Value& default_instance(); + void UnsafeArenaSwap(Int64Value* other); void Swap(Int64Value* other); // implements Message ---------------------------------------------- @@ -246,6 +277,11 @@ class LIBPROTOBUF_EXPORT Int64Value : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(Int64Value* other); + protected: + explicit Int64Value(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -271,6 +307,9 @@ class LIBPROTOBUF_EXPORT Int64Value : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::int64 value_; mutable int _cached_size_; @@ -295,9 +334,14 @@ class LIBPROTOBUF_EXPORT UInt64Value : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const UInt64Value& default_instance(); + void UnsafeArenaSwap(UInt64Value* other); void Swap(UInt64Value* other); // implements Message ---------------------------------------------- @@ -324,6 +368,11 @@ class LIBPROTOBUF_EXPORT UInt64Value : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(UInt64Value* other); + protected: + explicit UInt64Value(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -349,6 +398,9 @@ class LIBPROTOBUF_EXPORT UInt64Value : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::uint64 value_; mutable int _cached_size_; @@ -373,9 +425,14 @@ class LIBPROTOBUF_EXPORT Int32Value : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const Int32Value& default_instance(); + void UnsafeArenaSwap(Int32Value* other); void Swap(Int32Value* other); // implements Message ---------------------------------------------- @@ -402,6 +459,11 @@ class LIBPROTOBUF_EXPORT Int32Value : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(Int32Value* other); + protected: + explicit Int32Value(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -427,6 +489,9 @@ class LIBPROTOBUF_EXPORT Int32Value : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::int32 value_; mutable int _cached_size_; @@ -451,9 +516,14 @@ class LIBPROTOBUF_EXPORT UInt32Value : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const UInt32Value& default_instance(); + void UnsafeArenaSwap(UInt32Value* other); void Swap(UInt32Value* other); // implements Message ---------------------------------------------- @@ -480,6 +550,11 @@ class LIBPROTOBUF_EXPORT UInt32Value : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(UInt32Value* other); + protected: + explicit UInt32Value(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -505,6 +580,9 @@ class LIBPROTOBUF_EXPORT UInt32Value : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::uint32 value_; mutable int _cached_size_; @@ -529,9 +607,14 @@ class LIBPROTOBUF_EXPORT BoolValue : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const BoolValue& default_instance(); + void UnsafeArenaSwap(BoolValue* other); void Swap(BoolValue* other); // implements Message ---------------------------------------------- @@ -558,6 +641,11 @@ class LIBPROTOBUF_EXPORT BoolValue : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(BoolValue* other); + protected: + explicit BoolValue(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -583,6 +671,9 @@ class LIBPROTOBUF_EXPORT BoolValue : public ::google::protobuf::Message { private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; bool value_; mutable int _cached_size_; @@ -607,9 +698,14 @@ class LIBPROTOBUF_EXPORT StringValue : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const StringValue& default_instance(); + void UnsafeArenaSwap(StringValue* other); void Swap(StringValue* other); // implements Message ---------------------------------------------- @@ -636,6 +732,11 @@ class LIBPROTOBUF_EXPORT StringValue : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(StringValue* other); + protected: + explicit StringValue(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -661,11 +762,17 @@ class LIBPROTOBUF_EXPORT StringValue : public ::google::protobuf::Message { ::std::string* mutable_value(); ::std::string* release_value(); void set_allocated_value(::std::string* value); + ::std::string* unsafe_arena_release_value(); + void unsafe_arena_set_allocated_value( + ::std::string* value); // @@protoc_insertion_point(class_scope:google.protobuf.StringValue) private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::internal::ArenaStringPtr value_; mutable int _cached_size_; @@ -690,9 +797,14 @@ class LIBPROTOBUF_EXPORT BytesValue : public ::google::protobuf::Message { return *this; } + inline ::google::protobuf::Arena* GetArena() const { return GetArenaNoVirtual(); } + inline void* GetMaybeArenaPointer() const { + return MaybeArenaPtr(); + } static const ::google::protobuf::Descriptor* descriptor(); static const BytesValue& default_instance(); + void UnsafeArenaSwap(BytesValue* other); void Swap(BytesValue* other); // implements Message ---------------------------------------------- @@ -719,6 +831,11 @@ class LIBPROTOBUF_EXPORT BytesValue : public ::google::protobuf::Message { void SharedDtor(); void SetCachedSize(int size) const; void InternalSwap(BytesValue* other); + protected: + explicit BytesValue(::google::protobuf::Arena* arena); + private: + static void ArenaDtor(void* object); + inline void RegisterArenaDtor(::google::protobuf::Arena* arena); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return _internal_metadata_.arena(); @@ -744,11 +861,17 @@ class LIBPROTOBUF_EXPORT BytesValue : public ::google::protobuf::Message { ::std::string* mutable_value(); ::std::string* release_value(); void set_allocated_value(::std::string* value); + ::std::string* unsafe_arena_release_value(); + void unsafe_arena_set_allocated_value( + ::std::string* value); // @@protoc_insertion_point(class_scope:google.protobuf.BytesValue) private: ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; + friend class ::google::protobuf::Arena; + typedef void InternalArenaConstructable_; + typedef void DestructorSkippable_; bool _is_default_instance_; ::google::protobuf::internal::ArenaStringPtr value_; mutable int _cached_size_; @@ -895,36 +1018,44 @@ inline void BoolValue::set_value(bool value) { // optional string value = 1; inline void StringValue::clear_value() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } inline const ::std::string& StringValue::value() const { // @@protoc_insertion_point(field_get:google.protobuf.StringValue.value) - return value_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Get(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } inline void StringValue::set_value(const ::std::string& value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set:google.protobuf.StringValue.value) } inline void StringValue::set_value(const char* value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_char:google.protobuf.StringValue.value) } -inline void StringValue::set_value(const char* value, size_t size) { +inline void StringValue::set_value(const char* value, + size_t size) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_pointer:google.protobuf.StringValue.value) } inline ::std::string* StringValue::mutable_value() { // @@protoc_insertion_point(field_mutable:google.protobuf.StringValue.value) - return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } inline ::std::string* StringValue::release_value() { - return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Release(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); +} +inline ::std::string* StringValue::unsafe_arena_release_value() { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + + return value_.UnsafeArenaRelease(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); } inline void StringValue::set_allocated_value(::std::string* value) { if (value != NULL) { @@ -932,7 +1063,20 @@ inline void StringValue::set_allocated_value(::std::string* value) { } else { } - value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.SetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, + GetArenaNoVirtual()); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.StringValue.value) +} +inline void StringValue::unsafe_arena_set_allocated_value( + ::std::string* value) { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + if (value != NULL) { + + } else { + + } + value_.UnsafeArenaSetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_allocated:google.protobuf.StringValue.value) } @@ -942,36 +1086,44 @@ inline void StringValue::set_allocated_value(::std::string* value) { // optional bytes value = 1; inline void BytesValue::clear_value() { - value_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + value_.ClearToEmpty(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } inline const ::std::string& BytesValue::value() const { // @@protoc_insertion_point(field_get:google.protobuf.BytesValue.value) - return value_.GetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Get(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } inline void BytesValue::set_value(const ::std::string& value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set:google.protobuf.BytesValue.value) } inline void BytesValue::set_value(const char* value) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value), + GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_char:google.protobuf.BytesValue.value) } -inline void BytesValue::set_value(const void* value, size_t size) { +inline void BytesValue::set_value(const void* value, + size_t size) { - value_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), - ::std::string(reinterpret_cast(value), size)); + value_.Set(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string( + reinterpret_cast(value), size), GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_pointer:google.protobuf.BytesValue.value) } inline ::std::string* BytesValue::mutable_value() { // @@protoc_insertion_point(field_mutable:google.protobuf.BytesValue.value) - return value_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Mutable(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); } inline ::std::string* BytesValue::release_value() { - return value_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return value_.Release(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); +} +inline ::std::string* BytesValue::unsafe_arena_release_value() { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + + return value_.UnsafeArenaRelease(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); } inline void BytesValue::set_allocated_value(::std::string* value) { if (value != NULL) { @@ -979,7 +1131,20 @@ inline void BytesValue::set_allocated_value(::std::string* value) { } else { } - value_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + value_.SetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value, + GetArenaNoVirtual()); + // @@protoc_insertion_point(field_set_allocated:google.protobuf.BytesValue.value) +} +inline void BytesValue::unsafe_arena_set_allocated_value( + ::std::string* value) { + GOOGLE_DCHECK(GetArenaNoVirtual() != NULL); + if (value != NULL) { + + } else { + + } + value_.UnsafeArenaSetAllocated(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + value, GetArenaNoVirtual()); // @@protoc_insertion_point(field_set_allocated:google.protobuf.BytesValue.value) } diff --git a/src/google/protobuf/wrappers.proto b/src/google/protobuf/wrappers.proto index a1d6e446..040d8a24 100644 --- a/src/google/protobuf/wrappers.proto +++ b/src/google/protobuf/wrappers.proto @@ -37,11 +37,12 @@ syntax = "proto3"; package google.protobuf; -option java_generate_equals_and_hash = true; -option java_multiple_files = true; -option java_outer_classname = "WrappersProto"; -option java_package = "com.google.protobuf"; option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option java_package = "com.google.protobuf"; +option java_outer_classname = "WrappersProto"; +option java_multiple_files = true; +option java_generate_equals_and_hash = true; option objc_class_prefix = "GPB"; // Wrapper message for `double`. -- cgit v1.2.3