diff options
102 files changed, 1528 insertions, 532 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..2f6d0263 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Create a report to help us improve + +--- + +**What version of protobuf and what language are you using?** +Version: master/v3.6.0/v3.5.0 etc. +Language: C++/Java/Python/C#/Ruby/PHP/Objective-C/Javascript + +**What operating system (Linux, Windows, ...) and version?** + +**What runtime / compiler are you using (e.g., python version or gcc version)** + +**What did you do?** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**What did you expect to see** + +**What did you see instead?** + +Make sure you include information that can help us debug (full error message, exception listing, stack trace, logs). + +**Anything else we should know about your project / environment** diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..722db4ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Suggest an idea for this project + +--- + +**What language does this apply to?** +If it's a proto syntax change, is it for proto2 or proto3? +If it's about generated code change, what programming language? + +**Describe the problem you are trying to solve.** + +**Describe the solution you'd like** + +**Describe alternatives you've considered** + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 00000000..bfa6ddea --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,7 @@ +--- +name: Question +about: Questions and troubleshooting + +--- + + @@ -47,7 +47,6 @@ any_test.pb.* map*unittest.pb.* unittest*.pb.* cpp_test*.pb.* -src/google/protobuf/compiler/js/well_known_types_embed.cc src/google/protobuf/util/**/*.pb.cc src/google/protobuf/util/**/*.pb.h @@ -1,4 +1,4 @@ -# Bazel (http://bazel.io/) BUILD file for Protobuf. +# Bazel (https://bazel.build/) BUILD file for Protobuf. licenses(["notice"]) @@ -22,7 +22,20 @@ config_setting( MSVC_COPTS = [ "/DHAVE_PTHREAD", "/wd4018", # -Wno-sign-compare + "/wd4065", # switch statement contains 'default' but no 'case' labels + "/wd4146", # unary minus operator applied to unsigned type, result still unsigned + "/wd4244", # 'conversion' conversion from 'type1' to 'type2', possible loss of data + "/wd4251", # 'identifier' : class 'type' needs to have dll-interface to be used by clients of class 'type2' + "/wd4267", # 'var' : conversion from 'size_t' to 'type', possible loss of data + "/wd4305", # 'identifier' : truncation from 'type1' to 'type2' + "/wd4307", # 'operator' : integral constant overflow + "/wd4309", # 'conversion' : truncation of constant value + "/wd4334", # 'operator' : result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?) + "/wd4355", # 'this' : used in base member initializer list + "/wd4506", # no definition for inline function 'function' "/wd4514", # -Wno-unused-function + "/wd4800", # 'type' : forcing value to bool 'true' or 'false' (performance warning) + "/wd4996", # The compiler encountered a deprecated declaration. ] COPTS = select({ @@ -54,7 +67,10 @@ config_setting( # Android and MSVC builds do not need to link in a separate pthread library. LINK_OPTS = select({ ":android": [], - ":msvc": [], + ":msvc": [ + # Suppress linker warnings about files with no symbols defined. + "-ignore:4221", + ], "//conditions:default": ["-lpthread", "-lm"], }) @@ -257,24 +273,6 @@ internal_copied_filegroup( # Protocol Buffers Compiler ################################################################################ -cc_binary( - name = "js_embed", - srcs = ["src/google/protobuf/compiler/js/embed.cc"], - visibility = ["//visibility:public"], -) - -genrule( - name = "generate_js_well_known_types_embed", - srcs = [ - "src/google/protobuf/compiler/js/well_known_types/any.js", - "src/google/protobuf/compiler/js/well_known_types/struct.js", - "src/google/protobuf/compiler/js/well_known_types/timestamp.js", - ], - outs = ["src/google/protobuf/compiler/js/well_known_types_embed.cc"], - cmd = "$(location :js_embed) $(SRCS) > $@", - tools = [":js_embed"], -) - cc_library( name = "protoc_lib", srcs = [ @@ -365,7 +363,25 @@ cc_library( ], copts = COPTS, includes = ["src/"], - linkopts = LINK_OPTS, + linkopts = LINK_OPTS + select({ + ":msvc": [ + # Linking to setargv.obj makes the default command line argument + # parser expand wildcards, so the main method's argv will contain the + # expanded list instead of the wildcards. + # + # Adding dummy "-DEFAULTLIB:kernel32.lib", because: + # - Microsoft ships this object file next to default libraries + # - but this file is not a library, just a precompiled object + # - "-WHOLEARCHIVE" and "-DEFAULTLIB" only accept library, + # not precompiled object. + # - Bazel would assume linkopt that does not start with "-" or "$" + # as a label to a target, so we add a harmless "-DEFAULTLIB:kernel32.lib" + # before "setargv.obj". + # See https://msdn.microsoft.com/en-us/library/8bch7bkk.aspx + "-DEFAULTLIB:kernel32.lib setargv.obj", + ], + "//conditions:default": [], + }), visibility = ["//visibility:public"], deps = [":protobuf"], ) @@ -628,6 +644,7 @@ py_library( name = "python_srcs", srcs = glob( [ + "python/google/__init__.py", "python/google/protobuf/*.py", "python/google/protobuf/**/*.py", ], @@ -676,6 +693,7 @@ cc_binary( linkstatic = 1, deps = [ ":protobuf", + ":proto_api", ] + select({ "//conditions:default": [], ":use_fast_cpp_protos": ["//external:python_headers"], @@ -818,6 +836,15 @@ internal_protobuf_py_tests( deps = [":python_tests"], ) +cc_library( + name = "proto_api", + hdrs = ["python/google/protobuf/proto_api.h"], + deps = [ + "//external:python_headers", + ], + visibility = ["//visibility:public"], +) + proto_lang_toolchain( name = "cc_toolchain", command_line = "--cpp_out=$(OUT)", diff --git a/Makefile.am b/Makefile.am index ae4ac858..76bb2ba2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -51,6 +51,7 @@ pkgconfig_DATA = protobuf.pc protobuf-lite.pc csharp_EXTRA_DIST= \ csharp/.gitignore \ csharp/CHANGES.txt \ + csharp/Google.Protobuf.Tools.targets \ csharp/Google.Protobuf.Tools.nuspec \ csharp/README.md \ csharp/build_packages.bat \ @@ -804,6 +805,7 @@ python_EXTRA_DIST= \ python/google/protobuf/message.py \ python/google/protobuf/message_factory.py \ python/google/protobuf/python_protobuf.h \ + python/google/protobuf/proto_api.h \ python/google/protobuf/proto_builder.py \ python/google/protobuf/pyext/README \ python/google/protobuf/pyext/__init__.py \ @@ -1036,6 +1038,7 @@ EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ cmake/protobuf.pc.cmake \ cmake/protoc.cmake \ cmake/tests.cmake \ + cmake/version.rc.in \ editors/README.txt \ editors/proto.vim \ editors/protobuf-mode.el \ @@ -60,7 +60,7 @@ how to install protobuf runtime for that specific language: | Objective-C | [objectivec](objectivec) | | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fobjectivec_cocoapods_integration%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fobjectivec_ios_debug%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fobjectivec_ios_release%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fobjectivec_osx%2Fcontinuous) | | | C# | [csharp](csharp) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fubuntu%2Fcsharp%2Fcontinuous) | | [](https://ci.appveyor.com/project/protobuf/protobuf) | | JavaScript | [js](js) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fubuntu%2Fjavascript%2Fcontinuous) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fjavascript%2Fcontinuous) | | -| Ruby | [ruby](ruby) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fubuntu%2Fruby_all%2Fcontinuous) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fruby21%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fruby22%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fjruby%2Fcontinuous) | | +| Ruby | [ruby](ruby) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fubuntu%2Fruby_all%2Fcontinuous) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fruby21%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fruby22%2Fcontinuous) | | | Go | [golang/protobuf](https://github.com/golang/protobuf) | | | | | PHP | [php](php) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fubuntu%2Fphp_all%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fubuntu%2F32-bit%2Fcontinuous) | [](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fphp5.6_mac%2Fcontinuous)<br/>[](https://fusion.corp.google.com/projectanalysis/current/KOKORO/prod:protobuf%2Fgithub%2Fmaster%2Fmacos%2Fphp7.0_mac%2Fcontinuous) | | | Dart | [dart-lang/protobuf](https://github.com/dart-lang/protobuf) | | | | @@ -1,16 +1,24 @@ workspace(name = "com_google_protobuf") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") new_local_repository( name = "submodule_gmock", path = "third_party/googletest", - build_file = "third_party/googletest/BUILD.bazel" + build_file = "@//:third_party/googletest/BUILD.bazel" ) -new_http_archive( +http_archive( name = "six_archive", - build_file = "six.BUILD", + build_file = "@//:six.BUILD", sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", - url = "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55", + urls = ["https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55"], +) + +http_archive( + name = "bazel_skylib", + sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d", + strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], ) bind( diff --git a/appveyor.bat b/appveyor.bat index e59ebccc..29ec4922 100644 --- a/appveyor.bat +++ b/appveyor.bat @@ -1,11 +1,23 @@ setlocal +IF %platform%==MinGW GOTO build_mingw IF %language%==cpp GOTO build_cpp IF %language%==csharp GOTO build_csharp -echo Unsupported language %language%. Exiting. +echo Unsupported language %language% and platform %platform%. Exiting. goto :error +:build_mingw +echo Building MinGW +set PATH=C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH:C:\Program Files\Git\usr\bin;=% +mkdir build_mingw +cd build_mingw +cmake -G "%generator%" -Dprotobuf_BUILD_SHARED_LIBS=%BUILD_DLL% -Dprotobuf_UNICODE=%UNICODE% -Dprotobuf_BUILD_TESTS=0 ../cmake +mingw32-make -j8 all || goto error +rem cd %configuration% +rem tests.exe || goto error +goto :EOF + :build_cpp echo Building C++ mkdir build_msvc diff --git a/appveyor.yml b/appveyor.yml index 9644e066..91f737a8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,22 +1,29 @@ -platform: - - Win64 - configuration: - Debug environment: matrix: - - language: cpp + # Only test few combinations: "Visual Studio 2015 (14) + Win64/MinGW + Debug + DLL". We can + # test more combinations but AppVeyor just takes too long to finish (each + # combination takes ~15mins). + - platform: MinGW + language: cpp + image: Visual Studio 2015 + + - platform: Win64 + language: cpp image: Visual Studio 2015 BUILD_DLL: ON UNICODE: ON - - language: cpp + - platform: Win64 + language: cpp image: Visual Studio 2017 BUILD_DLL: OFF UNICODE: ON - - language: csharp + - platform: Win64 + language: csharp image: Visual Studio 2017 # Our build scripts run tests automatically; we don't want AppVeyor @@ -27,6 +34,7 @@ install: - git submodule update --init --recursive before_build: + - if %platform%==MinGW set generator=MinGW Makefiles - if %platform%==Win32 set generator=Visual Studio 14 - if %platform%==Win64 set generator=Visual Studio 14 Win64 - if %platform%==Win32 set vcplatform=Win32 diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 564fdab3..9f609228 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -126,8 +126,10 @@ java_benchmark_testing_files = \ java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2 - cp -r $(srcdir)/java tmp && cd tmp/java && mvn clean compile assembly:single - cd ../.. + cp -r $(srcdir)/java tmp + mkdir -p tmp/java/lib + cp $(top_srcdir)/java/core/target/*.jar tmp/java/lib/protobuf-java.jar + cd tmp/java && mvn clean compile assembly:single -Dprotobuf.version=$(PACKAGE_VERSION) && cd ../.. @touch javac_middleman java-benchmark: javac_middleman @@ -137,10 +139,10 @@ java-benchmark: javac_middleman @echo 'conf=()' >> java-benchmark @echo 'data_files=""' >> java-benchmark @echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+="$$arg,"; fi; done' >> java-benchmark - @echo 'java -cp '"tmp/java/target/*.jar"' com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\" >> java-benchmark - @echo '-b serializeToByteString,serializeToByteArray,serializeToMemoryStream,'"\\" >> java-benchmark - @echo 'deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark - @echo '-DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark + @echo 'java -cp '\"tmp/java/target/*:$(top_srcdir)/java/core/target/*:$(top_srcdir)/java/util/target/*\"" \\" >>java-benchmark + @echo ' com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\" >> java-benchmark + @echo ' -b serializeToByteArray,serializeToMemoryStream,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark + @echo ' -DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark @chmod +x java-benchmark java: protoc_middleman protoc_middleman2 java-benchmark @@ -179,8 +181,8 @@ nodist_libbenchmark_messages_la_SOURCES = \ python-pure-python-benchmark: python_add_init @echo "Writing shortcut script python-pure-python-benchmark..." @echo '#! /bin/bash' > python-pure-python-benchmark - @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark - @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark + @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-pure-python-benchmark + @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-pure-python-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'python\' >> python-pure-python-benchmark @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-pure-python-benchmark @echo python tmp/py_benchmark.py '$$@' >> python-pure-python-benchmark @@ -189,8 +191,8 @@ python-pure-python-benchmark: python_add_init python-cpp-reflection-benchmark: python_add_init @echo "Writing shortcut script python-cpp-reflection-benchmark..." @echo '#! /bin/bash' > python-cpp-reflection-benchmark - @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark - @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark + @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-reflection-benchmark + @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-reflection-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-reflection-benchmark @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-reflection-benchmark @echo python tmp/py_benchmark.py '$$@' >> python-cpp-reflection-benchmark @@ -199,8 +201,8 @@ python-cpp-reflection-benchmark: python_add_init python-cpp-generated-code-benchmark: python_add_init libbenchmark_messages.la @echo "Writing shortcut script python-cpp-generated-code-benchmark..." @echo '#! /bin/bash' > python-cpp-generated-code-benchmark - @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark - @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark + @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-generated-code-benchmark + @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-generated-code-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-generated-code-benchmark @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-generated-code-benchmark @echo python tmp/py_benchmark.py --cpp_generated '$$@' >> python-cpp-generated-code-benchmark @@ -347,11 +349,11 @@ gogo_proto_middleman: protoc-gen-gogoproto oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-gogoproto --gogoproto_out=$$oldpwd/tmp/gogo_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) touch gogo_proto_middleman -new_data = $$(for data in $(all_data); do echo "tmp$${data\#$(srcdir)}"; done | xargs) +gogo_data = $$(for data in $(all_data); do echo "tmp/gogo_data$${data\#$(srcdir)}"; done | xargs) generate_gogo_data: protoc_middleman protoc_middleman2 gogo-data-scrubber - mkdir -p `dirname $(new_data)` - ./gogo-data-scrubber $(all_data) $(new_data) + mkdir -p `dirname $(gogo_data)` + ./gogo-data-scrubber $(all_data) $(gogo_data) touch generate_gogo_data make_tmp_dir_gogo: @@ -406,8 +408,6 @@ gogoslick_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message4) ) touch gogoslick_protoc_middleman -gogo_data = $$(find . -type f -name "dataset.*.pb" -path "./tmp/*") - generate-gogo-benchmark-code: @echo '#! /bin/bash' > generate-gogo-benchmark-code @echo 'cp $(srcdir)/go/go_benchmark_test.go tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code @@ -451,7 +451,7 @@ gogoslick: gogoslick_protoc_middleman generate_gogo_data gogo-benchmark generat ############ UTIL RULES BEGIN ############ -bin_PROGRAMS += protoc-gen-gogoproto gogo-data-scrubber +bin_PROGRAMS += protoc-gen-gogoproto gogo-data-scrubber protoc-gen-proto2_to_proto3 proto3-data-stripper protoc_gen_gogoproto_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/src/libprotoc.la protoc_gen_gogoproto_SOURCES = util/protoc-gen-gogoproto.cc @@ -466,9 +466,40 @@ nodist_gogo_data_scrubber_SOURCES = \ $(benchmarks_protoc_outputs_proto2) \ $(benchmarks_protoc_outputs_proto2_header) \ $(benchmarks_protoc_outputs_header) + +protoc_gen_proto2_to_proto3_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/src/libprotoc.la +protoc_gen_proto2_to_proto3_SOURCES = util/protoc-gen-proto2_to_proto3.cc +protoc_gen_proto2_to_proto3_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util + +proto3_data_stripper_LDADD = $(top_srcdir)/src/libprotobuf.la +proto3_data_stripper_SOURCES = util/proto3_data_stripper.cc +proto3_data_stripper_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util +util/proto3_data_stripper-proto3_data_stripper.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) +nodist_proto3_data_stripper_SOURCES = \ + $(benchmarks_protoc_outputs) \ + $(benchmarks_protoc_outputs_proto2) \ + $(benchmarks_protoc_outputs_proto2_header) \ + $(benchmarks_protoc_outputs_header) + ############ UTIL RULES END ############ +############ PROTO3 PREPARATION BEGIN ############# + +proto3_proto_middleman: protoc-gen-proto2_to_proto3 + mkdir -p "tmp/proto3_proto" + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-proto2_to_proto3 --proto2_to_proto3_out=$$oldpwd/tmp/proto3_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) + touch proto3_proto_middleman + +proto3_data = $$(for data in $(all_data); do echo "tmp/proto3_data$${data\#$(srcdir)}"; done | xargs) + +generate_proto3_data: protoc_middleman protoc_middleman2 proto3-data-stripper + mkdir -p `dirname $(proto3_data)` + ./proto3-data-stripper $(all_data) $(proto3_data) + touch generate_proto3_data + +############ PROTO3 PREPARATION END ############# + MAINTAINERCLEANFILES = \ Makefile.in @@ -511,7 +542,9 @@ CLEANFILES = \ gogoslick_protoc_middleman \ gogoslick \ gogo-benchmark \ - gogo/cpp_no_group/cpp_benchmark.* + gogo/cpp_no_group/cpp_benchmark.* \ + proto3_proto_middleman \ + generate_proto3_data clean-local: diff --git a/benchmarks/README.md b/benchmarks/README.md index 21cd7352..ae5c7ddd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -5,7 +5,7 @@ This directory contains benchmarking schemas and data sets that you can use to test a variety of performance scenarios against your protobuf language runtime. If you are looking for performance numbers of officially support languages, see [here]( -https://github.com/google/protobuf/blob/master/docs/Performance.md) +https://github.com/google/protobuf/blob/master/docs/performance.md) ## Prerequisite diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index c2cd78a1..570bd664 100755 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -14,7 +14,10 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>3.5.0</version> + <version>${protobuf.version}</version> + <type>jar</type> + <scope>system</scope> + <systemPath>${project.basedir}/lib/protobuf-java.jar</systemPath> </dependency> <dependency> <groupId>com.google.caliper</groupId> diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java index 94568aea..c766d74e 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -5,6 +5,7 @@ import com.google.caliper.BeforeExperiment; import com.google.caliper.AfterExperiment; import com.google.caliper.Benchmark; import com.google.caliper.Param; +import com.google.caliper.api.VmOptions; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.ExtensionRegistry; @@ -22,6 +23,12 @@ import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.List; +// Caliper set CICompilerCount to 1 for making sure compilation doesn't run in parallel with itself, +// This makes TieredCompilation not working. We just disable TieredCompilation by default. In master +// branch this has been disabled by default in caliper: +// https://github.com/google/caliper/blob/master/caliper-runner/src/main/java/com/google/caliper/runner/target/Jvm.java#L38:14 +// But this haven't been added into most recent release. +@VmOptions("-XX:-TieredCompilation") public class ProtoCaliperBenchmark { public enum BenchmarkMessageType { GOOGLE_MESSAGE1_PROTO3 { @@ -152,18 +159,6 @@ public class ProtoCaliperBenchmark { @Benchmark - void serializeToByteString(int reps) throws IOException { - if (sampleMessageList.size() == 0) { - return; - } - for (int i = 0; i < reps; i++) { - for (int j = 0; j < sampleMessageList.size(); j++) { - sampleMessageList.get(j).toByteString(); - } - } - } - - @Benchmark void serializeToByteArray(int reps) throws IOException { if (sampleMessageList.size() == 0) { return; @@ -189,19 +184,6 @@ public class ProtoCaliperBenchmark { } @Benchmark - void deserializeFromByteString(int reps) throws IOException { - if (inputStringList.size() == 0) { - return; - } - for (int i = 0; i < reps; i++) { - for (int j = 0; j < inputStringList.size(); j++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStringList.get(j), extensions); - } - } - } - - @Benchmark void deserializeFromByteArray(int reps) throws IOException { if (inputDataList.size() == 0) { return; diff --git a/benchmarks/python/py_benchmark.py b/benchmarks/python/py_benchmark.py index 6942d208..e86b61e7 100755 --- a/benchmarks/python/py_benchmark.py +++ b/benchmarks/python/py_benchmark.py @@ -1,3 +1,4 @@ +from __future__ import print_function import sys import os import timeit @@ -138,15 +139,15 @@ if __name__ == "__main__": results.append(run_one_test(file)) if args.json != "no": - print json.dumps(results) + print(json.dumps(results)) else: for result in results: - print "Message %s of dataset file %s" % \ - (result["message_name"], result["filename"]) - print "Average time for parse_from_benchmark: %.2f ns" % \ + print("Message %s of dataset file %s" % \ + (result["message_name"], result["filename"])) + print("Average time for parse_from_benchmark: %.2f ns" % \ (result["benchmarks"][ \ - args.behavior_prefix + "_parse_from_benchmark"]) - print "Average time for serialize_to_benchmark: %.2f ns" % \ + args.behavior_prefix + "_parse_from_benchmark"])) + print("Average time for serialize_to_benchmark: %.2f ns" % \ (result["benchmarks"][ \ - args.behavior_prefix + "_serialize_to_benchmark"]) - print "" + args.behavior_prefix + "_serialize_to_benchmark"])) + print("") diff --git a/benchmarks/util/big_query_utils.py b/benchmarks/util/big_query_utils.py index 14105aa6..aea55bbd 100755 --- a/benchmarks/util/big_query_utils.py +++ b/benchmarks/util/big_query_utils.py @@ -1,5 +1,6 @@ #!/usr/bin/env python2.7 +from __future__ import print_function import argparse import json import uuid @@ -37,11 +38,11 @@ def create_dataset(biq_query, project_id, dataset_id): dataset_req.execute(num_retries=NUM_RETRIES) except HttpError as http_error: if http_error.resp.status == 409: - print 'Warning: The dataset %s already exists' % dataset_id + print('Warning: The dataset %s already exists' % dataset_id) else: # Note: For more debugging info, print "http_error.content" - print 'Error in creating dataset: %s. Err: %s' % (dataset_id, - http_error) + print('Error in creating dataset: %s. Err: %s' % (dataset_id, + http_error)) is_success = False return is_success @@ -109,13 +110,13 @@ def create_table2(big_query, table_req = big_query.tables().insert( projectId=project_id, datasetId=dataset_id, body=body) res = table_req.execute(num_retries=NUM_RETRIES) - print 'Successfully created %s "%s"' % (res['kind'], res['id']) + print('Successfully created %s "%s"' % (res['kind'], res['id'])) except HttpError as http_error: if http_error.resp.status == 409: - print 'Warning: Table %s already exists' % table_id + print('Warning: Table %s already exists' % table_id) else: - print 'Error in creating table: %s. Err: %s' % (table_id, - http_error) + print('Error in creating table: %s. Err: %s' % (table_id, + http_error)) is_success = False return is_success @@ -141,9 +142,9 @@ def patch_table(big_query, project_id, dataset_id, table_id, fields_schema): tableId=table_id, body=body) res = table_req.execute(num_retries=NUM_RETRIES) - print 'Successfully patched %s "%s"' % (res['kind'], res['id']) + print('Successfully patched %s "%s"' % (res['kind'], res['id'])) except HttpError as http_error: - print 'Error in creating table: %s. Err: %s' % (table_id, http_error) + print('Error in creating table: %s. Err: %s' % (table_id, http_error)) is_success = False return is_success @@ -159,10 +160,10 @@ def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): body=body) res = insert_req.execute(num_retries=NUM_RETRIES) if res.get('insertErrors', None): - print 'Error inserting rows! Response: %s' % res + print('Error inserting rows! Response: %s' % res) is_success = False except HttpError as http_error: - print 'Error inserting rows to the table %s' % table_id + print('Error inserting rows to the table %s' % table_id) is_success = False return is_success @@ -176,8 +177,8 @@ def sync_query_job(big_query, project_id, query, timeout=5000): projectId=project_id, body=query_data).execute(num_retries=NUM_RETRIES) except HttpError as http_error: - print 'Query execute job failed with error: %s' % http_error - print http_error.content + print('Query execute job failed with error: %s' % http_error) + print(http_error.content) return query_job diff --git a/benchmarks/util/data_proto2_to_proto3_util.h b/benchmarks/util/data_proto2_to_proto3_util.h new file mode 100644 index 00000000..5eea8509 --- /dev/null +++ b/benchmarks/util/data_proto2_to_proto3_util.h @@ -0,0 +1,64 @@ +#ifndef PROTOBUF_BENCHMARKS_UTIL_DATA_PROTO2_TO_PROTO3_UTIL_H_ +#define PROTOBUF_BENCHMARKS_UTIL_DATA_PROTO2_TO_PROTO3_UTIL_H_ + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" + +using google::protobuf::FieldDescriptor; +using google::protobuf::Message; +using google::protobuf::Reflection; + +namespace google { +namespace protobuf { +namespace util { + +class DataStripper { + public: + void StripMessage(Message *message) { + std::vector<const FieldDescriptor*> set_fields; + const Reflection* reflection = message->GetReflection(); + reflection->ListFields(*message, &set_fields); + + for (size_t i = 0; i < set_fields.size(); i++) { + const FieldDescriptor* field = set_fields[i]; + if (ShouldBeClear(field)) { + reflection->ClearField(message, field); + continue; + } + if (field->type() == FieldDescriptor::TYPE_MESSAGE) { + if (field->is_repeated()) { + for (int j = 0; j < reflection->FieldSize(*message, field); j++) { + StripMessage(reflection->MutableRepeatedMessage(message, field, j)); + } + } else { + StripMessage(reflection->MutableMessage(message, field)); + } + } + } + + reflection->MutableUnknownFields(message)->Clear(); + } + private: + virtual bool ShouldBeClear(const FieldDescriptor *field) = 0; +}; + +class GogoDataStripper : public DataStripper { + private: + virtual bool ShouldBeClear(const FieldDescriptor *field) { + return field->type() == FieldDescriptor::TYPE_GROUP; + } +}; + +class Proto3DataStripper : public DataStripper { + private: + virtual bool ShouldBeClear(const FieldDescriptor *field) { + return field->type() == FieldDescriptor::TYPE_GROUP || + field->is_extension(); + } +}; + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // PROTOBUF_BENCHMARKS_UTIL_DATA_PROTO2_TO_PROTO3_UTIL_H_ diff --git a/benchmarks/util/gogo_data_scrubber.cc b/benchmarks/util/gogo_data_scrubber.cc index fb9af6e2..9ef57b0d 100644 --- a/benchmarks/util/gogo_data_scrubber.cc +++ b/benchmarks/util/gogo_data_scrubber.cc @@ -4,43 +4,11 @@ #include "datasets/google_message2/benchmark_message2.pb.h" #include "datasets/google_message3/benchmark_message3.pb.h" #include "datasets/google_message4/benchmark_message4.pb.h" - -#include "google/protobuf/message.h" -#include "google/protobuf/descriptor.h" +#include "data_proto2_to_proto3_util.h" #include <fstream> -using google::protobuf::FieldDescriptor; -using google::protobuf::Message; -using google::protobuf::Reflection; - - -class DataGroupStripper { - public: - static void StripMessage(Message *message) { - std::vector<const FieldDescriptor*> set_fields; - const Reflection* reflection = message->GetReflection(); - reflection->ListFields(*message, &set_fields); - - for (size_t i = 0; i < set_fields.size(); i++) { - const FieldDescriptor* field = set_fields[i]; - if (field->type() == FieldDescriptor::TYPE_GROUP) { - reflection->ClearField(message, field); - } - if (field->type() == FieldDescriptor::TYPE_MESSAGE) { - if (field->is_repeated()) { - for (int j = 0; j < reflection->FieldSize(*message, field); j++) { - StripMessage(reflection->MutableRepeatedMessage(message, field, j)); - } - } else { - StripMessage(reflection->MutableMessage(message, field)); - } - } - } - - reflection->MutableUnknownFields(message)->Clear(); - } -}; +using google::protobuf::util::GogoDataStripper; std::string ReadFile(const std::string& name) { std::ifstream file(name.c_str()); @@ -91,7 +59,8 @@ int main(int argc, char *argv[]) { for (int i = 0; i < dataset.payload_size(); i++) { message->ParseFromString(dataset.payload(i)); - DataGroupStripper::StripMessage(message); + GogoDataStripper stripper; + stripper.StripMessage(message); dataset.set_payload(i, message->SerializeAsString()); } diff --git a/benchmarks/util/proto3_data_stripper.cc b/benchmarks/util/proto3_data_stripper.cc new file mode 100644 index 00000000..3096c4c1 --- /dev/null +++ b/benchmarks/util/proto3_data_stripper.cc @@ -0,0 +1,74 @@ +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" +#include "data_proto2_to_proto3_util.h" + +#include <fstream> + +using google::protobuf::util::Proto3DataStripper; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" + << name + << "', please make sure you are running this command from the benchmarks" + << " directory.\n"; + return std::string((std::istreambuf_iterator<char>(file)), + std::istreambuf_iterator<char>()); +} + +int main(int argc, char *argv[]) { + if (argc % 2 == 0 || argc == 1) { + std::cerr << "Usage: [input_files] [output_file_names] where " << + "input_files are one to one mapping to output_file_names." << + std::endl; + return 1; + } + + for (int i = argc / 2; i > 0; i--) { + const std::string &input_file = argv[i]; + const std::string &output_file = argv[i + argc / 2]; + + std::cerr << "Generating " << input_file + << " to " << output_file << std::endl; + benchmarks::BenchmarkDataset dataset; + Message* message; + std::string dataset_payload = ReadFile(input_file); + GOOGLE_CHECK(dataset.ParseFromString(dataset_payload)) + << "Can' t parse data file " << input_file; + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + message = new benchmarks::proto3::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + message = new benchmarks::proto2::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + message = new benchmarks::proto2::GoogleMessage2; + } else if (dataset.message_name() == + "benchmarks.google_message3.GoogleMessage3") { + message = new benchmarks::google_message3::GoogleMessage3; + } else if (dataset.message_name() == + "benchmarks.google_message4.GoogleMessage4") { + message = new benchmarks::google_message4::GoogleMessage4; + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } + + for (int i = 0; i < dataset.payload_size(); i++) { + message->ParseFromString(dataset.payload(i)); + Proto3DataStripper stripper; + stripper.StripMessage(message); + dataset.set_payload(i, message->SerializeAsString()); + } + + std::ofstream ofs(output_file); + ofs << dataset.SerializeAsString(); + ofs.close(); + } + + + return 0; +} diff --git a/benchmarks/util/protoc-gen-gogoproto.cc b/benchmarks/util/protoc-gen-gogoproto.cc index bfa6a5e5..9c1b3d04 100644 --- a/benchmarks/util/protoc-gen-gogoproto.cc +++ b/benchmarks/util/protoc-gen-gogoproto.cc @@ -12,7 +12,7 @@ using google::protobuf::FileDescriptor; using google::protobuf::DescriptorPool; using google::protobuf::io::Printer; using google::protobuf::util::SchemaGroupStripper; -using google::protobuf::util::SchemaAddZeroEnumValue; +using google::protobuf::util::EnumScrubber; namespace google { namespace protobuf { @@ -74,7 +74,7 @@ class GoGoProtoGenerator : public CodeGenerator { file->CopyTo(&new_file); SchemaGroupStripper::StripFile(file, &new_file); - SchemaAddZeroEnumValue enum_scrubber; + EnumScrubber enum_scrubber; enum_scrubber.ScrubFile(&new_file); string filename = file->name(); diff --git a/benchmarks/util/protoc-gen-proto2_to_proto3.cc b/benchmarks/util/protoc-gen-proto2_to_proto3.cc new file mode 100644 index 00000000..d0a89023 --- /dev/null +++ b/benchmarks/util/protoc-gen-proto2_to_proto3.cc @@ -0,0 +1,115 @@ +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/printer.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "schema_proto2_to_proto3_util.h" + +#include "google/protobuf/compiler/plugin.h" + +using google::protobuf::FileDescriptorProto; +using google::protobuf::FileDescriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::io::Printer; +using google::protobuf::util::SchemaGroupStripper; +using google::protobuf::util::EnumScrubber; +using google::protobuf::util::ExtensionStripper; +using google::protobuf::util::FieldScrubber; + +namespace google { +namespace protobuf { +namespace compiler { + +namespace { + +string StripProto(string filename) { + return filename.substr(0, filename.rfind(".proto")); +} + +DescriptorPool* GetPool() { + static DescriptorPool *pool = new DescriptorPool(); + return pool; +} + +} // namespace + +class Proto2ToProto3Generator final : public CodeGenerator { + public: + bool GenerateAll(const std::vector<const FileDescriptor*>& files, + const string& parameter, + GeneratorContext* context, + string* error) const { + for (int i = 0; i < files.size(); i++) { + for (auto file : files) { + if (CanGenerate(file)) { + Generate(file, parameter, context, error); + break; + } + } + } + + return true; + } + + bool Generate(const FileDescriptor* file, + const string& parameter, + GeneratorContext* context, + string* error) const { + FileDescriptorProto new_file; + file->CopyTo(&new_file); + SchemaGroupStripper::StripFile(file, &new_file); + + EnumScrubber enum_scrubber; + enum_scrubber.ScrubFile(&new_file); + ExtensionStripper::StripFile(&new_file); + FieldScrubber::ScrubFile(&new_file); + new_file.set_syntax("proto3"); + + string filename = file->name(); + string basename = StripProto(filename); + + std::vector<std::pair<string,string>> option_pairs; + ParseGeneratorParameter(parameter, &option_pairs); + + std::unique_ptr<google::protobuf::io::ZeroCopyOutputStream> output( + context->Open(basename + ".proto")); + string content = GetPool()->BuildFile(new_file)->DebugString(); + Printer printer(output.get(), '$'); + printer.WriteRaw(content.c_str(), content.size()); + + return true; + } + private: + bool CanGenerate(const FileDescriptor* file) const { + if (GetPool()->FindFileByName(file->name()) != nullptr) { + return false; + } + for (int j = 0; j < file->dependency_count(); j++) { + if (GetPool()->FindFileByName(file->dependency(j)->name()) == nullptr) { + return false; + } + } + for (int j = 0; j < file->public_dependency_count(); j++) { + if (GetPool()->FindFileByName( + file->public_dependency(j)->name()) == nullptr) { + return false; + } + } + for (int j = 0; j < file->weak_dependency_count(); j++) { + if (GetPool()->FindFileByName( + file->weak_dependency(j)->name()) == nullptr) { + return false; + } + } + return true; + } +}; + +} // namespace compiler +} // namespace protobuf +} // namespace google + +int main(int argc, char* argv[]) { + google::protobuf::compiler::Proto2ToProto3Generator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/benchmarks/util/run_and_upload.py b/benchmarks/util/run_and_upload.py index ae22a668..43c9fa2d 100755 --- a/benchmarks/util/run_and_upload.py +++ b/benchmarks/util/run_and_upload.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import import argparse import os import re @@ -5,7 +7,7 @@ import copy import uuid import calendar import time -import big_query_utils +from . import big_query_utils import datetime import json # This import depends on the automake rule protoc_middleman, please make sure @@ -255,7 +257,7 @@ def upload_result(result_list, metadata): if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET, _TABLE + "$" + _NOW, [row]): - print 'Error when uploading result', new_result + print('Error when uploading result', new_result) if __name__ == "__main__": @@ -280,11 +282,11 @@ if __name__ == "__main__": parse_go_result(args.go_input_file) metadata = get_metadata() - print "uploading cpp results..." + print("uploading cpp results...") upload_result(cpp_result, metadata) - print "uploading java results..." + print("uploading java results...") upload_result(java_result, metadata) - print "uploading python results..." + print("uploading python results...") upload_result(python_result, metadata) - print "uploading go results..." + print("uploading go results...") upload_result(go_result, metadata) diff --git a/benchmarks/util/schema_proto2_to_proto3_util.h b/benchmarks/util/schema_proto2_to_proto3_util.h index 089012dd..0079f6f1 100644 --- a/benchmarks/util/schema_proto2_to_proto3_util.h +++ b/benchmarks/util/schema_proto2_to_proto3_util.h @@ -74,10 +74,10 @@ class SchemaGroupStripper { }; -class SchemaAddZeroEnumValue { +class EnumScrubber { public: - SchemaAddZeroEnumValue() + EnumScrubber() : total_added_(0) { } @@ -130,6 +130,63 @@ class SchemaAddZeroEnumValue { int total_added_; }; +class ExtensionStripper { + public: + static void StripFile(FileDescriptorProto *file) { + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + StripMessage(file->mutable_message_type(i)); + } + file->mutable_extension()->Clear(); + } + private: + static void StripMessage(DescriptorProto *message_type) { + message_type->mutable_extension()->Clear(); + message_type->clear_extension_range(); + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + StripMessage(message_type->mutable_nested_type(i)); + } + } +}; + + +class FieldScrubber { + public: + static void ScrubFile(FileDescriptorProto *file) { + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + ScrubMessage(file->mutable_message_type(i)); + } + for (int i = 0; i < file->mutable_extension()->size(); i++) { + file->mutable_extension(i)->clear_default_value(); + if (ShouldClearLabel(file->mutable_extension(i))) { + file->mutable_extension(i)->clear_label(); + } + } + } + private: + static bool ShouldClearLabel(const FieldDescriptorProto *field) { + return field->label() == FieldDescriptorProto::LABEL_REQUIRED; + } + + static void ScrubMessage(DescriptorProto *message_type) { + message_type->mutable_extension()->Clear(); + for (int i = 0; i < message_type->mutable_extension()->size(); i++) { + message_type->mutable_extension(i)->clear_default_value(); + if (ShouldClearLabel(message_type->mutable_extension(i))) { + message_type->mutable_extension(i)->clear_label(); + } + } + for (int i = 0; i < message_type->mutable_field()->size(); i++) { + message_type->mutable_field(i)->clear_default_value(); + if (ShouldClearLabel(message_type->mutable_field(i))) { + message_type->mutable_field(i)->clear_label(); + } + } + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + ScrubMessage(message_type->mutable_nested_type(i)); + } + } +}; + } // namespace util } // namespace protobuf } // namespace google diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 02174e96..ece39f7f 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -7,7 +7,11 @@ endif() # CMake policies cmake_policy(SET CMP0022 NEW) - +# On MacOS use @rpath/ for target's install name prefix path +if (POLICY CMP0042) + cmake_policy(SET CMP0042 NEW) +endif () +# Clear VERSION variables when no VERSION is given to project() if(POLICY CMP0048) cmake_policy(SET CMP0048 NEW) endif() @@ -166,12 +170,23 @@ if (MSVC) add_definitions(/bigobj) string(REPLACE "/" "\\" PROTOBUF_SOURCE_WIN32_PATH ${protobuf_SOURCE_DIR}) string(REPLACE "/" "\\" PROTOBUF_BINARY_WIN32_PATH ${protobuf_BINARY_DIR}) + string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") configure_file(extract_includes.bat.in extract_includes.bat) # Suppress linker warnings about files with no symbols defined. set(CMAKE_STATIC_LINKER_FLAGS /ignore:4221) + + # Configure Resource Compiler + enable_language(RC) + # use English language (0x409) in resource compiler + set(rc_flags "/l0x409") + # fix rc.exe invocations because of usage of add_definitions() + set(CMAKE_RC_COMPILE_OBJECT "<CMAKE_RC_COMPILER> ${rc_flags} <DEFINES> /fo<OBJECT> <SOURCE>") + + configure_file(version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY) endif (MSVC) + get_filename_component(protobuf_source_dir ${protobuf_SOURCE_DIR} PATH) include_directories( diff --git a/cmake/install.cmake b/cmake/install.cmake index 82036cb6..9b2ae93c 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -15,6 +15,13 @@ foreach(_library ${_protobuf_libraries}) PROPERTY INTERFACE_INCLUDE_DIRECTORIES $<BUILD_INTERFACE:${protobuf_source_dir}/src> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>) + if (UNIX AND NOT APPLE) + set_property(TARGET ${_library} + PROPERTY INSTALL_RPATH "$ORIGIN") + elseif (APPLE) + set_property(TARGET ${_library} + PROPERTY INSTALL_RPATH "@loader_path") + endif() install(TARGETS ${_library} EXPORT protobuf-targets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT ${_library} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${_library} @@ -24,6 +31,13 @@ endforeach() if (protobuf_BUILD_PROTOC_BINARIES) install(TARGETS protoc EXPORT protobuf-targets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT protoc) + if (UNIX AND NOT APPLE) + set_property(TARGET protoc + PROPERTY INSTALL_RPATH "$ORIGIN/../lib") + elseif (APPLE) + set_property(TARGET protoc + PROPERTY INSTALL_RPATH "@loader_path/../lib") + endif() endif (protobuf_BUILD_PROTOC_BINARIES) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc ${CMAKE_CURRENT_BINARY_DIR}/protobuf-lite.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index 911fce5e..1d0b0b84 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -48,8 +48,14 @@ set(libprotobuf_lite_includes ${protobuf_source_dir}/src/google/protobuf/wire_format_lite.h ) +if (MSVC) +set(libprotobuf_lite_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc +) +endif() + add_library(libprotobuf-lite ${protobuf_SHARED_OR_STATIC} - ${libprotobuf_lite_files} ${libprotobuf_lite_includes}) + ${libprotobuf_lite_files} ${libprotobuf_lite_includes} ${libprotobuf_lite_rc_files}) target_link_libraries(libprotobuf-lite ${CMAKE_THREAD_LIBS_INIT}) target_include_directories(libprotobuf-lite PUBLIC ${protobuf_source_dir}/src) if(MSVC AND protobuf_BUILD_SHARED_LIBS) diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 65d05c19..bd570be7 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -112,8 +112,14 @@ set(libprotobuf_includes ${protobuf_source_dir}/src/google/protobuf/wrappers.pb.h ) +if (MSVC) +set(libprotobuf_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc +) +endif() + add_library(libprotobuf ${protobuf_SHARED_OR_STATIC} - ${libprotobuf_lite_files} ${libprotobuf_files} ${libprotobuf_includes}) + ${libprotobuf_lite_files} ${libprotobuf_files} ${libprotobuf_includes} ${libprotobuf_rc_files}) target_link_libraries(libprotobuf ${CMAKE_THREAD_LIBS_INIT}) if(protobuf_WITH_ZLIB) target_link_libraries(libprotobuf ${ZLIB_LIBRARIES}) diff --git a/cmake/libprotoc.cmake b/cmake/libprotoc.cmake index ec6958bc..92dfd306 100644 --- a/cmake/libprotoc.cmake +++ b/cmake/libprotoc.cmake @@ -161,20 +161,14 @@ set(libprotoc_headers ${protobuf_source_dir}/src/google/protobuf/compiler/zip_writer.h ) -set(js_well_known_types_sources - ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/any.js - ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/struct.js - ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types/timestamp.js -) -add_executable(js_embed ${protobuf_source_dir}/src/google/protobuf/compiler/js/embed.cc) -add_custom_command( - OUTPUT ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc - DEPENDS js_embed ${js_well_known_types_sources} - COMMAND js_embed ${js_well_known_types_sources} > ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc +if (MSVC) +set(libprotoc_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc ) +endif() add_library(libprotoc ${protobuf_SHARED_OR_STATIC} - ${libprotoc_files} ${libprotoc_headers}) + ${libprotoc_files} ${libprotoc_headers} ${libprotoc_rc_files}) target_link_libraries(libprotoc libprotobuf) if(MSVC AND protobuf_BUILD_SHARED_LIBS) target_compile_definitions(libprotoc diff --git a/cmake/protobuf-config.cmake.in b/cmake/protobuf-config.cmake.in index acedcc7a..3480c17c 100644 --- a/cmake/protobuf-config.cmake.in +++ b/cmake/protobuf-config.cmake.in @@ -11,7 +11,7 @@ function(protobuf_generate) include(CMakeParseArguments) set(_options APPEND_PATH) - set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO) + set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR) if(COMMAND target_sources) list(APPEND _singleargs TARGET) endif() @@ -34,15 +34,19 @@ function(protobuf_generate) endif() string(TOLOWER ${protobuf_generate_LANGUAGE} protobuf_generate_LANGUAGE) + if(NOT protobuf_generate_PROTOC_OUT_DIR) + set(protobuf_generate_PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) + endif() + if(protobuf_generate_EXPORT_MACRO AND protobuf_generate_LANGUAGE STREQUAL cpp) set(_dll_export_decl "dllexport_decl=${protobuf_generate_EXPORT_MACRO}:") endif() - if(NOT protobuf_GENERATE_EXTENSIONS) + if(NOT protobuf_generate_GENERATE_EXTENSIONS) if(protobuf_generate_LANGUAGE STREQUAL cpp) - set(protobuf_GENERATE_EXTENSIONS .pb.h .pb.cc) + set(protobuf_generate_GENERATE_EXTENSIONS .pb.h .pb.cc) elseif(protobuf_generate_LANGUAGE STREQUAL python) - set(protobuf_GENERATE_EXTENSIONS _pb2.py) + set(protobuf_generate_GENERATE_EXTENSIONS _pb2.py) else() message(SEND_ERROR "Error: protobuf_generate given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS") return() @@ -91,16 +95,16 @@ function(protobuf_generate) get_filename_component(_basename ${_proto} NAME_WE) set(_generated_srcs) - foreach(_ext ${protobuf_GENERATE_EXTENSIONS}) - list(APPEND _generated_srcs "${CMAKE_CURRENT_BINARY_DIR}/${_basename}${_ext}") + foreach(_ext ${protobuf_generate_GENERATE_EXTENSIONS}) + list(APPEND _generated_srcs "${protobuf_generate_PROTOC_OUT_DIR}/${_basename}${_ext}") endforeach() list(APPEND _generated_srcs_all ${_generated_srcs}) add_custom_command( OUTPUT ${_generated_srcs} COMMAND protobuf::protoc - ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${_abs_file} - DEPENDS ${ABS_FIL} protobuf::protoc + ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_protobuf_include_path} ${_abs_file} + DEPENDS ${_abs_file} protobuf::protoc COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}" VERBATIM ) endforeach() diff --git a/cmake/protoc.cmake b/cmake/protoc.cmake index 5777b16e..bb160f47 100644 --- a/cmake/protoc.cmake +++ b/cmake/protoc.cmake @@ -2,6 +2,15 @@ set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc ) -add_executable(protoc ${protoc_files}) +if (MSVC) +set(protoc_rc_files + ${CMAKE_CURRENT_BINARY_DIR}/version.rc +) +endif() + +add_executable(protoc ${protoc_files} ${protoc_rc_files}) target_link_libraries(protoc libprotobuf libprotoc) add_executable(protobuf::protoc ALIAS protoc) + +set_target_properties(protoc PROPERTIES + VERSION ${protobuf_VERSION}) diff --git a/cmake/tests.cmake b/cmake/tests.cmake index ec790e33..f91567b8 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -204,6 +204,10 @@ if(protobuf_ABSOLUTE_TEST_PLUGIN_PATH) add_compile_options(-DGOOGLE_PROTOBUF_TEST_PLUGIN_PATH="$<TARGET_FILE:test_plugin>") endif() +if(MINGW) + set_source_files_properties(${tests_files} PROPERTIES COMPILE_FLAGS "-Wno-narrowing") +endif() + add_executable(tests ${tests_files} ${common_test_files} ${tests_proto_files} ${lite_test_proto_files}) target_link_libraries(tests libprotoc libprotobuf gmock_main) diff --git a/cmake/version.rc.in b/cmake/version.rc.in new file mode 100644 index 00000000..cbce1e53 --- /dev/null +++ b/cmake/version.rc.in @@ -0,0 +1,45 @@ +#define VS_FF_DEBUG 0x1L +#define VS_VERSION_INFO 0x1L +#define VS_FFI_FILEFLAGSMASK 0x17L +#define VER_PRIVATEBUILD 0x0L +#define VER_PRERELEASE 0x0L +#define VOS__WINDOWS32 0x4L +#define VFT_DLL 0x2L +#define VFT2_UNKNOWN 0x0L + +#ifndef DEBUG +#define VER_DEBUG 0 +#else +#define VER_DEBUG VS_FF_DEBUG +#endif + + +VS_VERSION_INFO VERSIONINFO + FILEVERSION @protobuf_RC_FILEVERSION@,0 + PRODUCTVERSION @protobuf_RC_FILEVERSION@,0 + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK + FILEFLAGS VER_DEBUG + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL +BEGIN + BLOCK "VarFileInfo" + BEGIN + // English language (0x409) and the Windows Unicode codepage (1200) + VALUE "Translation", 0x409, 1200 + END + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "Compiled with @CMAKE_CXX_COMPILER_ID@ @CMAKE_CXX_COMPILER_VERSION@\0" + VALUE "ProductVersion", "@protobuf_VERSION@\0" + VALUE "FileVersion", "@protobuf_VERSION@\0" + VALUE "InternalName", "protobuf\0" + VALUE "ProductName", "Protocol Buffers - Google's Data Interchange Format\0" + VALUE "CompanyName", "Google Inc.\0" + VALUE "LegalCopyright", "Copyright 2008 Google Inc. All rights reserved.\0" + VALUE "Licence", "BSD\0" + VALUE "Info", "https://developers.google.com/protocol-buffers/\0" + END + END +END diff --git a/configure.ac b/configure.ac index aec10cfa..69b3fe30 100644 --- a/configure.ac +++ b/configure.ac @@ -190,7 +190,7 @@ AM_CONDITIONAL([HAVE_PTHREAD], [test "x$ax_pthread_ok" = "xyes"]) AC_CXX_STL_HASH case "$target_os" in - mingw* | cygwin* | win* | aix*) + mingw* | cygwin* | win* | aix* | *android* ) ;; *) # Need to link against rt on Solaris diff --git a/csharp/Google.Protobuf.Tools.nuspec b/csharp/Google.Protobuf.Tools.nuspec index ae8cdd74..0ba7cc9f 100644 --- a/csharp/Google.Protobuf.Tools.nuspec +++ b/csharp/Google.Protobuf.Tools.nuspec @@ -33,5 +33,7 @@ <file src="..\src\google\protobuf\timestamp.proto" target="tools\google\protobuf" /> <file src="..\src\google\protobuf\type.proto" target="tools\google\protobuf" /> <file src="..\src\google\protobuf\wrappers.proto" target="tools\google\protobuf" /> + <file src="Google.Protobuf.Tools.targets" target="buildCrossTargeting" /> + <file src="Google.Protobuf.Tools.targets" target="build" /> </files> </package> diff --git a/csharp/Google.Protobuf.Tools.targets b/csharp/Google.Protobuf.Tools.targets new file mode 100644 index 00000000..0d0b6725 --- /dev/null +++ b/csharp/Google.Protobuf.Tools.targets @@ -0,0 +1,16 @@ +<Project>
+ <PropertyGroup>
+ <protoc_linux64>$(MSBuildThisFileDirectory)..\tools\linux_x64\protoc</protoc_linux64>
+ <protoc_linux86>$(MSBuildThisFileDirectory)..\tools\linux_x86\protoc</protoc_linux86>
+ <protoc_macosx64>$(MSBuildThisFileDirectory)..\tools\macosx_x64\protoc</protoc_macosx64>
+ <protoc_macosx86>$(MSBuildThisFileDirectory)..\tools\macosx_x86\protoc</protoc_macosx86>
+ <protoc_windows64>$(MSBuildThisFileDirectory)..\tools\windows_x64\protoc.exe</protoc_windows64>
+ <protoc_windows86>$(MSBuildThisFileDirectory)..\tools\windows_x86\protoc.exe</protoc_windows86>
+ <protoc Condition="'$([MSBuild]::IsOsPlatform(Linux))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X64'">$(protoc_linux64)</protoc>
+ <protoc Condition="'$([MSBuild]::IsOsPlatform(Linux))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X86'">$(protoc_linux86)</protoc>
+ <protoc Condition="'$([MSBuild]::IsOsPlatform(OSX))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X64'">$(protoc_macosx64)</protoc>
+ <protoc Condition="'$([MSBuild]::IsOsPlatform(OSX))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X86'">$(protoc_macosx86)</protoc>
+ <protoc Condition="'$([MSBuild]::IsOsPlatform(Windows))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X64'">$(protoc_windows64)</protoc>
+ <protoc Condition="'$([MSBuild]::IsOsPlatform(Windows))' And '$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)'=='X86'">$(protoc_windows86)</protoc>
+ </PropertyGroup>
+</Project>
\ No newline at end of file diff --git a/csharp/build_packages.bat b/csharp/build_packages.bat index d7205659..8157bbab 100644 --- a/csharp/build_packages.bat +++ b/csharp/build_packages.bat @@ -1,7 +1,7 @@ @rem Builds Google.Protobuf NuGet packages dotnet restore src/Google.Protobuf.sln -dotnet pack -c Release src/Google.Protobuf.sln || goto :error +dotnet pack -c Release src/Google.Protobuf.sln /p:SourceLinkCreate=true || goto :error goto :EOF diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index d19531aa..46728b72 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -17,8 +17,6 @@ <PackageLicenseUrl>https://github.com/google/protobuf/blob/master/LICENSE</PackageLicenseUrl> <RepositoryType>git</RepositoryType> <RepositoryUrl>https://github.com/google/protobuf.git</RepositoryUrl> - <IncludeSymbols>true</IncludeSymbols> - <IncludeSource>true</IncludeSource> </PropertyGroup> <!-- @@ -30,4 +28,8 @@ <TargetFrameworks>netstandard1.0</TargetFrameworks> </PropertyGroup> + <ItemGroup> + <PackageReference Include="SourceLink.Create.CommandLine" Version="2.7.6" PrivateAssets="All" /> + </ItemGroup> + </Project> diff --git a/docs/performance.md b/docs/performance.md index 6a5316c8..3bca58af 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,4 +1,4 @@ -# Protobuf Perforamcne +# Protobuf Performance This benchmark result is tested on workstation with processor of Intel® Xeon® Processor E5-2630 and 32GB RAM This table contains 3 languages' results: diff --git a/docs/third_party.md b/docs/third_party.md index 3f267dde..4eeb2787 100644 --- a/docs/third_party.md +++ b/docs/third_party.md @@ -25,6 +25,7 @@ These are projects we know about implementing Protocol Buffers for other program * Clojure: http://github.com/ninjudd/clojure-protobuf * Common Lisp: http://github.com/ndantam/s-protobuf * Common Lisp: http://github.com/brown/protobuf +* D: https://github.com/dcarp/protobuf-d * D: https://github.com/msoucy/dproto * D: http://256.makerslocal.org/wiki/index.php/ProtocolBuffer * D: https://github.com/opticron/ProtocolBuffer @@ -39,6 +40,7 @@ These are projects we know about implementing Protocol Buffers for other program * Erlang: https://code.google.com/p/protoc-gen-erl/ * Erlang: https://github.com/basho/erlang_protobuffs * Erlang: https://github.com/tomas-abrahamsson/gpb +* GDScript: https://github.com/oniksan/godobuf (Godot v3 engine plugin) * Go: https://github.com/golang/protobuf (Google-official implementation) * Go: https://github.com/akunspy/gopbuf * Go: https://github.com/gogo/protobuf @@ -87,7 +89,7 @@ These are projects we know about implementing Protocol Buffers for other program * Scala: http://github.com/jeffplaisance/scala-protobuf * Scala: http://code.google.com/p/protobuf-scala * Scala: https://github.com/SandroGrzicic/ScalaBuff -* Scala: http://trueaccord.github.io/ScalaPB/ +* Scala: https://scalapb.github.io * Swift: https://github.com/alexeyxo/protobuf-swift * Swift: https://github.com/apple/swift-protobuf/ * Vala: https://launchpad.net/protobuf-vala diff --git a/generate_changelog.py b/generate_changelog.py index d90a9b70..8e5bf423 100755 --- a/generate_changelog.py +++ b/generate_changelog.py @@ -20,11 +20,10 @@ languages = [ ]), Language("Java", [ "java", - "javanano", - "src/google/protobuf/compiler/cpp", + "src/google/protobuf/compiler/java", ]), Language("Python", [ - "javanano", + "python", "src/google/protobuf/compiler/python", ]), Language("JavaScript", [ diff --git a/java/core/pom.xml b/java/core/pom.xml index b4edcbc7..a7d4ea37 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -135,6 +135,7 @@ <Bundle-DocURL>https://developers.google.com/protocol-buffers/</Bundle-DocURL> <Bundle-SymbolicName>com.google.protobuf</Bundle-SymbolicName> <Export-Package>com.google.protobuf;version=${project.version}</Export-Package> + <Import-Package>sun.misc;resolution:=optional,*</Import-Package> </instructions> </configuration> </plugin> diff --git a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java index c9e2904b..d84ef3c5 100644 --- a/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java +++ b/java/core/src/main/java/com/google/protobuf/UnsafeUtil.java @@ -33,7 +33,6 @@ package com.google.protobuf; import java.lang.reflect.Field; import java.nio.Buffer; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.security.AccessController; import java.security.PrivilegedExceptionAction; import java.util.logging.Level; @@ -72,8 +71,6 @@ final class UnsafeUtil { private static final long BUFFER_ADDRESS_OFFSET = fieldOffset(bufferAddressField()); - private static final long STRING_VALUE_OFFSET = fieldOffset(stringValueField()); - private UnsafeUtil() {} static boolean hasUnsafeArrayOperations() { @@ -149,10 +146,6 @@ final class UnsafeUtil { return MEMORY_ACCESSOR.getObject(target, offset); } - static void putObject(Object target, long offset, Object value) { - MEMORY_ACCESSOR.putObject(target, offset, value); - } - static byte getByte(byte[] target, long index) { return MEMORY_ACCESSOR.getByte(target, BYTE_ARRAY_BASE_OFFSET + index); } @@ -262,26 +255,6 @@ final class UnsafeUtil { return MEMORY_ACCESSOR.getLong(buffer, BUFFER_ADDRESS_OFFSET); } - /** - * Returns a new {@link String} backed by the given {@code chars}. The char array should not - * be mutated any more after calling this function. - */ - static String moveToString(char[] chars) { - if (STRING_VALUE_OFFSET == -1) { - // In the off-chance that this JDK does not implement String as we'd expect, just do a copy. - return new String(chars); - } - final String str; - try { - str = (String) UNSAFE.allocateInstance(String.class); - } catch (InstantiationException e) { - // This should never happen, but return a copy as a fallback just in case. - return new String(chars); - } - putObject(str, STRING_VALUE_OFFSET, chars); - return str; - } - static Object getStaticObject(Field field) { return MEMORY_ACCESSOR.getStaticObject(field); } diff --git a/java/core/src/main/java/com/google/protobuf/Utf8.java b/java/core/src/main/java/com/google/protobuf/Utf8.java index 6968abb3..de75fe6b 100644 --- a/java/core/src/main/java/com/google/protobuf/Utf8.java +++ b/java/core/src/main/java/com/google/protobuf/Utf8.java @@ -1474,10 +1474,7 @@ final class Utf8 { } } - if (resultPos < resultArr.length) { - resultArr = Arrays.copyOf(resultArr, resultPos); - } - return UnsafeUtil.moveToString(resultArr); + return new String(resultArr, 0, resultPos); } @Override @@ -1553,10 +1550,7 @@ final class Utf8 { } } - if (resultPos < resultArr.length) { - resultArr = Arrays.copyOf(resultArr, resultPos); - } - return UnsafeUtil.moveToString(resultArr); + return new String(resultArr, 0, resultPos); } @Override diff --git a/kokoro/linux/bazel/build.sh b/kokoro/linux/bazel/build.sh new file mode 100755 index 00000000..d8aea724 --- /dev/null +++ b/kokoro/linux/bazel/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +git submodule update --init --recursive +bazel test :protobuf_test diff --git a/kokoro/macos/jruby/continuous.cfg b/kokoro/linux/bazel/continuous.cfg index f1310fd3..13cfef15 100644 --- a/kokoro/macos/jruby/continuous.cfg +++ b/kokoro/linux/bazel/continuous.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/jruby/build.sh" -timeout_mins: 1440 +build_file: "protobuf/kokoro/linux/bazel/build.sh" +timeout_mins: 15 diff --git a/kokoro/macos/jruby/presubmit.cfg b/kokoro/linux/bazel/presubmit.cfg index f1310fd3..13cfef15 100644 --- a/kokoro/macos/jruby/presubmit.cfg +++ b/kokoro/linux/bazel/presubmit.cfg @@ -1,5 +1,5 @@ # Config file for running tests in Kokoro # Location of the build script in repository -build_file: "protobuf/kokoro/macos/jruby/build.sh" -timeout_mins: 1440 +build_file: "protobuf/kokoro/linux/bazel/build.sh" +timeout_mins: 15 diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh index 750c3e5c..af5b299e 100755 --- a/kokoro/linux/benchmark/build.sh +++ b/kokoro/linux/benchmark/build.sh @@ -27,8 +27,8 @@ cd $oldpwd ./configure CXXFLAGS="-fPIC -O2" make -j8 cd python -python setup.py -q build --cpp_implementation -pip install . +python setup.py build --cpp_implementation +pip install . --user # build and run Python benchmark @@ -41,10 +41,10 @@ echo "benchmarking pure python..." ./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json echo "," >> "tmp/python_result.json" echo "benchmarking python cpp reflection..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json echo "," >> "tmp/python_result.json" echo "benchmarking python cpp generated code..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json echo "]" >> "tmp/python_result.json" cd $oldpwd @@ -52,6 +52,11 @@ cd $oldpwd ./configure make clean && make -j8 +# build Java protobuf +cd java +mvn package +cd .. + # build CPP benchmark cd benchmarks mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp @@ -81,7 +86,7 @@ echo "benchmarking java..." # upload result to bq make python_add_init -python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \ +env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \ -python="../tmp/python_result.json" -go="../tmp/go_result.txt" cd $oldpwd diff --git a/kokoro/macos/jruby/build.sh b/kokoro/macos/jruby/build.sh deleted file mode 100755 index c82eaebf..00000000 --- a/kokoro/macos/jruby/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# -# Build file to set up and run tests - -# Change to repo root -cd $(dirname $0)/../../.. - -# Prepare worker environment to run tests -source kokoro/macos/prepare_build_macos_rc - -./tests.sh jruby diff --git a/objectivec/GPBDescriptor.h b/objectivec/GPBDescriptor.h index 651f4de0..292bce13 100644 --- a/objectivec/GPBDescriptor.h +++ b/objectivec/GPBDescriptor.h @@ -223,9 +223,12 @@ typedef NS_ENUM(uint8_t, GPBFieldType) { /** * Returns the enum value name for the given raw enum. * + * Note that there can be more than one name corresponding to a given value + * if the allow_alias option is used. + * * @param number The raw enum value. * - * @return The name of the enum value passed, or nil if not valid. + * @return The first name that matches the enum value passed, or nil if not valid. **/ - (nullable NSString *)enumNameForValue:(int32_t)number; @@ -244,7 +247,7 @@ typedef NS_ENUM(uint8_t, GPBFieldType) { * * @param number The raw enum value. * - * @return The text format name for the raw enum value, or nil if not valid. + * @return The first text format name which matches the enum value, or nil if not valid. **/ - (nullable NSString *)textFormatNameForValue:(int32_t)number; @@ -258,6 +261,33 @@ typedef NS_ENUM(uint8_t, GPBFieldType) { **/ - (BOOL)getValue:(nullable int32_t *)outValue forEnumTextFormatName:(NSString *)textFormatName; +/** + * Gets the number of defined enum names. + * + * @return Count of the number of enum names, including any aliases. + */ +@property(nonatomic, readonly) uint32_t enumNameCount; + +/** + * Gets the enum name corresponding to the given index. + * + * @param index Index into the available names. The defined range is from 0 + * to self.enumNameCount - 1. + * + * @returns The enum name at the given index, or nil if the index is out of range. + */ +- (nullable NSString *)getEnumNameForIndex:(uint32_t)index; + +/** + * Gets the enum text format name corresponding to the given index. + * + * @param index Index into the available names. The defined range is from 0 + * to self.enumNameCount - 1. + * + * @returns The text format name at the given index, or nil if the index is out of range. + */ +- (nullable NSString *)getEnumTextFormatNameForIndex:(uint32_t)index; + @end /** diff --git a/objectivec/GPBDescriptor.m b/objectivec/GPBDescriptor.m index 4b39c63b..ad46ef4f 100644 --- a/objectivec/GPBDescriptor.m +++ b/objectivec/GPBDescriptor.m @@ -830,13 +830,9 @@ uint32_t GPBFieldAlternateTag(GPBFieldDescriptor *self) { } - (NSString *)enumNameForValue:(int32_t)number { - if (nameOffsets_ == NULL) [self calcValueNameOffsets]; - for (uint32_t i = 0; i < valueCount_; ++i) { if (values_[i] == number) { - const char *valueName = valueNames_ + nameOffsets_[i]; - NSString *fullName = [NSString stringWithFormat:@"%@_%s", name_, valueName]; - return fullName; + return [self getEnumNameForIndex:i]; } } return nil; @@ -886,8 +882,6 @@ uint32_t GPBFieldAlternateTag(GPBFieldDescriptor *self) { } - (NSString *)textFormatNameForValue:(int32_t)number { - if (nameOffsets_ == NULL) [self calcValueNameOffsets]; - // Find the EnumValue descriptor and its index. BOOL foundIt = NO; uint32_t valueDescriptorIndex; @@ -902,16 +896,39 @@ uint32_t GPBFieldAlternateTag(GPBFieldDescriptor *self) { if (!foundIt) { return nil; } + return [self getEnumTextFormatNameForIndex:valueDescriptorIndex]; +} + +- (uint32_t)enumNameCount { + return valueCount_; +} + +- (NSString *)getEnumNameForIndex:(uint32_t)index { + if (nameOffsets_ == NULL) [self calcValueNameOffsets]; + if (index >= valueCount_) { + return nil; + } + const char *valueName = valueNames_ + nameOffsets_[index]; + NSString *fullName = [NSString stringWithFormat:@"%@_%s", name_, valueName]; + return fullName; +} + +- (NSString *)getEnumTextFormatNameForIndex:(uint32_t)index { + if (nameOffsets_ == NULL) [self calcValueNameOffsets]; + + if (index >= valueCount_) { + return nil; + } NSString *result = nil; // Naming adds an underscore between enum name and value name, skip that also. - const char *valueName = valueNames_ + nameOffsets_[valueDescriptorIndex]; + const char *valueName = valueNames_ + nameOffsets_[index]; NSString *shortName = @(valueName); // See if it is in the map of special format handling. if (extraTextFormatInfo_) { result = GPBDecodeTextFormatName(extraTextFormatInfo_, - (int32_t)valueDescriptorIndex, shortName); + (int32_t)index, shortName); } // Logic here needs to match what objectivec_enum.cc does in the proto // compiler. diff --git a/objectivec/README.md b/objectivec/README.md index a02df33f..21ac07d8 100644 --- a/objectivec/README.md +++ b/objectivec/README.md @@ -20,9 +20,10 @@ The Objective C implementation requires: Installation ------------ -The full distribution pulled from github includes the sources for both the -compiler (protoc) and the runtime (this directory). To build the compiler -and run the runtime tests, you can use: +The distribution pulled from github includes the sources for both the +compiler (protoc) and the runtime (this directory). After cloning the distribution +and needed submodules ([see the src directory's README](../src/README.md)), +to build the compiler and run the runtime tests, you can use: $ objectivec/DevTools/full_mac_build.sh diff --git a/objectivec/Tests/GPBDescriptorTests.m b/objectivec/Tests/GPBDescriptorTests.m index 199ea655..d47cc30f 100644 --- a/objectivec/Tests/GPBDescriptorTests.m +++ b/objectivec/Tests/GPBDescriptorTests.m @@ -190,6 +190,63 @@ XCTAssertFalse([descriptor getValue:&value forEnumTextFormatName:@"Unknown"]); } +- (void)testEnumDescriptorIntrospection { + GPBEnumDescriptor *descriptor = TestAllTypes_NestedEnum_EnumDescriptor(); + + XCTAssertEqual(descriptor.enumNameCount, 4U); + XCTAssertEqualObjects([descriptor getEnumNameForIndex:0], + @"TestAllTypes_NestedEnum_Foo"); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:0], @"FOO"); + XCTAssertEqualObjects([descriptor getEnumNameForIndex:1], + @"TestAllTypes_NestedEnum_Bar"); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:1], @"BAR"); + XCTAssertEqualObjects([descriptor getEnumNameForIndex:2], + @"TestAllTypes_NestedEnum_Baz"); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:2], @"BAZ"); + XCTAssertEqualObjects([descriptor getEnumNameForIndex:3], + @"TestAllTypes_NestedEnum_Neg"); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:3], @"NEG"); +} + +- (void)testEnumDescriptorIntrospectionWithAlias { + GPBEnumDescriptor *descriptor = TestEnumWithDupValue_EnumDescriptor(); + NSString *enumName; + int32_t value; + + XCTAssertEqual(descriptor.enumNameCount, 5U); + + enumName = [descriptor getEnumNameForIndex:0]; + XCTAssertEqualObjects(enumName, @"TestEnumWithDupValue_Foo1"); + XCTAssertTrue([descriptor getValue:&value forEnumName:enumName]); + XCTAssertEqual(value, 1); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:0], @"FOO1"); + + enumName = [descriptor getEnumNameForIndex:1]; + XCTAssertEqualObjects(enumName, @"TestEnumWithDupValue_Bar1"); + XCTAssertTrue([descriptor getValue:&value forEnumName:enumName]); + XCTAssertEqual(value, 2); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:1], @"BAR1"); + + enumName = [descriptor getEnumNameForIndex:2]; + XCTAssertEqualObjects(enumName, @"TestEnumWithDupValue_Baz"); + XCTAssertTrue([descriptor getValue:&value forEnumName:enumName]); + XCTAssertEqual(value, 3); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:2], @"BAZ"); + + enumName = [descriptor getEnumNameForIndex:3]; + XCTAssertEqualObjects(enumName, @"TestEnumWithDupValue_Foo2"); + XCTAssertTrue([descriptor getValue:&value forEnumName:enumName]); + XCTAssertEqual(value, 1); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:3], @"FOO2"); + + enumName = [descriptor getEnumNameForIndex:4]; + XCTAssertEqualObjects(enumName, @"TestEnumWithDupValue_Bar2"); + XCTAssertTrue([descriptor getValue:&value forEnumName:enumName]); + XCTAssertEqual(value, 2); + XCTAssertEqualObjects([descriptor getEnumTextFormatNameForIndex:4], @"BAR2"); + +} + - (void)testEnumValueValidator { GPBDescriptor *descriptor = [TestAllTypes descriptor]; GPBFieldDescriptor *fieldDescriptor = diff --git a/php/src/Google/Protobuf/Internal/GPBUtil.php b/php/src/Google/Protobuf/Internal/GPBUtil.php index ec0bf6bd..b75d9bab 100644 --- a/php/src/Google/Protobuf/Internal/GPBUtil.php +++ b/php/src/Google/Protobuf/Internal/GPBUtil.php @@ -305,8 +305,11 @@ class GPBUtil $name, $file_proto) { - $classname = implode('_', explode('.', $name)); - return static::getClassNamePrefix($classname, $file_proto) . $classname; + $parts = explode('.', $name); + foreach ($parts as $i => $part) { + $parts[$i] = static::getClassNamePrefix($parts[$i], $file_proto) . $parts[$i]; + } + return implode('\\', $parts); } public static function getClassNameWithoutPackage( diff --git a/php/src/Google/Protobuf/Timestamp.php b/php/src/Google/Protobuf/Timestamp.php index e21f3231..a793c7e3 100644 --- a/php/src/Google/Protobuf/Timestamp.php +++ b/php/src/Google/Protobuf/Timestamp.php @@ -55,7 +55,9 @@ use Google\Protobuf\Internal\GPBUtil; * {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional * seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), * are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone - * is required, though only UTC (as indicated by "Z") is presently supported. + * is required. A proto3 JSON serializer should always use UTC (as indicated by + * "Z") when printing the Timestamp type and a proto3 JSON parser should be + * able to accept both UTC and other timezones (as indicated by an offset). * For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past * 01:30 UTC on January 15, 2017. * In JavaScript, one can convert a Date object to this format using the @@ -64,8 +66,8 @@ use Google\Protobuf\Internal\GPBUtil; * to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) * with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one * can use the Joda Time's [`ISODateTimeFormat.dateTime()`]( - * http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--) - * to obtain a formatter capable of generating timestamps in this format. + * http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime-- + * ) to obtain a formatter capable of generating timestamps in this format. * * Generated from protobuf message <code>google.protobuf.Timestamp</code> */ diff --git a/protobuf.bzl b/protobuf.bzl index 78f19c62..8bbd0564 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,3 +1,5 @@ +load("@bazel_skylib//:lib.bzl", "versions") + def _GetPath(ctx, path): if ctx.label.workspace_root: return ctx.label.workspace_root + '/' + path @@ -266,8 +268,8 @@ def internal_gen_well_known_protos_java(srcs): Args: srcs: the well known protos """ - root = Label("%s//protobuf_java" % (REPOSITORY_NAME)).workspace_root - pkg = PACKAGE_NAME + "/" if PACKAGE_NAME else "" + root = Label("%s//protobuf_java" % (native.repository_name())).workspace_root + pkg = native.package_name() + "/" if native.package_name() else "" if root == "": include = " -I%ssrc " % pkg else: @@ -411,7 +413,4 @@ def check_protobuf_required_bazel_version(): This ensures bazel supports our approach to proto_library() depending on a copied filegroup. (Fixed in bazel 0.5.4) """ - expected = apple_common.dotted_version("0.5.4") - current = apple_common.dotted_version(native.bazel_version) - if current.compare_to(expected) < 0: - fail("Bazel must be newer than 0.5.4") + versions.check(minimum_bazel_version = "0.5.4") diff --git a/python/google/protobuf/proto_api.h b/python/google/protobuf/proto_api.h new file mode 100644 index 00000000..64d8dda9 --- /dev/null +++ b/python/google/protobuf/proto_api.h @@ -0,0 +1,92 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file can be included by other C++ libraries, typically extension modules +// which want to interact with the Python Messages coming from the "cpp" +// implementation of protocol buffers. +// +// Usage: +// Declare a (probably static) variable to hold the API: +// const PyProto_API* py_proto_api; +// In some initialization function, write: +// py_proto_api = static_cast<const PyProto_API*>(PyCapsule_Import( +// PyProtoAPICapsuleName(), 0)); +// if (!py_proto_api) { ...handle ImportError... } +// Then use the methods of the returned class: +// py_proto_api->GetMessagePointer(...); + +#ifndef PYTHON_GOOGLE_PROTOBUF_PROTO_API_H__ +#define PYTHON_GOOGLE_PROTOBUF_PROTO_API_H__ + +#include <Python.h> + +namespace google { +namespace protobuf { + +class Message; + +namespace python { + +// Note on the implementation: +// This API is designed after +// https://docs.python.org/3/extending/extending.html#providing-a-c-api-for-an-extension-module +// The class below contains no mutable state, and all methods are "const"; +// we use a C++ class instead of a C struct with functions pointers just because +// the code looks more readable. +struct PyProto_API { + // The API object is created at initialization time and never freed. + // This destructor is never called. + virtual ~PyProto_API() {} + + // Operations on Messages. + + // If the passed object is a Python Message, returns its internal pointer. + // Otherwise, returns NULL with an exception set. + virtual const Message* GetMessagePointer(PyObject* msg) const = 0; + + // If the passed object is a Python Message, returns a mutable pointer. + // Otherwise, returns NULL with an exception set. + // This function will succeed only if there are no other Python objects + // pointing to the message, like submessages or repeated containers. + // With the current implementation, only empty messages are in this case. + virtual Message* GetMutableMessagePointer(PyObject* msg) const = 0; +}; + +inline const char* PyProtoAPICapsuleName() { + static const char kCapsuleName[] = + "protobuf.python.google.protobuf.cpp._message.proto_API"; + return kCapsuleName; +} + +} // namespace python +} // namespace protobuf +} // namespace google + +#endif // PYTHON_GOOGLE_PROTOBUF_PROTO_API_H__ diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 5893533a..53736b9c 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -2866,17 +2866,38 @@ const Message* (*GetCProtoInsidePyProtoPtr)(PyObject* msg); Message* (*MutableCProtoInsidePyProtoPtr)(PyObject* msg); static const Message* GetCProtoInsidePyProtoImpl(PyObject* msg) { + const Message* message = PyMessage_GetMessagePointer(msg); + if (message == NULL) { + PyErr_Clear(); + return NULL; + } + return message; +} + +static Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { + Message* message = PyMessage_GetMutableMessagePointer(msg); + if (message == NULL) { + PyErr_Clear(); + return NULL; + } + return message; +} + +const Message* PyMessage_GetMessagePointer(PyObject* msg) { if (!PyObject_TypeCheck(msg, &CMessage_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a Message instance"); return NULL; } CMessage* cmsg = reinterpret_cast<CMessage*>(msg); return cmsg->message; } -static Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { +Message* PyMessage_GetMutableMessagePointer(PyObject* msg) { if (!PyObject_TypeCheck(msg, &CMessage_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a Message instance"); return NULL; } + CMessage* cmsg = reinterpret_cast<CMessage*>(msg); if ((cmsg->composite_fields && PyDict_Size(cmsg->composite_fields) != 0) || (cmsg->extensions != NULL && @@ -2885,6 +2906,9 @@ static Message* MutableCProtoInsidePyProtoImpl(PyObject* msg) { // the underlying C++ message back to the CMessage (e.g. removed repeated // composite containers). We only allow direct mutation of the underlying // C++ message if there is no child data in the CMessage. + PyErr_SetString(PyExc_ValueError, + "Cannot reliably get a mutable pointer " + "to a message with extra references"); return NULL; } cmessage::AssureWritable(cmsg); diff --git a/python/google/protobuf/pyext/message.h b/python/google/protobuf/pyext/message.h index 72bcfa83..d754e62a 100644 --- a/python/google/protobuf/pyext/message.h +++ b/python/google/protobuf/pyext/message.h @@ -341,6 +341,9 @@ bool CheckFieldBelongsToMessage(const FieldDescriptor* field_descriptor, extern PyObject* PickleError_class; +const Message* PyMessage_GetMessagePointer(PyObject* msg); +Message* PyMessage_GetMutableMessagePointer(PyObject* msg); + bool InitProto2MessageModule(PyObject *m); #if LANG_CXX11 diff --git a/python/google/protobuf/pyext/message_module.cc b/python/google/protobuf/pyext/message_module.cc index 7c4df47f..f5c8f295 100644 --- a/python/google/protobuf/pyext/message_module.cc +++ b/python/google/protobuf/pyext/message_module.cc @@ -31,9 +31,26 @@ #include <Python.h> #include <google/protobuf/pyext/message.h> +#include <google/protobuf/proto_api.h> #include <google/protobuf/message_lite.h> +namespace { + +// C++ API. Clients get at this via proto_api.h +struct ApiImplementation : google::protobuf::python::PyProto_API { + const google::protobuf::Message* + GetMessagePointer(PyObject* msg) const override { + return google::protobuf::python::PyMessage_GetMessagePointer(msg); + } + google::protobuf::Message* + GetMutableMessagePointer(PyObject* msg) const override { + return google::protobuf::python::PyMessage_GetMutableMessagePointer(msg); + } +}; + +} // namespace + static PyObject* GetPythonProto3PreserveUnknownsDefault( PyObject* /*m*/, PyObject* /*args*/) { if (google::protobuf::internal::GetProto3PreserveUnknownsDefault()) { diff --git a/python/setup.py b/python/setup.py index a9df075e..6f8c83f0 100755 --- a/python/setup.py +++ b/python/setup.py @@ -44,6 +44,7 @@ def GetVersion(): with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: exec(version_file.read(), globals()) + global __version__ return __version__ diff --git a/python/stubout.py b/python/stubout.py index aee4f2da..ba391045 100755 --- a/python/stubout.py +++ b/python/stubout.py @@ -17,6 +17,9 @@ # This file is used for testing. The original is at: # http://code.google.com/p/pymox/ +import inspect + + class StubOutForTesting: """Sample Usage: You want os.path.exists() to always return true during testing. diff --git a/ruby/lib/google/protobuf/well_known_types.rb b/ruby/lib/google/protobuf/well_known_types.rb index 921ddbc0..3e759591 100644 --- a/ruby/lib/google/protobuf/well_known_types.rb +++ b/ruby/lib/google/protobuf/well_known_types.rb @@ -39,6 +39,12 @@ module Google module Protobuf Any.class_eval do + def self.pack(msg, type_url_prefix='type.googleapis.com/') + any = self.new + any.pack(msg, type_url_prefix) + any + end + def pack(msg, type_url_prefix='type.googleapis.com/') if type_url_prefix.empty? or type_url_prefix[-1] != '/' then self.type_url = "#{type_url_prefix}/#{msg.class.descriptor.name}" diff --git a/ruby/tests/well_known_types_test.rb b/ruby/tests/well_known_types_test.rb index bd24c328..240281e7 100644 --- a/ruby/tests/well_known_types_test.rb +++ b/ruby/tests/well_known_types_test.rb @@ -120,11 +120,17 @@ class TestWellKnownTypes < Test::Unit::TestCase end def test_any - any = Google::Protobuf::Any.new ts = Google::Protobuf::Timestamp.new(seconds: 12345, nanos: 6789) + + any = Google::Protobuf::Any.new any.pack(ts) assert any.is(Google::Protobuf::Timestamp) assert_equal ts, any.unpack(Google::Protobuf::Timestamp) + + any = Google::Protobuf::Any.pack(ts) + + assert any.is(Google::Protobuf::Timestamp) + assert_equal ts, any.unpack(Google::Protobuf::Timestamp) end end diff --git a/src/Makefile.am b/src/Makefile.am index fb247300..09ab76f7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -56,9 +56,7 @@ clean-local: CLEANFILES = $(protoc_outputs) unittest_proto_middleman \ testzip.jar testzip.list testzip.proto testzip.zip \ - no_warning_test.cc \ - google/protobuf/compiler/js/well_known_types_embed.cc \ - js_embed$(EXEEXT) + no_warning_test.cc MAINTAINERCLEANFILES = \ Makefile.in @@ -471,22 +469,6 @@ bin_PROGRAMS = protoc protoc_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la protoc_SOURCES = google/protobuf/compiler/main.cc -# The special JS code for the well-known types is linked into the compiler via -# well_known_types_embed.cc, which is generated from .js source files. We have -# to build the js_embed binary using $(CXX_FOR_BUILD) so that it is executable -# on the build machine in a cross-compilation setup. -js_embed$(EXEEXT): $(srcdir)/google/protobuf/compiler/js/embed.cc - $(CXX_FOR_BUILD) -o $@ $< -js_well_known_types_sources = \ - google/protobuf/compiler/js/well_known_types/any.js \ - google/protobuf/compiler/js/well_known_types/struct.js \ - google/protobuf/compiler/js/well_known_types/timestamp.js -# We have to cd to $(srcdir) so that out-of-tree builds work properly. -google/protobuf/compiler/js/well_known_types_embed.cc: js_embed$(EXEEXT) $(js_well_known_types_sources) - mkdir -p `dirname $@` && \ - oldpwd=`pwd` && cd $(srcdir) && \ - $$oldpwd/js_embed$(EXEEXT) $(js_well_known_types_sources) > $$oldpwd/$@ - # Tests ============================================================== protoc_inputs = \ @@ -565,7 +547,6 @@ EXTRA_DIST = \ google/protobuf/package_info.h \ google/protobuf/io/package_info.h \ google/protobuf/util/package_info.h \ - google/protobuf/compiler/js/embed.cc \ google/protobuf/compiler/ruby/ruby_generated_code.proto \ google/protobuf/compiler/ruby/ruby_generated_code_pb.rb \ google/protobuf/compiler/package_info.h \ diff --git a/src/google/protobuf/any.pb.cc b/src/google/protobuf/any.pb.cc index 9d632efe..d0fc2905 100644 --- a/src/google/protobuf/any.pb.cc +++ b/src/google/protobuf/any.pb.cc @@ -66,14 +66,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Any_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/any.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -84,7 +84,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\031google/protobuf/any.proto\022\017google.prot" diff --git a/src/google/protobuf/api.pb.cc b/src/google/protobuf/api.pb.cc index e0a249d0..6ad9f48d 100644 --- a/src/google/protobuf/api.pb.cc +++ b/src/google/protobuf/api.pb.cc @@ -149,14 +149,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Mixin_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/api.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -167,7 +167,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 3); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\031google/protobuf/api.proto\022\017google.prot" diff --git a/src/google/protobuf/compiler/cpp/cpp_file.cc b/src/google/protobuf/compiler/cpp/cpp_file.cc index 42525687..02f360bb 100644 --- a/src/google/protobuf/compiler/cpp/cpp_file.cc +++ b/src/google/protobuf/compiler/cpp/cpp_file.cc @@ -791,7 +791,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) { // AssignDescriptors(). All later times, waits for the first call to // complete and then returns. printer->Print( - "void protobuf_AssignDescriptors() {\n" + "static void protobuf_AssignDescriptors() {\n" // Make sure the file has found its way into the pool. If a descriptor // is requested *during* static init then AddDescriptors() may not have // been called yet, so we call it manually. Note that it's fine if @@ -812,7 +812,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) { printer->Print( "}\n" "\n" - "void protobuf_AssignDescriptorsOnce() {\n" + "static void protobuf_AssignDescriptorsOnce() {\n" " static ::google::protobuf::internal::once_flag once;\n" " ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors);\n" "}\n" @@ -850,7 +850,7 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* printer) { // Now generate the AddDescriptors() function. printer->Print( - "void AddDescriptorsImpl() {\n" + "static void AddDescriptorsImpl() {\n" " InitDefaults();\n"); printer->Indent(); diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc index 8c2336af..778fc406 100644 --- a/src/google/protobuf/compiler/cpp/cpp_message.cc +++ b/src/google/protobuf/compiler/cpp/cpp_message.cc @@ -2280,7 +2280,7 @@ GenerateArenaDestructorCode(io::Printer* printer) { "classname", classname_); } else { printer->Print( - "void $classname$::RegisterArenaDtor(::google::protobuf::Arena* arena) {\n" + "void $classname$::RegisterArenaDtor(::google::protobuf::Arena*) {\n" "}\n", "classname", classname_); } diff --git a/src/google/protobuf/compiler/csharp/csharp_helpers.cc b/src/google/protobuf/compiler/csharp/csharp_helpers.cc index 5bca1ffa..04b61074 100644 --- a/src/google/protobuf/compiler/csharp/csharp_helpers.cc +++ b/src/google/protobuf/compiler/csharp/csharp_helpers.cc @@ -169,7 +169,7 @@ std::string UnderscoresToCamelCase(const std::string& input, } } // Add a trailing "_" if the name should be altered. - if (input[input.size() - 1] == '#') { + if (input.size() > 0 && input[input.size() - 1] == '#') { result += '_'; } return result; diff --git a/src/google/protobuf/compiler/java/java_enum_field.cc b/src/google/protobuf/compiler/java/java_enum_field.cc index 0686ea0f..ef64d88b 100644 --- a/src/google/protobuf/compiler/java/java_enum_field.cc +++ b/src/google/protobuf/compiler/java/java_enum_field.cc @@ -305,11 +305,15 @@ GenerateBuildingCode(io::Printer* printer) const { if (SupportFieldPresence(descriptor_->file())) { printer->Print(variables_, "if ($get_has_field_bit_from_local$) {\n" + " result.$name$_ = $name$_;\n" " $set_has_field_bit_to_local$;\n" + "} else {\n" + " result.$name$_ = $default_number$;\n" "}\n"); + } else { + printer->Print(variables_, + "result.$name$_ = $name$_;\n"); } - printer->Print(variables_, - "result.$name$_ = $name$_;\n"); } void ImmutableEnumFieldGenerator:: diff --git a/src/google/protobuf/compiler/java/java_lazy_message_field.cc b/src/google/protobuf/compiler/java/java_lazy_message_field.cc index abf8e55c..6544bea0 100644 --- a/src/google/protobuf/compiler/java/java_lazy_message_field.cc +++ b/src/google/protobuf/compiler/java/java_lazy_message_field.cc @@ -233,12 +233,9 @@ void ImmutableLazyMessageFieldGenerator:: GenerateBuildingCode(io::Printer* printer) const { printer->Print(variables_, "if ($get_has_field_bit_from_local$) {\n" + " result.$name$_.set($name$_);\n" " $set_has_field_bit_to_local$;\n" "}\n"); - - printer->Print(variables_, - "result.$name$_.set(\n" - " $name$_);\n"); } void ImmutableLazyMessageFieldGenerator:: diff --git a/src/google/protobuf/compiler/java/java_message_field.cc b/src/google/protobuf/compiler/java/java_message_field.cc index baa7f872..bda4fcc0 100644 --- a/src/google/protobuf/compiler/java/java_message_field.cc +++ b/src/google/protobuf/compiler/java/java_message_field.cc @@ -255,7 +255,7 @@ GenerateBuilderMembers(io::Printer* printer) const { bool support_field_presence = SupportFieldPresence(descriptor_->file()); printer->Print(variables_, - "private $type$ $name$_ = null;\n"); + "private $type$ $name$_;\n"); printer->Print(variables_, // If this builder is non-null, it is used and the other fields are @@ -444,15 +444,20 @@ void ImmutableMessageFieldGenerator:: GenerateBuildingCode(io::Printer* printer) const { if (SupportFieldPresence(descriptor_->file())) { printer->Print(variables_, - "if ($get_has_field_bit_from_local$) {\n" - " $set_has_field_bit_to_local$;\n" - "}\n"); + "if ($get_has_field_bit_from_local$) {\n"); + printer->Indent(); + PrintNestedBuilderCondition(printer, + "result.$name$_ = $name$_;\n", + "result.$name$_ = $name$Builder_.build();\n"); + printer->Outdent(); + printer->Print(variables_, + " $set_has_field_bit_to_local$;\n" + "}\n"); + } else { + PrintNestedBuilderCondition(printer, + "result.$name$_ = $name$_;\n", + "result.$name$_ = $name$Builder_.build();\n"); } - - PrintNestedBuilderCondition(printer, - "result.$name$_ = $name$_;\n", - - "result.$name$_ = $name$Builder_.build();\n"); } void ImmutableMessageFieldGenerator:: diff --git a/src/google/protobuf/compiler/java/java_primitive_field.cc b/src/google/protobuf/compiler/java/java_primitive_field.cc index 71ee0992..e6ce69c7 100644 --- a/src/google/protobuf/compiler/java/java_primitive_field.cc +++ b/src/google/protobuf/compiler/java/java_primitive_field.cc @@ -264,7 +264,9 @@ GenerateFieldBuilderInitializationCode(io::Printer* printer) const { void ImmutablePrimitiveFieldGenerator:: GenerateInitializationCode(io::Printer* printer) const { - printer->Print(variables_, "$name$_ = $default$;\n"); + if (!IsDefaultValueJavaDefault(descriptor_)) { + printer->Print(variables_, "$name$_ = $default$;\n"); + } } void ImmutablePrimitiveFieldGenerator:: @@ -294,11 +296,21 @@ GenerateBuildingCode(io::Printer* printer) const { if (SupportFieldPresence(descriptor_->file())) { printer->Print(variables_, "if ($get_has_field_bit_from_local$) {\n" - " $set_has_field_bit_to_local$;\n" - "}\n"); + " result.$name$_ = $name$_;\n" + " $set_has_field_bit_to_local$;\n"); + if (IsDefaultValueJavaDefault(descriptor_)) { + printer->Print(variables_, + "}\n"); + } else { + printer->Print(variables_, + "} else {\n" + " result.$name$_ = $default$;\n" + "}\n"); + } + } else { + printer->Print(variables_, + "result.$name$_ = $name$_;\n"); } - printer->Print(variables_, - "result.$name$_ = $name$_;\n"); } void ImmutablePrimitiveFieldGenerator:: diff --git a/src/google/protobuf/compiler/java/java_string_field.cc b/src/google/protobuf/compiler/java/java_string_field.cc index 2b6e9381..b08febc0 100644 --- a/src/google/protobuf/compiler/java/java_string_field.cc +++ b/src/google/protobuf/compiler/java/java_string_field.cc @@ -405,11 +405,15 @@ GenerateBuildingCode(io::Printer* printer) const { if (SupportFieldPresence(descriptor_->file())) { printer->Print(variables_, "if ($get_has_field_bit_from_local$) {\n" + " result.$name$_ = $name$_;\n" " $set_has_field_bit_to_local$;\n" + "} else {\n" + " result.$name$_ = $default$;\n" "}\n"); + } else { + printer->Print(variables_, + "result.$name$_ = $name$_;\n"); } - printer->Print(variables_, - "result.$name$_ = $name$_;\n"); } void ImmutableStringFieldGenerator:: diff --git a/src/google/protobuf/compiler/js/embed.cc b/src/google/protobuf/compiler/js/embed.cc deleted file mode 100644 index f0f946e5..00000000 --- a/src/google/protobuf/compiler/js/embed.cc +++ /dev/null @@ -1,110 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include <cassert> -#include <cstdlib> -#include <fstream> -#include <iostream> -#include <string> - -static bool AsciiIsPrint(unsigned char c) { - return c >= 32 && c < 127; -} - -static char ToDecimalDigit(int num) { - assert(num < 10); - return '0' + num; -} - -static std::string CEscape(const std::string& str) { - std::string dest; - - for (size_t i = 0; i < str.size(); ++i) { - unsigned char ch = str[i]; - switch (ch) { - case '\n': dest += "\\n"; break; - case '\r': dest += "\\r"; break; - case '\t': dest += "\\t"; break; - case '\"': dest += "\\\""; break; - case '\\': dest += "\\\\"; break; - default: - if (AsciiIsPrint(ch)) { - dest += ch; - } else { - dest += "\\"; - dest += ToDecimalDigit(ch / 64); - dest += ToDecimalDigit((ch % 64) / 8); - dest += ToDecimalDigit(ch % 8); - } - break; - } - } - - return dest; -} - -static void AddFile(const char* name, std::basic_ostream<char>* out) { - std::ifstream in(name); - - if (!in.is_open()) { - std::cerr << "Couldn't open input file: " << name << "\n"; - std::exit(EXIT_FAILURE); - } - - // Make canonical name only include the final element. - for (const char *p = name; *p; p++) { - if (*p == '/') { - name = p + 1; - } - } - - *out << "{\"" << CEscape(name) << "\",\n"; - - for (std::string line; std::getline(in, line); ) { - *out << " \"" << CEscape(line) << "\\n\"\n"; - } - - *out << "},\n"; -} - -int main(int argc, char *argv[]) { - std::cout << "#include " - "\"google/protobuf/compiler/js/well_known_types_embed.h\"\n"; - std::cout << "struct FileToc well_known_types_js[] = {\n"; - - for (int i = 1; i < argc; i++) { - AddFile(argv[i], &std::cout); - } - - std::cout << " {NULL, NULL} // Terminate the list.\n"; - std::cout << "};\n"; - - return EXIT_SUCCESS; -} diff --git a/src/google/protobuf/compiler/js/js_generator.cc b/src/google/protobuf/compiler/js/js_generator.cc index b5771f26..dfa60068 100755 --- a/src/google/protobuf/compiler/js/js_generator.cc +++ b/src/google/protobuf/compiler/js/js_generator.cc @@ -3481,7 +3481,8 @@ void Generator::GenerateFile(const GeneratorOptions& options, GenerateExtension(options, printer, *it); } - if (options.import_style == GeneratorOptions::kImportCommonJs) { + // if provided is empty, do not export anything + if (options.import_style == GeneratorOptions::kImportCommonJs && !provided.empty()) { printer->Print("goog.object.extend(exports, $package$);\n", "package", GetFilePath(options, file)); } diff --git a/src/google/protobuf/compiler/js/well_known_types_embed.cc b/src/google/protobuf/compiler/js/well_known_types_embed.cc new file mode 100644 index 00000000..e5ee5510 --- /dev/null +++ b/src/google/protobuf/compiler/js/well_known_types_embed.cc @@ -0,0 +1,225 @@ +#include <google/protobuf/compiler/js/well_known_types_embed.h> + +struct FileToc well_known_types_js[] = { + {"any.js", + "/* This code will be inserted into generated code for\n" + " * google/protobuf/any.proto. */\n" + "\n" + "/**\n" + " * Returns the type name contained in this instance, if any.\n" + " * @return {string|undefined}\n" + " */\n" + "proto.google.protobuf.Any.prototype.getTypeName = function() {\n" + " return this.getTypeUrl().split('/').pop();\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Packs the given message instance into this Any.\n" + " * @param {!Uint8Array} serialized The serialized data to pack.\n" + " * @param {string} name The type name of this message object.\n" + " * @param {string=} opt_typeUrlPrefix the type URL prefix.\n" + " */\n" + "proto.google.protobuf.Any.prototype.pack = function(serialized, name,\n" + " opt_typeUrlPrefix) " + "{\n" + " if (!opt_typeUrlPrefix) {\n" + " opt_typeUrlPrefix = 'type.googleapis.com/';\n" + " }\n" + "\n" + " if (opt_typeUrlPrefix.substr(-1) != '/') {\n" + " this.setTypeUrl(opt_typeUrlPrefix + '/' + name);\n" + " } else {\n" + " this.setTypeUrl(opt_typeUrlPrefix + name);\n" + " }\n" + "\n" + " this.setValue(serialized);\n" + "};\n" + "\n" + "\n" + "/**\n" + " * @template T\n" + " * Unpacks this Any into the given message object.\n" + " * @param {function(Uint8Array):T} deserialize Function that will " + "deserialize\n" + " * the binary data properly.\n" + " * @param {string} name The expected type name of this message object.\n" + " * @return {?T} If the name matched the expected name, returns the " + "deserialized\n" + " * object, otherwise returns null.\n" + " */\n" + "proto.google.protobuf.Any.prototype.unpack = function(deserialize, name) " + "{\n" + " if (this.getTypeName() == name) {\n" + " return deserialize(this.getValue_asU8());\n" + " } else {\n" + " return null;\n" + " }\n" + "};\n"}, + {"timestamp.js", + "/* This code will be inserted into generated code for\n" + " * google/protobuf/timestamp.proto. */\n" + "\n" + "/**\n" + " * Returns a JavaScript 'Date' object corresponding to this Timestamp.\n" + " * @return {!Date}\n" + " */\n" + "proto.google.protobuf.Timestamp.prototype.toDate = function() {\n" + " var seconds = this.getSeconds();\n" + " var nanos = this.getNanos();\n" + "\n" + " return new Date((seconds * 1000) + (nanos / 1000000));\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Sets the value of this Timestamp object to be the given Date.\n" + " * @param {!Date} value The value to set.\n" + " */\n" + "proto.google.protobuf.Timestamp.prototype.fromDate = function(value) {\n" + " this.setSeconds(Math.floor(value.getTime() / 1000));\n" + " this.setNanos(value.getMilliseconds() * 1000000);\n" + "};\n"}, + {"struct.js", + "/* This code will be inserted into generated code for\n" + " * google/protobuf/struct.proto. */\n" + "\n" + "/**\n" + " * Typedef representing plain JavaScript values that can go into a\n" + " * Struct.\n" + " * @typedef {null|number|string|boolean|Array|Object}\n" + " */\n" + "proto.google.protobuf.JavaScriptValue;\n" + "\n" + "\n" + "/**\n" + " * Converts this Value object to a plain JavaScript value.\n" + " * @return {?proto.google.protobuf.JavaScriptValue} a plain JavaScript\n" + " * value representing this Struct.\n" + " */\n" + "proto.google.protobuf.Value.prototype.toJavaScript = function() {\n" + " var kindCase = proto.google.protobuf.Value.KindCase;\n" + " switch (this.getKindCase()) {\n" + " case kindCase.NULL_VALUE:\n" + " return null;\n" + " case kindCase.NUMBER_VALUE:\n" + " return this.getNumberValue();\n" + " case kindCase.STRING_VALUE:\n" + " return this.getStringValue();\n" + " case kindCase.BOOL_VALUE:\n" + " return this.getBoolValue();\n" + " case kindCase.STRUCT_VALUE:\n" + " return this.getStructValue().toJavaScript();\n" + " case kindCase.LIST_VALUE:\n" + " return this.getListValue().toJavaScript();\n" + " default:\n" + " throw new Error('Unexpected struct type');\n" + " }\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this JavaScript value to a new Value proto.\n" + " * @param {!proto.google.protobuf.JavaScriptValue} value The value to\n" + " * convert.\n" + " * @return {!proto.google.protobuf.Value} The newly constructed value.\n" + " */\n" + "proto.google.protobuf.Value.fromJavaScript = function(value) {\n" + " var ret = new proto.google.protobuf.Value();\n" + " switch (goog.typeOf(value)) {\n" + " case 'string':\n" + " ret.setStringValue(/** @type {string} */ (value));\n" + " break;\n" + " case 'number':\n" + " ret.setNumberValue(/** @type {number} */ (value));\n" + " break;\n" + " case 'boolean':\n" + " ret.setBoolValue(/** @type {boolean} */ (value));\n" + " break;\n" + " case 'null':\n" + " ret.setNullValue(proto.google.protobuf.NullValue.NULL_VALUE);\n" + " break;\n" + " case 'array':\n" + " ret.setListValue(proto.google.protobuf.ListValue.fromJavaScript(\n" + " /** @type{!Array} */ (value)));\n" + " break;\n" + " case 'object':\n" + " ret.setStructValue(proto.google.protobuf.Struct.fromJavaScript(\n" + " /** @type{!Object} */ (value)));\n" + " break;\n" + " default:\n" + " throw new Error('Unexpected struct type.');\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this ListValue object to a plain JavaScript array.\n" + " * @return {!Array} a plain JavaScript array representing this List.\n" + " */\n" + "proto.google.protobuf.ListValue.prototype.toJavaScript = function() {\n" + " var ret = [];\n" + " var values = this.getValuesList();\n" + "\n" + " for (var i = 0; i < values.length; i++) {\n" + " ret[i] = values[i].toJavaScript();\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Constructs a ListValue protobuf from this plain JavaScript array.\n" + " * @param {!Array} array a plain JavaScript array\n" + " * @return {proto.google.protobuf.ListValue} a new ListValue object\n" + " */\n" + "proto.google.protobuf.ListValue.fromJavaScript = function(array) {\n" + " var ret = new proto.google.protobuf.ListValue();\n" + "\n" + " for (var i = 0; i < array.length; i++) {\n" + " " + "ret.addValues(proto.google.protobuf.Value.fromJavaScript(array[i]));\n" + " }\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Converts this Struct object to a plain JavaScript object.\n" + " * @return {!Object<string, !proto.google.protobuf.JavaScriptValue>} a " + "plain\n" + " * JavaScript object representing this Struct.\n" + " */\n" + "proto.google.protobuf.Struct.prototype.toJavaScript = function() {\n" + " var ret = {};\n" + "\n" + " this.getFieldsMap().forEach(function(value, key) {\n" + " ret[key] = value.toJavaScript();\n" + " });\n" + "\n" + " return ret;\n" + "};\n" + "\n" + "\n" + "/**\n" + " * Constructs a Struct protobuf from this plain JavaScript object.\n" + " * @param {!Object} obj a plain JavaScript object\n" + " * @return {proto.google.protobuf.Struct} a new Struct object\n" + " */\n" + "proto.google.protobuf.Struct.fromJavaScript = function(obj) {\n" + " var ret = new proto.google.protobuf.Struct();\n" + " var map = ret.getFieldsMap();\n" + "\n" + " for (var property in obj) {\n" + " var val = obj[property];\n" + " map.set(property, proto.google.protobuf.Value.fromJavaScript(val));\n" + " }\n" + "\n" + " return ret;\n" + "};\n"}, + {NULL, NULL} // Terminate the list. +}; diff --git a/src/google/protobuf/compiler/plugin.pb.cc b/src/google/protobuf/compiler/plugin.pb.cc index ef52def3..20aa82c7 100644 --- a/src/google/protobuf/compiler/plugin.pb.cc +++ b/src/google/protobuf/compiler/plugin.pb.cc @@ -183,14 +183,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::compiler::_CodeGeneratorResponse_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/compiler/plugin.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -201,7 +201,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 4); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n%google/protobuf/compiler/plugin.proto\022" diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index 83eae519..60b2a30e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -406,9 +406,10 @@ typedef std::pair<const EnumDescriptor*, int> EnumIntPair; template<typename PairType> struct PointerIntegerPairHash { size_t operator()(const PairType& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. - return reinterpret_cast<intptr_t>(p.first) * ((1 << 16) - 1) + p.second; + static const size_t prime1 = 16777499; + static const size_t prime2 = 16777619; + return reinterpret_cast<size_t>(p.first) * prime1 ^ + static_cast<size_t>(p.second) * prime2; } #ifdef _MSC_VER @@ -424,11 +425,10 @@ struct PointerIntegerPairHash { struct PointerStringPairHash { size_t operator()(const PointerStringPair& p) const { - // FIXME(kenton): What is the best way to compute this hash? I have - // no idea! This seems a bit better than an XOR. + static const size_t prime = 16777619; hash<const char*> cstring_hash; - return reinterpret_cast<intptr_t>(p.first) * ((1 << 16) - 1) + - cstring_hash(p.second); + return reinterpret_cast<size_t>(p.first) * prime ^ + static_cast<size_t>(cstring_hash(p.second)); } #ifdef _MSC_VER diff --git a/src/google/protobuf/descriptor.pb.cc b/src/google/protobuf/descriptor.pb.cc index 59b04aba..2ccf189c 100644 --- a/src/google/protobuf/descriptor.pb.cc +++ b/src/google/protobuf/descriptor.pb.cc @@ -1077,14 +1077,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_GeneratedCodeInfo_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/descriptor.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, file_level_enum_descriptors, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -1095,7 +1095,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 27); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n google/protobuf/descriptor.proto\022\017goog" @@ -1487,7 +1487,7 @@ void FileDescriptorSet::ArenaDtor(void* object) { FileDescriptorSet* _this = reinterpret_cast< FileDescriptorSet* >(object); (void)_this; } -void FileDescriptorSet::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void FileDescriptorSet::RegisterArenaDtor(::google::protobuf::Arena*) { } void FileDescriptorSet::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -1840,7 +1840,7 @@ void FileDescriptorProto::ArenaDtor(void* object) { FileDescriptorProto* _this = reinterpret_cast< FileDescriptorProto* >(object); (void)_this; } -void FileDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void FileDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void FileDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -2642,7 +2642,7 @@ void DescriptorProto_ExtensionRange::ArenaDtor(void* object) { DescriptorProto_ExtensionRange* _this = reinterpret_cast< DescriptorProto_ExtensionRange* >(object); (void)_this; } -void DescriptorProto_ExtensionRange::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void DescriptorProto_ExtensionRange::RegisterArenaDtor(::google::protobuf::Arena*) { } void DescriptorProto_ExtensionRange::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -2994,7 +2994,7 @@ void DescriptorProto_ReservedRange::ArenaDtor(void* object) { DescriptorProto_ReservedRange* _this = reinterpret_cast< DescriptorProto_ReservedRange* >(object); (void)_this; } -void DescriptorProto_ReservedRange::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void DescriptorProto_ReservedRange::RegisterArenaDtor(::google::protobuf::Arena*) { } void DescriptorProto_ReservedRange::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -3350,7 +3350,7 @@ void DescriptorProto::ArenaDtor(void* object) { DescriptorProto* _this = reinterpret_cast< DescriptorProto* >(object); (void)_this; } -void DescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void DescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void DescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -4032,7 +4032,7 @@ void ExtensionRangeOptions::ArenaDtor(void* object) { ExtensionRangeOptions* _this = reinterpret_cast< ExtensionRangeOptions* >(object); (void)_this; } -void ExtensionRangeOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void ExtensionRangeOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void ExtensionRangeOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -4390,7 +4390,7 @@ void FieldDescriptorProto::ArenaDtor(void* object) { FieldDescriptorProto* _this = reinterpret_cast< FieldDescriptorProto* >(object); (void)_this; } -void FieldDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void FieldDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void FieldDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -5120,7 +5120,7 @@ void OneofDescriptorProto::ArenaDtor(void* object) { OneofDescriptorProto* _this = reinterpret_cast< OneofDescriptorProto* >(object); (void)_this; } -void OneofDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void OneofDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void OneofDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -5450,7 +5450,7 @@ void EnumDescriptorProto_EnumReservedRange::ArenaDtor(void* object) { EnumDescriptorProto_EnumReservedRange* _this = reinterpret_cast< EnumDescriptorProto_EnumReservedRange* >(object); (void)_this; } -void EnumDescriptorProto_EnumReservedRange::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void EnumDescriptorProto_EnumReservedRange::RegisterArenaDtor(::google::protobuf::Arena*) { } void EnumDescriptorProto_EnumReservedRange::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -5791,7 +5791,7 @@ void EnumDescriptorProto::ArenaDtor(void* object) { EnumDescriptorProto* _this = reinterpret_cast< EnumDescriptorProto* >(object); (void)_this; } -void EnumDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void EnumDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void EnumDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -6283,7 +6283,7 @@ void EnumValueDescriptorProto::ArenaDtor(void* object) { EnumValueDescriptorProto* _this = reinterpret_cast< EnumValueDescriptorProto* >(object); (void)_this; } -void EnumValueDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void EnumValueDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void EnumValueDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -6676,7 +6676,7 @@ void ServiceDescriptorProto::ArenaDtor(void* object) { ServiceDescriptorProto* _this = reinterpret_cast< ServiceDescriptorProto* >(object); (void)_this; } -void ServiceDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void ServiceDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void ServiceDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -7096,7 +7096,7 @@ void MethodDescriptorProto::ArenaDtor(void* object) { MethodDescriptorProto* _this = reinterpret_cast< MethodDescriptorProto* >(object); (void)_this; } -void MethodDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void MethodDescriptorProto::RegisterArenaDtor(::google::protobuf::Arena*) { } void MethodDescriptorProto::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -7694,7 +7694,7 @@ void FileOptions::ArenaDtor(void* object) { FileOptions* _this = reinterpret_cast< FileOptions* >(object); (void)_this; } -void FileOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void FileOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void FileOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -8885,7 +8885,7 @@ void MessageOptions::ArenaDtor(void* object) { MessageOptions* _this = reinterpret_cast< MessageOptions* >(object); (void)_this; } -void MessageOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void MessageOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void MessageOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -9329,7 +9329,7 @@ void FieldOptions::ArenaDtor(void* object) { FieldOptions* _this = reinterpret_cast< FieldOptions* >(object); (void)_this; } -void FieldOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void FieldOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void FieldOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -9848,7 +9848,7 @@ void OneofOptions::ArenaDtor(void* object) { OneofOptions* _this = reinterpret_cast< OneofOptions* >(object); (void)_this; } -void OneofOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void OneofOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void OneofOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -10145,7 +10145,7 @@ void EnumOptions::ArenaDtor(void* object) { EnumOptions* _this = reinterpret_cast< EnumOptions* >(object); (void)_this; } -void EnumOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void EnumOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void EnumOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -10514,7 +10514,7 @@ void EnumValueOptions::ArenaDtor(void* object) { EnumValueOptions* _this = reinterpret_cast< EnumValueOptions* >(object); (void)_this; } -void EnumValueOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void EnumValueOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void EnumValueOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -10842,7 +10842,7 @@ void ServiceOptions::ArenaDtor(void* object) { ServiceOptions* _this = reinterpret_cast< ServiceOptions* >(object); (void)_this; } -void ServiceOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void ServiceOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void ServiceOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -11175,7 +11175,7 @@ void MethodOptions::ArenaDtor(void* object) { MethodOptions* _this = reinterpret_cast< MethodOptions* >(object); (void)_this; } -void MethodOptions::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void MethodOptions::RegisterArenaDtor(::google::protobuf::Arena*) { } void MethodOptions::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -11559,7 +11559,7 @@ void UninterpretedOption_NamePart::ArenaDtor(void* object) { UninterpretedOption_NamePart* _this = reinterpret_cast< UninterpretedOption_NamePart* >(object); (void)_this; } -void UninterpretedOption_NamePart::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void UninterpretedOption_NamePart::RegisterArenaDtor(::google::protobuf::Arena*) { } void UninterpretedOption_NamePart::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -11924,7 +11924,7 @@ void UninterpretedOption::ArenaDtor(void* object) { UninterpretedOption* _this = reinterpret_cast< UninterpretedOption* >(object); (void)_this; } -void UninterpretedOption::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void UninterpretedOption::RegisterArenaDtor(::google::protobuf::Arena*) { } void UninterpretedOption::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -12473,7 +12473,7 @@ void SourceCodeInfo_Location::ArenaDtor(void* object) { SourceCodeInfo_Location* _this = reinterpret_cast< SourceCodeInfo_Location* >(object); (void)_this; } -void SourceCodeInfo_Location::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void SourceCodeInfo_Location::RegisterArenaDtor(::google::protobuf::Arena*) { } void SourceCodeInfo_Location::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -12979,7 +12979,7 @@ void SourceCodeInfo::ArenaDtor(void* object) { SourceCodeInfo* _this = reinterpret_cast< SourceCodeInfo* >(object); (void)_this; } -void SourceCodeInfo::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void SourceCodeInfo::RegisterArenaDtor(::google::protobuf::Arena*) { } void SourceCodeInfo::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -13258,7 +13258,7 @@ void GeneratedCodeInfo_Annotation::ArenaDtor(void* object) { GeneratedCodeInfo_Annotation* _this = reinterpret_cast< GeneratedCodeInfo_Annotation* >(object); (void)_this; } -void GeneratedCodeInfo_Annotation::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void GeneratedCodeInfo_Annotation::RegisterArenaDtor(::google::protobuf::Arena*) { } void GeneratedCodeInfo_Annotation::SetCachedSize(int size) const { _cached_size_.Set(size); @@ -13676,7 +13676,7 @@ void GeneratedCodeInfo::ArenaDtor(void* object) { GeneratedCodeInfo* _this = reinterpret_cast< GeneratedCodeInfo* >(object); (void)_this; } -void GeneratedCodeInfo::RegisterArenaDtor(::google::protobuf::Arena* arena) { +void GeneratedCodeInfo::RegisterArenaDtor(::google::protobuf::Arena*) { } void GeneratedCodeInfo::SetCachedSize(int size) const { _cached_size_.Set(size); diff --git a/src/google/protobuf/duration.pb.cc b/src/google/protobuf/duration.pb.cc index ddc063db..0c67f86e 100644 --- a/src/google/protobuf/duration.pb.cc +++ b/src/google/protobuf/duration.pb.cc @@ -66,14 +66,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Duration_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/duration.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -84,7 +84,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\036google/protobuf/duration.proto\022\017google" diff --git a/src/google/protobuf/empty.pb.cc b/src/google/protobuf/empty.pb.cc index 9b53c546..7bc46121 100644 --- a/src/google/protobuf/empty.pb.cc +++ b/src/google/protobuf/empty.pb.cc @@ -64,14 +64,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Empty_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/empty.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -82,7 +82,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\033google/protobuf/empty.proto\022\017google.pr" diff --git a/src/google/protobuf/field_mask.pb.cc b/src/google/protobuf/field_mask.pb.cc index 000fa8d3..2ce061d9 100644 --- a/src/google/protobuf/field_mask.pb.cc +++ b/src/google/protobuf/field_mask.pb.cc @@ -65,14 +65,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_FieldMask_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/field_mask.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -83,7 +83,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n google/protobuf/field_mask.proto\022\017goog" diff --git a/src/google/protobuf/generated_message_util.h b/src/google/protobuf/generated_message_util.h index 706df383..4da04cae 100644 --- a/src/google/protobuf/generated_message_util.h +++ b/src/google/protobuf/generated_message_util.h @@ -201,7 +201,7 @@ struct SerializationTable { }; LIBPROTOBUF_EXPORT void SerializeInternal(const uint8* base, const FieldMetadata* table, - int num_fields, ::google::protobuf::io::CodedOutputStream* output); + int32 num_fields, ::google::protobuf::io::CodedOutputStream* output); inline void TableSerialize(const ::google::protobuf::MessageLite& msg, const SerializationTable* table, @@ -219,7 +219,7 @@ inline void TableSerialize(const ::google::protobuf::MessageLite& msg, } uint8* SerializeInternalToArray(const uint8* base, const FieldMetadata* table, - int num_fields, bool is_deterministic, + int32 num_fields, bool is_deterministic, uint8* buffer); inline uint8* TableSerializeToArray(const ::google::protobuf::MessageLite& msg, diff --git a/src/google/protobuf/message_lite.cc b/src/google/protobuf/message_lite.cc index 123b142d..65af7cea 100644 --- a/src/google/protobuf/message_lite.cc +++ b/src/google/protobuf/message_lite.cc @@ -316,7 +316,11 @@ bool MessageLite::SerializeToArray(void* data, int size) const { } bool MessageLite::SerializePartialToArray(void* data, int size) const { - int byte_size = ByteSizeLong(); + size_t byte_size = ByteSizeLong(); + if (byte_size > INT_MAX) { + GOOGLE_LOG(ERROR) << "Exceeded maximum protobuf size of 2GB: " << size; + return false; + } if (size < byte_size) return false; uint8* start = reinterpret_cast<uint8*>(data); uint8* end = SerializeWithCachedSizesToArray(start); diff --git a/src/google/protobuf/source_context.pb.cc b/src/google/protobuf/source_context.pb.cc index 739172f8..af93e6b2 100644 --- a/src/google/protobuf/source_context.pb.cc +++ b/src/google/protobuf/source_context.pb.cc @@ -65,14 +65,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_SourceContext_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/source_context.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -83,7 +83,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n$google/protobuf/source_context.proto\022\017" diff --git a/src/google/protobuf/struct.pb.cc b/src/google/protobuf/struct.pb.cc index 8f2d4848..4bf5d45c 100644 --- a/src/google/protobuf/struct.pb.cc +++ b/src/google/protobuf/struct.pb.cc @@ -140,14 +140,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_ListValue_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/struct.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, file_level_enum_descriptors, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -158,7 +158,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 4); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\034google/protobuf/struct.proto\022\017google.p" diff --git a/src/google/protobuf/stubs/int128.cc b/src/google/protobuf/stubs/int128.cc index a5090801..7b993e8f 100644 --- a/src/google/protobuf/stubs/int128.cc +++ b/src/google/protobuf/stubs/int128.cc @@ -76,52 +76,36 @@ static inline int Fls128(uint128 n) { return Fls64(Uint128Low64(n)); } -// Long division/modulo for uint128 implemented using the shift-subtract -// division algorithm adapted from: -// http://stackoverflow.com/questions/5386377/division-without-using void uint128::DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret, uint128* remainder_ret) { if (divisor == 0) { GOOGLE_LOG(FATAL) << "Division or mod by zero: dividend.hi=" << dividend.hi_ << ", lo=" << dividend.lo_; - } - - if (divisor > dividend) { + } else if (dividend < divisor) { *quotient_ret = 0; *remainder_ret = dividend; return; - } - - if (divisor == dividend) { - *quotient_ret = 1; - *remainder_ret = 0; - return; - } - - uint128 denominator = divisor; - uint128 position = 1; - uint128 quotient = 0; - - // Left aligns the MSB of the denominator and the dividend. - int shift = Fls128(dividend) - Fls128(denominator); - denominator <<= shift; - position <<= shift; - - // Uses shift-subtract algorithm to divide dividend by denominator. The - // remainder will be left in dividend. - while (position > 0) { - if (dividend >= denominator) { - dividend -= denominator; - quotient |= position; + } else { + int dividend_bit_length = Fls128(dividend); + int divisor_bit_length = Fls128(divisor); + int difference = dividend_bit_length - divisor_bit_length; + uint128 quotient = 0; + while (difference >= 0) { + quotient <<= 1; + uint128 shifted_divisor = divisor << difference; + if (shifted_divisor <= dividend) { + dividend -= shifted_divisor; + quotient += 1; + } + difference -= 1; } - position >>= 1; - denominator >>= 1; + //record the final quotient and remainder + *quotient_ret = quotient; + *remainder_ret = dividend; } - - *quotient_ret = quotient; - *remainder_ret = dividend; } + uint128& uint128::operator/=(const uint128& divisor) { uint128 quotient = 0; uint128 remainder = 0; diff --git a/src/google/protobuf/stubs/mutex.h b/src/google/protobuf/stubs/mutex.h index b9b7d2e1..47edb7a3 100644 --- a/src/google/protobuf/stubs/mutex.h +++ b/src/google/protobuf/stubs/mutex.h @@ -34,6 +34,18 @@ #include <google/protobuf/stubs/macros.h> +// Define thread-safety annotations for use below, if we are building with +// Clang. +#if defined(__clang__) && !defined(SWIG) +#define GOOGLE_PROTOBUF_ACQUIRE(...) \ + __attribute__((acquire_capability(__VA_ARGS__))) +#define GOOGLE_PROTOBUF_RELEASE(...) \ + __attribute__((release_capability(__VA_ARGS__))) +#else +#define GOOGLE_PROTOBUF_ACQUIRE(...) +#define GOOGLE_PROTOBUF_RELEASE(...) +#endif + // =================================================================== // emulates google3/base/mutex.h namespace google { @@ -48,8 +60,8 @@ namespace internal { class LIBPROTOBUF_EXPORT WrappedMutex { public: WrappedMutex() = default; - void Lock() { mu_.lock(); } - void Unlock() { mu_.unlock(); } + void Lock() GOOGLE_PROTOBUF_ACQUIRE() { mu_.lock(); } + void Unlock() GOOGLE_PROTOBUF_RELEASE() { mu_.unlock(); } // Crash if this Mutex is not held exclusively by this thread. // May fail to crash when it should; will never crash when it should not. void AssertHeld() const {} @@ -123,8 +135,10 @@ using internal::ReaderMutexLock; using internal::WriterMutexLock; using internal::MutexLockMaybe; - } // namespace protobuf } // namespace google +#undef GOOGLE_PROTOBUF_ACQUIRE +#undef GOOGLE_PROTOBUF_RELEASE + #endif // GOOGLE_PROTOBUF_STUBS_MUTEX_H_ diff --git a/src/google/protobuf/stubs/platform_macros.h b/src/google/protobuf/stubs/platform_macros.h index c3a64dd2..ff4dea7b 100644 --- a/src/google/protobuf/stubs/platform_macros.h +++ b/src/google/protobuf/stubs/platform_macros.h @@ -99,6 +99,7 @@ GOOGLE_PROTOBUF_PLATFORM_ERROR #if defined(__APPLE__) #define GOOGLE_PROTOBUF_OS_APPLE +#include <Availability.h> #include <TargetConditionals.h> #if TARGET_OS_IPHONE #define GOOGLE_PROTOBUF_OS_IPHONE @@ -125,4 +126,9 @@ GOOGLE_PROTOBUF_PLATFORM_ERROR #define GOOGLE_PROTOBUF_NO_THREADLOCAL #endif +#if defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 1070 +// __thread keyword requires at least 10.7 +#define GOOGLE_PROTOBUF_NO_THREADLOCAL +#endif + #endif // GOOGLE_PROTOBUF_PLATFORM_MACROS_H_ diff --git a/src/google/protobuf/timestamp.pb.cc b/src/google/protobuf/timestamp.pb.cc index 8e6b5f23..3184bd97 100644 --- a/src/google/protobuf/timestamp.pb.cc +++ b/src/google/protobuf/timestamp.pb.cc @@ -66,14 +66,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Timestamp_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/timestamp.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -84,7 +84,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 1); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\037google/protobuf/timestamp.proto\022\017googl" diff --git a/src/google/protobuf/type.pb.cc b/src/google/protobuf/type.pb.cc index 130af52a..17d5ce2a 100644 --- a/src/google/protobuf/type.pb.cc +++ b/src/google/protobuf/type.pb.cc @@ -219,14 +219,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_Option_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/type.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, file_level_enum_descriptors, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -237,7 +237,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 5); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\032google/protobuf/type.proto\022\017google.pro" diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h index 28496963..f2b4f42f 100644 --- a/src/google/protobuf/util/internal/proto_writer.h +++ b/src/google/protobuf/util/internal/proto_writer.h @@ -110,7 +110,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { return RenderDataPiece(name, DataPiece(value, use_strict_base64_decoding())); } - virtual ProtoWriter* RenderBytes(StringPiece name, StringPiece value) { + ProtoWriter* RenderBytes(StringPiece name, StringPiece value) override { return RenderDataPiece( name, DataPiece(value, false, use_strict_base64_decoding())); } diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc index 56e6db12..b0d86c17 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource.cc @@ -648,6 +648,13 @@ Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, // using a nested ProtoStreamObjectSource using our nested type information. ProtoStreamObjectSource nested_os(&in_stream, os->typeinfo_, *nested_type); + // TODO(htuch): This is somewhat fragile, since new options may be omitted. + // We should probably do this via the constructor or some object grouping + // options. + nested_os.set_use_lower_camel_for_enums(os->use_lower_camel_for_enums_); + nested_os.set_use_ints_for_enums(os->use_ints_for_enums_); + nested_os.set_preserve_proto_field_names(os->preserve_proto_field_names_); + // We manually call start and end object here so we can inject the @type. ow->StartObject(field_name); ow->RenderString("@type", type_url); diff --git a/src/google/protobuf/util/internal/protostream_objectsource_test.cc b/src/google/protobuf/util/internal/protostream_objectsource_test.cc index df790728..4d86b856 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource_test.cc @@ -100,6 +100,7 @@ class ProtostreamObjectSourceTest ow_(&mock_), use_lower_camel_for_enums_(false), use_ints_for_enums_(false), + use_preserve_proto_field_names_(false), add_trailing_zeros_(false), render_unknown_enum_values_(true) { helper_.ResetTypeInfo(Book::descriptor(), Proto3Message::descriptor()); @@ -123,6 +124,7 @@ class ProtostreamObjectSourceTest helper_.NewProtoSource(&in_stream, GetTypeUrl(descriptor))); if (use_lower_camel_for_enums_) os->set_use_lower_camel_for_enums(true); if (use_ints_for_enums_) os->set_use_ints_for_enums(true); + if (use_preserve_proto_field_names_) os->set_preserve_proto_field_names(true); os->set_max_recursion_depth(64); return os->WriteTo(&mock_); } @@ -272,6 +274,8 @@ class ProtostreamObjectSourceTest void UseIntsForEnums() { use_ints_for_enums_ = true; } + void UsePreserveProtoFieldNames() { use_preserve_proto_field_names_ = true; } + void AddTrailingZeros() { add_trailing_zeros_ = true; } void SetRenderUnknownEnumValues(bool value) { @@ -284,6 +288,7 @@ class ProtostreamObjectSourceTest ExpectingObjectWriter ow_; bool use_lower_camel_for_enums_; bool use_ints_for_enums_; + bool use_preserve_proto_field_names_; bool add_trailing_zeros_; bool render_unknown_enum_values_; }; @@ -536,6 +541,16 @@ TEST_P(ProtostreamObjectSourceTest, UseIntsForEnumsTest) { DoTest(book, Book::descriptor()); } +TEST_P(ProtostreamObjectSourceTest, UsePreserveProtoFieldNames) { + Book book; + book.set_snake_field("foo"); + + UsePreserveProtoFieldNames(); + + ow_.StartObject("")->RenderString("snake_field", "foo")->EndObject(); + DoTest(book, Book::descriptor()); +} + TEST_P(ProtostreamObjectSourceTest, UnknownEnumAreDroppedWhenRenderUnknownEnumValuesIsUnset) { Proto3Message message; @@ -769,6 +784,69 @@ TEST_P(ProtostreamObjectSourceAnysTest, BasicAny) { DoTest(out, AnyOut::descriptor()); } +TEST_P(ProtostreamObjectSourceAnysTest, LowerCamelEnumOutputSnakeCase) { + AnyOut out; + ::google::protobuf::Any* any = out.mutable_any(); + + Book book; + book.set_type(Book::arts_and_photography); + any->PackFrom(book); + + UseLowerCamelForEnums(); + + ow_.StartObject("") + ->StartObject("any") + ->RenderString("@type", + "type.googleapis.com/google.protobuf.testing.Book") + ->RenderString("type", "artsAndPhotography") + ->EndObject() + ->EndObject(); + + DoTest(out, AnyOut::descriptor()); +} + +TEST_P(ProtostreamObjectSourceAnysTest, UseIntsForEnumsTest) { + AnyOut out; + ::google::protobuf::Any* any = out.mutable_any(); + + Book book; + book.set_type(Book::ACTION_AND_ADVENTURE); + any->PackFrom(book); + + UseIntsForEnums(); + + ow_.StartObject("") + ->StartObject("any") + ->RenderString("@type", + "type.googleapis.com/google.protobuf.testing.Book") + ->RenderInt32("type", 3) + ->EndObject() + ->EndObject(); + + DoTest(out, AnyOut::descriptor()); +} + +TEST_P(ProtostreamObjectSourceAnysTest, UsePreserveProtoFieldNames) { + AnyOut out; + ::google::protobuf::Any* any = out.mutable_any(); + + Book book; + book.set_snake_field("foo"); + any->PackFrom(book); + + UsePreserveProtoFieldNames(); + + ow_.StartObject("") + ->StartObject("any") + ->RenderString("@type", + "type.googleapis.com/google.protobuf.testing.Book") + ->RenderString("snake_field", "foo") + ->EndObject() + ->EndObject(); + + DoTest(out, AnyOut::descriptor()); +} + TEST_P(ProtostreamObjectSourceAnysTest, RecursiveAny) { AnyOut out; ::google::protobuf::Any* any = out.mutable_any(); diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc index 2edfd075..a1a7030a 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc @@ -534,7 +534,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( Push("", Item::MESSAGE, false, false); ProtoWriter::RenderDataPiece("key", DataPiece(name, use_strict_base64_decoding())); - Push("value", Item::MESSAGE, true, false); + Push("value", IsAny(*Lookup("value")) ? Item::ANY : Item::MESSAGE, true, false); // Make sure we are valid so far after starting map fields. if (invalid_depth() > 0) return this; diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index 7f0df567..9c50e7dd 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -1679,6 +1679,22 @@ TEST_P(ProtoStreamObjectWriterMapTest, RepeatedMapKeyTest) { ->EndObject(); } +TEST_P(ProtoStreamObjectWriterMapTest, AnyInMap) { + MapIn mm; + google::protobuf::DoubleValue d; + d.set_value(40.2); + (*mm.mutable_map_any())["foo"].PackFrom(d); + ow_->StartObject("") + ->StartObject("map_any") + ->StartObject("foo") + ->RenderString("@type", "type.googleapis.com/google.protobuf.DoubleValue") + ->RenderDouble("value", 40.2) + ->EndObject() + ->EndObject() + ->EndObject(); + CheckOutput(mm); +} + class ProtoStreamObjectWriterAnyTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterAnyTest() { diff --git a/src/google/protobuf/util/internal/testdata/books.proto b/src/google/protobuf/util/internal/testdata/books.proto index 5630cc78..5e08a291 100644 --- a/src/google/protobuf/util/internal/testdata/books.proto +++ b/src/google/protobuf/util/internal/testdata/books.proto @@ -69,6 +69,9 @@ message Book { } optional Type type = 11; + // Useful for testing JSON snake/camel-case conversions. + optional string snake_field = 12; + extensions 200 to 499; } diff --git a/src/google/protobuf/util/internal/testdata/maps.proto b/src/google/protobuf/util/internal/testdata/maps.proto index 0f381b32..765254ea 100644 --- a/src/google/protobuf/util/internal/testdata/maps.proto +++ b/src/google/protobuf/util/internal/testdata/maps.proto @@ -32,6 +32,8 @@ syntax = "proto3"; package google.protobuf.testing; +import "google/protobuf/any.proto"; + // Top-level test cases proto used by MarshallingTest. See description // at the top of the class MarshallingTest for details on how to write // test cases. @@ -103,6 +105,7 @@ message MapIn { string other = 1; repeated string things = 2; map<string, string> map_input = 3; + map<string, google.protobuf.Any> map_any = 4; } message MapOut { diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index f81a7a30..9ad073d8 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -127,7 +127,8 @@ class StatusErrorListener : public converter::ErrorListener { virtual void InvalidName(const converter::LocationTrackerInterface& loc, StringPiece unknown_name, StringPiece message) { status_ = util::Status(util::error::INVALID_ARGUMENT, - loc.ToString() + ": " + string(message)); + loc.ToString() + ": invalid name " + + string(unknown_name) + ": " + string(message)); } virtual void InvalidValue(const converter::LocationTrackerInterface& loc, diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index ed9092df..a2a84b57 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -284,7 +284,7 @@ TEST_F(JsonUtilTest, ParsePrimitiveMapIn) { JsonPrintOptions print_options; print_options.always_print_primitive_fields = true; JsonParseOptions parse_options; - EXPECT_EQ("{\"other\":\"\",\"things\":[],\"mapInput\":{}}", + EXPECT_EQ("{\"other\":\"\",\"things\":[],\"mapInput\":{},\"mapAny\":{}}", ToJson(message, print_options)); MapIn other; ASSERT_TRUE(FromJson(ToJson(message, print_options), &other, parse_options)); diff --git a/src/google/protobuf/wrappers.pb.cc b/src/google/protobuf/wrappers.pb.cc index ffd905f5..386be61d 100644 --- a/src/google/protobuf/wrappers.pb.cc +++ b/src/google/protobuf/wrappers.pb.cc @@ -289,14 +289,14 @@ static ::google::protobuf::Message const * const file_default_instances[] = { reinterpret_cast<const ::google::protobuf::Message*>(&::google::protobuf::_BytesValue_default_instance_), }; -void protobuf_AssignDescriptors() { +static void protobuf_AssignDescriptors() { AddDescriptors(); AssignDescriptors( "google/protobuf/wrappers.proto", schemas, file_default_instances, TableStruct::offsets, file_level_metadata, NULL, NULL); } -void protobuf_AssignDescriptorsOnce() { +static void protobuf_AssignDescriptorsOnce() { static ::google::protobuf::internal::once_flag once; ::google::protobuf::internal::call_once(once, protobuf_AssignDescriptors); } @@ -307,7 +307,7 @@ void protobuf_RegisterTypes(const ::std::string&) { ::google::protobuf::internal::RegisterAllTypes(file_level_metadata, 9); } -void AddDescriptorsImpl() { +static void AddDescriptorsImpl() { InitDefaults(); static const char descriptor[] GOOGLE_PROTOBUF_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { "\n\036google/protobuf/wrappers.proto\022\017google" |