diff options
Diffstat (limited to 'benchmarks')
23 files changed, 1754 insertions, 349 deletions
diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 5aa35c66..9f609228 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -1,9 +1,11 @@ +benchmarks_protoc_inputs_benchmark_wrapper = \ + benchmarks.proto + benchmarks_protoc_inputs = \ - benchmarks.proto \ - datasets/google_message1/benchmark_message1_proto3.proto + datasets/google_message1/proto3/benchmark_message1_proto3.proto benchmarks_protoc_inputs_proto2 = \ - datasets/google_message1/benchmark_message1_proto2.proto \ + datasets/google_message1/proto2/benchmark_message1_proto2.proto \ datasets/google_message2/benchmark_message2.proto \ datasets/google_message3/benchmark_message3.proto \ datasets/google_message3/benchmark_message3_1.proto \ @@ -23,76 +25,62 @@ make_tmp_dir: mkdir -p 'tmp/java/src/main/java' touch make_tmp_dir -if USE_EXTERNAL_PROTOC - -protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs) - touch protoc_middleman - -protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs_proto2) - touch protoc_middleman2 - -else - # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. -protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) +protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) ) touch protoc_middleman protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) touch protoc_middleman2 -endif - -all_data = `find . -type f -name "dataset.*.pb"` +all_data = $$(find $(srcdir) -type f -name "dataset.*.pb" -not -path "./tmp/*") ############# CPP RULES ############## benchmarks_protoc_outputs = \ - benchmarks.pb.cc \ - datasets/google_message1/benchmark_message1_proto3.pb.cc - + cpp/benchmarks.pb.cc \ + cpp/datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc + benchmarks_protoc_outputs_header = \ - benchmarks.pb.h \ - datasets/google_message1/benchmark_message1_proto3.pb.h + cpp/benchmarks.pb.h \ + cpp/datasets/google_message1/proto3/benchmark_message1_proto3.pb.h benchmarks_protoc_outputs_proto2_header = \ - datasets/google_message1/benchmark_message1_proto2.pb.h \ - datasets/google_message2/benchmark_message2.pb.h \ - datasets/google_message3/benchmark_message3.pb.h \ - datasets/google_message3/benchmark_message3_1.pb.h \ - datasets/google_message3/benchmark_message3_2.pb.h \ - datasets/google_message3/benchmark_message3_3.pb.h \ - datasets/google_message3/benchmark_message3_4.pb.h \ - datasets/google_message3/benchmark_message3_5.pb.h \ - datasets/google_message3/benchmark_message3_6.pb.h \ - datasets/google_message3/benchmark_message3_7.pb.h \ - datasets/google_message3/benchmark_message3_8.pb.h \ - datasets/google_message4/benchmark_message4.pb.h \ - datasets/google_message4/benchmark_message4_1.pb.h \ - datasets/google_message4/benchmark_message4_2.pb.h \ - datasets/google_message4/benchmark_message4_3.pb.h + cpp/datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ + cpp/datasets/google_message2/benchmark_message2.pb.h \ + cpp/datasets/google_message3/benchmark_message3.pb.h \ + cpp/datasets/google_message3/benchmark_message3_1.pb.h \ + cpp/datasets/google_message3/benchmark_message3_2.pb.h \ + cpp/datasets/google_message3/benchmark_message3_3.pb.h \ + cpp/datasets/google_message3/benchmark_message3_4.pb.h \ + cpp/datasets/google_message3/benchmark_message3_5.pb.h \ + cpp/datasets/google_message3/benchmark_message3_6.pb.h \ + cpp/datasets/google_message3/benchmark_message3_7.pb.h \ + cpp/datasets/google_message3/benchmark_message3_8.pb.h \ + cpp/datasets/google_message4/benchmark_message4.pb.h \ + cpp/datasets/google_message4/benchmark_message4_1.pb.h \ + cpp/datasets/google_message4/benchmark_message4_2.pb.h \ + cpp/datasets/google_message4/benchmark_message4_3.pb.h benchmarks_protoc_outputs_proto2 = \ - datasets/google_message1/benchmark_message1_proto2.pb.cc \ - datasets/google_message2/benchmark_message2.pb.cc \ - datasets/google_message3/benchmark_message3.pb.cc \ - datasets/google_message3/benchmark_message3_1.pb.cc \ - datasets/google_message3/benchmark_message3_2.pb.cc \ - datasets/google_message3/benchmark_message3_3.pb.cc \ - datasets/google_message3/benchmark_message3_4.pb.cc \ - datasets/google_message3/benchmark_message3_5.pb.cc \ - datasets/google_message3/benchmark_message3_6.pb.cc \ - datasets/google_message3/benchmark_message3_7.pb.cc \ - datasets/google_message3/benchmark_message3_8.pb.cc \ - datasets/google_message4/benchmark_message4.pb.cc \ - datasets/google_message4/benchmark_message4_1.pb.cc \ - datasets/google_message4/benchmark_message4_2.pb.cc \ - datasets/google_message4/benchmark_message4_3.pb.cc + cpp/datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ + cpp/datasets/google_message2/benchmark_message2.pb.cc \ + cpp/datasets/google_message3/benchmark_message3.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_1.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_2.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_3.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_4.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_5.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_6.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_7.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_8.pb.cc \ + cpp/datasets/google_message4/benchmark_message4.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_1.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_2.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_3.pb.cc $(benchmarks_protoc_outputs): protoc_middleman @@ -107,8 +95,7 @@ initialize_submodule: cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make cd $$oldpwd touch initialize_submodule - -$(top_srcdir)/src/libprotobuf.la: initialize_submodule + $(top_srcdir)/third_party/benchmark/src/libbenchmark.a: initialize_submodule AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare @@ -116,12 +103,12 @@ AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Wov bin_PROGRAMS = cpp-benchmark cpp_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a -cpp_benchmark_SOURCES = cpp_benchmark.cc -cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_party/benchmark/include +cpp_benchmark_SOURCES = cpp/cpp_benchmark.cc +cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(top_srcdir)/third_party/benchmark/include # Explicit deps because BUILT_SOURCES are only done before a "make all/check" # so a direct "make test_cpp" could fail if parallel enough. # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually -cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp/cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a nodist_cpp_benchmark_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ @@ -136,18 +123,26 @@ cpp: protoc_middleman protoc_middleman2 cpp-benchmark initialize_submodule ############# JAVA RULES ############## java_benchmark_testing_files = \ - java/src/main/java/com/google/protobuf/ProtoBench.java \ java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2 - cp -r java tmp && cd tmp/java && mvn clean compile assembly:single - cd ../.. + cp -r $(srcdir)/java tmp + mkdir -p tmp/java/lib + cp $(top_srcdir)/java/core/target/*.jar tmp/java/lib/protobuf-java.jar + cd tmp/java && mvn clean compile assembly:single -Dprotobuf.version=$(PACKAGE_VERSION) && cd ../.. @touch javac_middleman java-benchmark: javac_middleman @echo "Writing shortcut script java-benchmark..." - @echo '#! /bin/sh' > java-benchmark - @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark + @echo '#! /bin/bash' > java-benchmark + @echo 'all_data=""' >> java-benchmark + @echo 'conf=()' >> java-benchmark + @echo 'data_files=""' >> java-benchmark + @echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+="$$arg,"; fi; done' >> java-benchmark + @echo 'java -cp '\"tmp/java/target/*:$(top_srcdir)/java/core/target/*:$(top_srcdir)/java/util/target/*\"" \\" >>java-benchmark + @echo ' com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\" >> java-benchmark + @echo ' -b serializeToByteArray,serializeToMemoryStream,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark + @echo ' -DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark @chmod +x java-benchmark java: protoc_middleman protoc_middleman2 java-benchmark @@ -172,11 +167,11 @@ python_add_init: protoc_middleman protoc_middleman2 python_cpp_pkg_flags = `pkg-config --cflags --libs python` lib_LTLIBRARIES = libbenchmark_messages.la -libbenchmark_messages_la_SOURCES = python_benchmark_messages.cc +libbenchmark_messages_la_SOURCES = python/python_benchmark_messages.cc libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la libbenchmark_messages_la_LDFLAGS = -version-info 1:0:0 -export-dynamic -libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) $(python_cpp_pkg_flags) -libbenchmark_messages_la-libbenchmark_messages_la.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) +libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp $(python_cpp_pkg_flags) +libbenchmark_messages_la-python_benchmark_messages.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) nodist_libbenchmark_messages_la_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ @@ -185,32 +180,32 @@ nodist_libbenchmark_messages_la_SOURCES = \ python-pure-python-benchmark: python_add_init @echo "Writing shortcut script python-pure-python-benchmark..." - @echo '#! /bin/sh' > python-pure-python-benchmark - @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark - @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark + @echo '#! /bin/bash' > python-pure-python-benchmark + @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-pure-python-benchmark + @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-pure-python-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'python\' >> python-pure-python-benchmark - @echo cp py_benchmark.py tmp >> python-pure-python-benchmark - @echo python tmp/py_benchmark.py false '$$@' >> python-pure-python-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-pure-python-benchmark + @echo python tmp/py_benchmark.py '$$@' >> python-pure-python-benchmark @chmod +x python-pure-python-benchmark python-cpp-reflection-benchmark: python_add_init @echo "Writing shortcut script python-cpp-reflection-benchmark..." - @echo '#! /bin/sh' > python-cpp-reflection-benchmark - @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark - @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark + @echo '#! /bin/bash' > python-cpp-reflection-benchmark + @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-reflection-benchmark + @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-reflection-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-reflection-benchmark - @echo cp py_benchmark.py tmp >> python-cpp-reflection-benchmark - @echo python tmp/py_benchmark.py false '$$@' >> python-cpp-reflection-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-reflection-benchmark + @echo python tmp/py_benchmark.py '$$@' >> python-cpp-reflection-benchmark @chmod +x python-cpp-reflection-benchmark python-cpp-generated-code-benchmark: python_add_init libbenchmark_messages.la @echo "Writing shortcut script python-cpp-generated-code-benchmark..." - @echo '#! /bin/sh' > python-cpp-generated-code-benchmark - @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark - @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark + @echo '#! /bin/bash' > python-cpp-generated-code-benchmark + @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-generated-code-benchmark + @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/.libs >> python-cpp-generated-code-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-generated-code-benchmark - @echo cp py_benchmark.py tmp >> python-cpp-generated-code-benchmark - @echo python tmp/py_benchmark.py true '$$@' >> python-cpp-generated-code-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-generated-code-benchmark + @echo python tmp/py_benchmark.py --cpp_generated '$$@' >> python-cpp-generated-code-benchmark @chmod +x python-cpp-generated-code-benchmark python-pure-python: python-pure-python-benchmark @@ -224,6 +219,287 @@ python-cpp-generated-code: python-cpp-generated-code-benchmark ############# PYTHON RULES END ############## +############# GO RULES BEGIN ############## + +benchmarks_protoc_inputs_proto2_message1 = \ + datasets/google_message1/proto2/benchmark_message1_proto2.proto + +benchmarks_protoc_inputs_proto2_message2 = \ + datasets/google_message2/benchmark_message2.proto + +benchmarks_protoc_inputs_proto2_message3 = \ + datasets/google_message3/benchmark_message3.proto \ + datasets/google_message3/benchmark_message3_1.proto \ + datasets/google_message3/benchmark_message3_2.proto \ + datasets/google_message3/benchmark_message3_3.proto \ + datasets/google_message3/benchmark_message3_4.proto \ + datasets/google_message3/benchmark_message3_5.proto \ + datasets/google_message3/benchmark_message3_6.proto \ + datasets/google_message3/benchmark_message3_7.proto \ + datasets/google_message3/benchmark_message3_8.proto + +benchmarks_protoc_inputs_proto2_message4 = \ + datasets/google_message4/benchmark_message4.proto \ + datasets/google_message4/benchmark_message4_1.proto \ + datasets/google_message4/benchmark_message4_2.proto \ + datasets/google_message4/benchmark_message4_3.proto + +go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) ) + touch go_protoc_middleman + +go-benchmark: go_protoc_middleman + @echo "Writing shortcut script go-benchmark..." + @echo '#! /bin/bash' > go-benchmark + @echo 'cd $(srcdir)/go' >> go-benchmark + @echo 'all_data=""' >> go-benchmark + @echo 'conf=()' >> go-benchmark + @echo 'data_files=()' >> go-benchmark + @echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+=("../$$arg"); fi; done' >> go-benchmark + @echo 'go test -bench=. $${conf[*]} -- $${data_files[*]}' >> go-benchmark + @echo 'cd ..' >> go-benchmark + @chmod +x go-benchmark + +go: go_protoc_middleman go-benchmark + ./go-benchmark $(all_data) + +############# GO RULES END ############## + +############# GOGO RULES BEGIN ############ + +cpp_no_group_benchmarks_protoc_outputs_header = \ + gogo/cpp_no_group/benchmarks.pb.h \ + gogo/cpp_no_group/datasets/google_message1/proto3/benchmark_message1_proto3.pb.h + +cpp_no_group_benchmarks_protoc_outputs = \ + gogo/cpp_no_group/benchmarks.pb.cc \ + gogo/cpp_no_group/datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc + +cpp_no_group_benchmarks_protoc_outputs_proto2_header = \ + gogo/cpp_no_group/datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ + gogo/cpp_no_group/datasets/google_message2/benchmark_message2.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_1.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_2.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_3.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_4.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_5.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_6.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_7.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_8.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_1.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_2.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_3.pb.h + +cpp_no_group_benchmarks_protoc_outputs_proto2 = \ + gogo/cpp_no_group/datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ + gogo/cpp_no_group/datasets/google_message2/benchmark_message2.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_1.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_2.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_3.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_4.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_5.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_6.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_7.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_8.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_1.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_2.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_3.pb.cc + +$(cpp_no_group_benchmarks_protoc_outputs): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_header): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_proto2): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_proto2_header): cpp_no_group_protoc_middleman + +generate_cpp_no_group_benchmark_code: + cp $(srcdir)/cpp/cpp_benchmark.cc gogo/cpp_no_group/cpp_benchmark.cc + sed -i -e "s/\#include \"datasets/\#include \"gogo\/cpp_no_group\/datasets/g" gogo/cpp_no_group/cpp_benchmark.cc + sed -i -e "s/\#include \"benchmarks.pb.h/\#include \"gogo\/cpp_no_group\/benchmarks.pb.h/g" gogo/cpp_no_group/cpp_benchmark.cc + touch generate_cpp_no_group_benchmark_code + +bin_PROGRAMS += cpp-no-group-benchmark +cpp_no_group_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp_no_group_benchmark_SOURCES = gogo/cpp_no_group/cpp_benchmark.cc +cpp_no_group_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/gogo/cpp_no_group -I$(top_srcdir)/third_party/benchmark/include +# Explicit deps because BUILT_SOURCES are only done before a "make all/check" +# so a direct "make test_cpp" could fail if parallel enough. +# See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +gogo/cpp_no_group/cpp_no_group_benchmark-cpp_benchmark.$(OBJEXT): $(cpp_no_group_benchmarks_protoc_outputs) $(cpp_no_group_benchmarks_protoc_outputs_proto2) $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/third_party/benchmark/src/libbenchmark.a generate_cpp_no_group_benchmark_code +gogo/cpp_no_group/cpp_benchmark.cc: generate_cpp_no_group_benchmark_code +nodist_cpp_no_group_benchmark_SOURCES = \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2) \ + $(cpp_no_group_benchmarks_protoc_outputs) \ + $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) + +cpp_no_group: cpp_no_group_protoc_middleman generate_gogo_data cpp-no-group-benchmark + ./cpp-no-group-benchmark $(gogo_data) + +gogo_proto_middleman: protoc-gen-gogoproto + mkdir -p "tmp/gogo_proto" + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-gogoproto --gogoproto_out=$$oldpwd/tmp/gogo_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) + touch gogo_proto_middleman + +gogo_data = $$(for data in $(all_data); do echo "tmp/gogo_data$${data\#$(srcdir)}"; done | xargs) + +generate_gogo_data: protoc_middleman protoc_middleman2 gogo-data-scrubber + mkdir -p `dirname $(gogo_data)` + ./gogo-data-scrubber $(all_data) $(gogo_data) + touch generate_gogo_data + +make_tmp_dir_gogo: + mkdir -p tmp/go_no_group/benchmark_code + mkdir -p tmp/gogofast/benchmark_code + mkdir -p tmp/gogofaster/benchmark_code + mkdir -p tmp/gogoslick/benchmark_code + touch make_tmp_dir_gogo + +go_no_group_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message4) ) + touch go_no_group_protoc_middleman + +cpp_no_group_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message4) ) + touch cpp_no_group_protoc_middleman + +gogofast_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogofast_protoc_middleman + +gogofaster_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogofaster_protoc_middleman + +gogoslick_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogoslick_protoc_middleman + +generate-gogo-benchmark-code: + @echo '#! /bin/bash' > generate-gogo-benchmark-code + @echo 'cp $(srcdir)/go/go_benchmark_test.go tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'sed -i -e "s/\.\.\/tmp/../g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'sed -i -e "s/b\.Run(\"\(.*\)\"/b.Run(\"\1\_$$1\"/g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'if [[ $$2 == 1 ]]; then sed -i -e "s/github\.com\/golang/github.com\/gogo/g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go; fi ' >> generate-gogo-benchmark-code + @chmod +x generate-gogo-benchmark-code + +generate_all_gogo_benchmark_code: generate-gogo-benchmark-code make_tmp_dir_gogo + ./generate-gogo-benchmark-code go_no_group 0 + ./generate-gogo-benchmark-code gogofast 1 + ./generate-gogo-benchmark-code gogofaster 1 + ./generate-gogo-benchmark-code gogoslick 1 + +gogo-benchmark: + @echo "Writing shortcut script gogo-benchmark..." + @echo '#! /bin/bash' > gogo-benchmark + @echo 'cd tmp/$$1/benchmark_code' >> gogo-benchmark + @echo 'shift' >> gogo-benchmark + @echo 'all_data=""' >> gogo-benchmark + @echo 'for data_file in $$@; do all_data="$$all_data ../../../$$data_file"; done' >> gogo-benchmark + @echo 'go test -bench=. -- $$all_data' >> gogo-benchmark + @echo 'cd ../..' >> gogo-benchmark + @chmod +x gogo-benchmark + +go_no_group: go_no_group_protoc_middleman generate_gogo_data generate_all_gogo_benchmark_code gogo-benchmark + ./gogo-benchmark go_no_group $(gogo_data) + +gogofast: gogofast_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogofast $(gogo_data) + +gogofaster: gogofaster_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogofaster $(gogo_data) + +gogoslick: gogoslick_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogoslick $(gogo_data) + + +############# GOGO RULES END ############ + + +############ UTIL RULES BEGIN ############ + +bin_PROGRAMS += protoc-gen-gogoproto gogo-data-scrubber protoc-gen-proto2_to_proto3 proto3-data-stripper + +protoc_gen_gogoproto_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/src/libprotoc.la +protoc_gen_gogoproto_SOURCES = util/protoc-gen-gogoproto.cc +protoc_gen_gogoproto_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util + +gogo_data_scrubber_LDADD = $(top_srcdir)/src/libprotobuf.la +gogo_data_scrubber_SOURCES = util/gogo_data_scrubber.cc +gogo_data_scrubber_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util +util/gogo_data_scrubber-gogo_data_scrubber.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) +nodist_gogo_data_scrubber_SOURCES = \ + $(benchmarks_protoc_outputs) \ + $(benchmarks_protoc_outputs_proto2) \ + $(benchmarks_protoc_outputs_proto2_header) \ + $(benchmarks_protoc_outputs_header) + +protoc_gen_proto2_to_proto3_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/src/libprotoc.la +protoc_gen_proto2_to_proto3_SOURCES = util/protoc-gen-proto2_to_proto3.cc +protoc_gen_proto2_to_proto3_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util + +proto3_data_stripper_LDADD = $(top_srcdir)/src/libprotobuf.la +proto3_data_stripper_SOURCES = util/proto3_data_stripper.cc +proto3_data_stripper_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util +util/proto3_data_stripper-proto3_data_stripper.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) +nodist_proto3_data_stripper_SOURCES = \ + $(benchmarks_protoc_outputs) \ + $(benchmarks_protoc_outputs_proto2) \ + $(benchmarks_protoc_outputs_proto2_header) \ + $(benchmarks_protoc_outputs_header) + + +############ UTIL RULES END ############ + +############ PROTO3 PREPARATION BEGIN ############# + +proto3_proto_middleman: protoc-gen-proto2_to_proto3 + mkdir -p "tmp/proto3_proto" + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-proto2_to_proto3 --proto2_to_proto3_out=$$oldpwd/tmp/proto3_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) + touch proto3_proto_middleman + +proto3_data = $$(for data in $(all_data); do echo "tmp/proto3_data$${data\#$(srcdir)}"; done | xargs) + +generate_proto3_data: protoc_middleman protoc_middleman2 proto3-data-stripper + mkdir -p `dirname $(proto3_data)` + ./proto3-data-stripper $(all_data) $(proto3_data) + touch generate_proto3_data + +############ PROTO3 PREPARATION END ############# + MAINTAINERCLEANFILES = \ Makefile.in @@ -241,7 +517,35 @@ CLEANFILES = \ python_cpp_proto_library \ python-pure-python-benchmark \ python-cpp-reflection-benchmark \ - python-cpp-generated-code-benchmark + python-cpp-generated-code-benchmark \ + go-benchmark \ + go_protoc_middleman \ + make_tmp_dir_gogo \ + gogo_proto_middleman \ + generate_gogo_data \ + go_no_group_protoc_middleman \ + go_no_group \ + go-no-group-benchmark \ + $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2) \ + generate_all_gogo_benchmark_code \ + generate-gogo-benchmark-code \ + cpp_no_group_protoc_middleman \ + generate_cpp_no_group_benchmark_code \ + generate_gogo_benchmark_code \ + gogofast_protoc_middleman \ + gogofast \ + gogofaster_protoc_middleman \ + gogofaster \ + gogoslick_protoc_middleman \ + gogoslick \ + gogo-benchmark \ + gogo/cpp_no_group/cpp_benchmark.* \ + proto3_proto_middleman \ + generate_proto3_data + clean-local: -rm -rf tmp/* diff --git a/benchmarks/README.md b/benchmarks/README.md index 459c7b9d..ae5c7ddd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -3,7 +3,9 @@ This directory contains benchmarking schemas and data sets that you can use to test a variety of performance scenarios against your -protobuf language runtime. +protobuf language runtime. If you are looking for performance +numbers of officially support languages, see [here]( +https://github.com/google/protobuf/blob/master/docs/performance.md) ## Prerequisite @@ -17,6 +19,11 @@ We are using [google/benchmark](https://github.com/google/benchmark) as the benchmark tool for testing cpp. This will be automaticly made during build the cpp benchmark. +The cpp protobuf performance can be improved by linking with [tcmalloc library]( +https://gperftools.github.io/gperftools/tcmalloc.html). For using tcmalloc, you +need to build [gpertools](https://github.com/gperftools/gperftools) to generate +libtcmallc.so library. + ### Java We're using maven to build the java benchmarks, which is the same as to build the Java protobuf. There're no other tools need to install. We're using @@ -36,6 +43,21 @@ $ sudo apt-get install python3-dev ``` And you also need to make sure `pkg-config` is installed. +### Go +Go protobufs are maintained at [github.com/golang/protobuf]( +http://github.com/golang/protobuf). If not done already, you need to install the +toolchain and the Go protoc-gen-go plugin for protoc. + +To install protoc-gen-go, run: + +``` +$ go get -u github.com/golang/protobuf/protoc-gen-go +$ export PATH=$PATH:$(go env GOPATH)/bin +``` + +The first command installs `protoc-gen-go` into the `bin` directory in your local `GOPATH`. +The second command adds the `bin` directory to your `PATH` so that `protoc` can locate the plugin later. + ### Big data There's some optional big testing data which is not included in the directory @@ -64,6 +86,12 @@ $ make java $ make cpp ``` +For linking with tcmalloc: + +``` +$ env LD_PRELOAD={directory to libtcmalloc.so} make cpp +``` + ### Python: We have three versions of python protobuf implementation: pure python, cpp @@ -87,45 +115,59 @@ $ make python-cpp-reflection $ make python-cpp-generated-code ``` -To run a specific dataset: +### Go +``` +$ make go +``` + +To run a specific dataset or run with specific options: ### Java: ``` $ make java-benchmark -$ ./java-benchmark $(specific generated dataset file name) [-- $(caliper option)] +$ ./java-benchmark $(specific generated dataset file name) [$(caliper options)] ``` ### CPP: ``` $ make cpp-benchmark -$ ./cpp-benchmark $(specific generated dataset file name) +$ ./cpp-benchmark $(specific generated dataset file name) [$(benchmark options)] ``` ### Python: +For Python benchmark we have `--json` for outputing the json result + #### Pure Python: ``` $ make python-pure-python-benchmark -$ ./python-pure-python-benchmark $(specific generated dataset file name) +$ ./python-pure-python-benchmark [--json] $(specific generated dataset file name) ``` #### CPP reflection: ``` $ make python-cpp-reflection-benchmark -$ ./python-cpp-reflection-benchmark $(specific generated dataset file name) +$ ./python-cpp-reflection-benchmark [--json] $(specific generated dataset file name) ``` #### CPP generated code: ``` $ make python-cpp-generated-code-benchmark -$ ./python-cpp-generated-code-benchmark $(specific generated dataset file name) +$ ./python-cpp-generated-code-benchmark [--json] $(specific generated dataset file name) ``` +### Go: +``` +$ make go-benchmark +$ ./go-benchmark $(specific generated dataset file name) [go testing options] +``` + + ## Benchmark datasets Each data set is in the format of benchmarks.proto: diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp/cpp_benchmark.cc index 0ba4dc52..f8b55291 100644 --- a/benchmarks/cpp_benchmark.cc +++ b/benchmarks/cpp/cpp_benchmark.cc @@ -32,8 +32,8 @@ #include <iostream> #include "benchmark/benchmark_api.h" #include "benchmarks.pb.h" -#include "datasets/google_message1/benchmark_message1_proto2.pb.h" -#include "datasets/google_message1/benchmark_message1_proto3.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" #include "datasets/google_message2/benchmark_message2.pb.h" #include "datasets/google_message3/benchmark_message3.pb.h" #include "datasets/google_message4/benchmark_message4.pb.h" diff --git a/benchmarks/datasets/google_message1/benchmark_message1_proto2.proto b/benchmarks/datasets/google_message1/proto2/benchmark_message1_proto2.proto index 21261905..21261905 100644 --- a/benchmarks/datasets/google_message1/benchmark_message1_proto2.proto +++ b/benchmarks/datasets/google_message1/proto2/benchmark_message1_proto2.proto diff --git a/benchmarks/datasets/google_message1/dataset.google_message1_proto2.pb b/benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb Binary files differindex f6fe7848..f6fe7848 100644 --- a/benchmarks/datasets/google_message1/dataset.google_message1_proto2.pb +++ b/benchmarks/datasets/google_message1/proto2/dataset.google_message1_proto2.pb diff --git a/benchmarks/datasets/google_message1/benchmark_message1_proto3.proto b/benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto index 090b554b..090b554b 100644 --- a/benchmarks/datasets/google_message1/benchmark_message1_proto3.proto +++ b/benchmarks/datasets/google_message1/proto3/benchmark_message1_proto3.proto diff --git a/benchmarks/datasets/google_message1/dataset.google_message1_proto3.pb b/benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb Binary files differindex 4955bed3..4955bed3 100644 --- a/benchmarks/datasets/google_message1/dataset.google_message1_proto3.pb +++ b/benchmarks/datasets/google_message1/proto3/dataset.google_message1_proto3.pb diff --git a/benchmarks/go/go_benchmark_test.go b/benchmarks/go/go_benchmark_test.go new file mode 100644 index 00000000..8c741b71 --- /dev/null +++ b/benchmarks/go/go_benchmark_test.go @@ -0,0 +1,124 @@ +package main + +import ( + benchmarkWrapper "../tmp" + googleMessage1Proto2 "../tmp/datasets/google_message1/proto2" + googleMessage1Proto3 "../tmp/datasets/google_message1/proto3" + googleMessage2 "../tmp/datasets/google_message2" + googleMessage3 "../tmp/datasets/google_message3" + googleMessage4 "../tmp/datasets/google_message4" + "flag" + "github.com/golang/protobuf/proto" + "io/ioutil" + "testing" +) + +// Data is returned by the Load function. +type Dataset struct { + name string + newMessage func() proto.Message + marshaled [][]byte + unmarshaled []proto.Message +} + +var datasets []Dataset + +// This is used to getDefaultInstance for a message type. +func generateNewMessageFunction(dataset benchmarkWrapper.BenchmarkDataset) func() proto.Message { + switch dataset.MessageName { + case "benchmarks.proto3.GoogleMessage1": + return func() proto.Message { return new(googleMessage1Proto3.GoogleMessage1) } + case "benchmarks.proto2.GoogleMessage1": + return func() proto.Message { return new(googleMessage1Proto2.GoogleMessage1) } + case "benchmarks.proto2.GoogleMessage2": + return func() proto.Message { return new(googleMessage2.GoogleMessage2) } + case "benchmarks.google_message3.GoogleMessage3": + return func() proto.Message { return new(googleMessage3.GoogleMessage3) } + case "benchmarks.google_message4.GoogleMessage4": + return func() proto.Message { return new(googleMessage4.GoogleMessage4) } + default: + panic("Unknown message type: " + dataset.MessageName) + } +} + +func init() { + flag.Parse() + for _, f := range flag.Args() { + // Load the benchmark. + b, err := ioutil.ReadFile(f) + if err != nil { + panic(err) + } + + // Parse the benchmark. + var dm benchmarkWrapper.BenchmarkDataset + if err := proto.Unmarshal(b, &dm); err != nil { + panic(err) + } + + // Determine the concrete protobuf message type to use. + var ds Dataset + ds.newMessage = generateNewMessageFunction(dm) + + // Unmarshal each test message. + for _, payload := range dm.Payload { + ds.marshaled = append(ds.marshaled, payload) + m := ds.newMessage() + if err := proto.Unmarshal(payload, m); err != nil { + panic(err) + } + ds.unmarshaled = append(ds.unmarshaled, m) + } + ds.name = f + + datasets = append(datasets, ds) + } +} + +func Benchmark(b *testing.B) { + for _, ds := range datasets { + b.Run(ds.name, func(b *testing.B) { + b.Run("Unmarshal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for j, payload := range ds.marshaled { + out := ds.newMessage() + if err := proto.Unmarshal(payload, out); err != nil { + b.Fatalf("can't unmarshal message %d %v", j, err) + } + } + } + }) + b.Run("Marshal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for j, m := range ds.unmarshaled { + if _, err := proto.Marshal(m); err != nil { + b.Fatalf("can't marshal message %d %+v: %v", j, m, err) + } + } + } + }) + b.Run("Size", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + proto.Size(m) + } + } + }) + b.Run("Clone", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + proto.Clone(m) + } + } + }) + b.Run("Merge", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + out := ds.newMessage() + proto.Merge(out, m) + } + } + }) + }) + } +} diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index c2cd78a1..570bd664 100755..100644 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -14,7 +14,10 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>3.5.0</version> + <version>${protobuf.version}</version> + <type>jar</type> + <scope>system</scope> + <systemPath>${project.basedir}/lib/protobuf-java.jar</systemPath> </dependency> <dependency> <groupId>com.google.caliper</groupId> diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java deleted file mode 100755 index f35b180a..00000000 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java +++ /dev/null @@ -1,178 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2009 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -package com.google.protobuf; - -import com.google.caliper.BeforeExperiment; -import com.google.caliper.Benchmark; -import com.google.caliper.Param; -import com.google.caliper.runner.CaliperMain; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.Message; -import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.RandomAccessFile; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Scanner; - - -public class ProtoBench { - - private ProtoBench() { - // Prevent instantiation - } - - public static void main(String[] args) { - if (args.length < 1) { - System.err.println("Usage: ./java-benchmark <input data>"); - System.err.println("input data is in the format of \"benchmarks.proto\""); - System.exit(1); - } - boolean success = true; - for (int i = 0; i < args.length; i++) { - success &= runTest(args[i]); - } - System.exit(success ? 0 : 1); - } - - - /** - * Runs a single test with specific test data. Error messages are displayed to stderr, - * and the return value indicates general success/failure. - */ - public static boolean runTest(String file) { - byte[] inputData; - BenchmarkDataset benchmarkDataset; - try { - inputData = readAllBytes(file); - benchmarkDataset = BenchmarkDataset.parseFrom(inputData); - } catch (IOException e) { - System.err.println("Unable to get input data"); - return false; - } - List<String> argsList = getCaliperOption(benchmarkDataset); - if (argsList == null) { - System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); - return false; - } - argsList.add("-DdataFile=" + file); - argsList.add("com.google.protobuf.ProtoCaliperBenchmark"); - - try { - String args[] = new String[argsList.size()]; - argsList.toArray(args); - CaliperMain.exitlessMain(args, - new PrintWriter(System.out, true), new PrintWriter(System.err, true)); - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - System.err.println("Detailed exception information:"); - e.printStackTrace(System.err); - return false; - } - try { - double mininumScale = 0; - // If the file not exist, this will throw IOException, which won't print the warning - // information below. - Scanner scanner = new Scanner(new String(readAllBytes("JavaBenchmarkWarning.txt"))); - while (scanner.hasNext()) { - mininumScale = Math.max(mininumScale, scanner.nextDouble()); - } - scanner.close(); - - System.out.println( - "WARNING: This benchmark's whole iterations are not enough, consider to config caliper to " - + "run for more time to make the result more convincing. You may change the configure " - + "code in com.google.protobuf.ProtoBench.getCaliperOption() of benchmark " - + benchmarkDataset.getMessageName() - + " to run for more time. e.g. Change the value of " - + "instrument.runtime.options.timingInterval or value of " - + "instrument.runtime.options.measurements to be at least " - + Math.round(mininumScale * 10 + 1) / 10.0 - + " times of before, then build and run the benchmark again\n"); - Files.deleteIfExists(Paths.get("JavaBenchmarkWarning.txt")); - } catch (IOException e) { - // The IOException here should be file not found, which means there's no warning generated by - // The benchmark, so this IOException should be discarded. - } - return true; - } - - - private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDataset) { - List<String> temp = new ArrayList<String>(); - if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message3.GoogleMessage3")) { - temp.add("-Cinstrument.runtime.options.timingInterval=3000ms"); - temp.add("-Cinstrument.runtime.options.measurements=20"); - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message4.GoogleMessage4")) { - temp.add("-Cinstrument.runtime.options.timingInterval=1500ms"); - temp.add("-Cinstrument.runtime.options.measurements=20"); - } else { - return null; - } - - temp.add("-i"); - temp.add("runtime"); - temp.add("-b"); - String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" - + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; - temp.add(benchmarkNames); - - return temp; - } - - public static byte[] readAllBytes(String filename) throws IOException { - if (filename.equals("")) { - return new byte[0]; - } - RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); - byte[] content = new byte[(int) file.length()]; - file.readFully(content); - return content; - } -} diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java index 7dff16dc..c766d74e 100755..100644 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -5,6 +5,7 @@ import com.google.caliper.BeforeExperiment; import com.google.caliper.AfterExperiment; import com.google.caliper.Benchmark; import com.google.caliper.Param; +import com.google.caliper.api.VmOptions; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.ExtensionRegistry; @@ -22,6 +23,12 @@ import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.List; +// Caliper set CICompilerCount to 1 for making sure compilation doesn't run in parallel with itself, +// This makes TieredCompilation not working. We just disable TieredCompilation by default. In master +// branch this has been disabled by default in caliper: +// https://github.com/google/caliper/blob/master/caliper-runner/src/main/java/com/google/caliper/runner/target/Jvm.java#L38:14 +// But this haven't been added into most recent release. +@VmOptions("-XX:-TieredCompilation") public class ProtoCaliperBenchmark { public enum BenchmarkMessageType { GOOGLE_MESSAGE1_PROTO3 { @@ -99,7 +106,6 @@ public class ProtoCaliperBenchmark { private List<ByteArrayInputStream> inputStreamList; private List<ByteString> inputStringList; private List<Message> sampleMessageList; - private long counter; private BenchmarkMessageType getMessageType() throws IOException { if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { @@ -149,30 +155,18 @@ public class ProtoCaliperBenchmark { sampleMessageList.add( defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); } - - counter = 0; } @Benchmark - void serializeToByteString(int reps) throws IOException { - if (sampleMessageList.size() == 0) { - return; - } - for (int i = 0; i < reps; i++) { - sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString(); - counter++; - } - } - - @Benchmark void serializeToByteArray(int reps) throws IOException { if (sampleMessageList.size() == 0) { return; } for (int i = 0; i < reps; i++) { - sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray(); - counter++; + for (int j = 0; j < sampleMessageList.size(); j++) { + sampleMessageList.get(j).toByteArray(); + } } } @@ -182,21 +176,10 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output); - counter++; - } - } - - @Benchmark - void deserializeFromByteString(int reps) throws IOException { - if (inputStringList.size() == 0) { - return; - } - for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStringList.get((int) (counter % inputStringList.size())), extensions); - counter++; + for (int j = 0; j < sampleMessageList.size(); j++) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + sampleMessageList.get(j).writeTo(output); + } } } @@ -206,9 +189,10 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputDataList.get((int) (counter % inputDataList.size())), extensions); - counter++; + for (int j = 0; j < inputDataList.size(); j++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputDataList.get(j), extensions); + } } } @@ -218,28 +202,13 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStreamList.get((int) (counter % inputStreamList.size())), extensions); - inputStreamList.get((int) (counter % inputStreamList.size())).reset(); - counter++; - } - } - - @AfterExperiment - void checkCounter() throws IOException { - if (counter == 1) { - // Dry run - return; - } - if (benchmarkDataset.getPayloadCount() != 1 - && counter < benchmarkDataset.getPayloadCount() * 10L) { - BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true)); - // If the total number of non-warmup reps is smaller than 100 times of the total number of - // datasets, then output the scale that need to multiply to the configuration (either extend - // the running time for one timingInterval or run for more measurements). - writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " "); - writer.close(); + for (int j = 0; j < inputStreamList.size(); j++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStreamList.get(j), extensions); + inputStreamList.get(j).reset(); + } } } } + diff --git a/benchmarks/python/__init__.py b/benchmarks/python/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/benchmarks/python/__init__.py diff --git a/benchmarks/py_benchmark.py b/benchmarks/python/py_benchmark.py index 48234f03..e86b61e7 100755 --- a/benchmarks/py_benchmark.py +++ b/benchmarks/python/py_benchmark.py @@ -1,24 +1,39 @@ +from __future__ import print_function import sys import os import timeit import math +import argparse import fnmatch - +import json + +parser = argparse.ArgumentParser(description="Python protobuf benchmark") +parser.add_argument("data_files", metavar="dataFile", nargs="+", + help="testing data files.") +parser.add_argument("--json", action="store_const", dest="json", + const="yes", default="no", + help="Whether to output json results") +parser.add_argument("--behavior_prefix", dest="behavior_prefix", + help="The output json format's behavior's name's prefix", + default="") +# BEGIN CPP GENERATED MESSAGE +parser.add_argument("--cpp_generated", action="store_const", + dest="cpp_generated", const="yes", default="no", + help="Whether to link generated code library") +# END CPP GENERATED MESSAGE +args = parser.parse_args() # BEGIN CPP GENERATED MESSAGE # CPP generated code must be linked before importing the generated Python code # for the descriptor can be found in the pool -if len(sys.argv) < 2: - raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") -if sys.argv[1] == "true": +if args.cpp_generated != "no": sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/.libs" ) import libbenchmark_messages sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/tmp" ) -elif sys.argv[1] != "false": - raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") # END CPP GENERATED MESSAGE -import datasets.google_message1.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 -import datasets.google_message1.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 + +import datasets.google_message1.proto2.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 +import datasets.google_message1.proto3.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2 import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2 import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2 @@ -26,19 +41,24 @@ import benchmarks_pb2 as benchmarks_pb2 def run_one_test(filename): - data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() + data = open(filename).read() benchmark_dataset = benchmarks_pb2.BenchmarkDataset() benchmark_dataset.ParseFromString(data) benchmark_util = Benchmark(full_iteration=len(benchmark_dataset.payload), module="py_benchmark", setup_method="init") - print "Message %s of dataset file %s" % \ - (benchmark_dataset.message_name, filename) + result={} + result["filename"] = filename + result["message_name"] = benchmark_dataset.message_name + result["benchmarks"] = {} benchmark_util.set_test_method("parse_from_benchmark") - print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) + result["benchmarks"][args.behavior_prefix + "_parse_from_benchmark"] = \ + benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) benchmark_util.set_test_method("serialize_to_benchmark") - print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) - print "" + result["benchmarks"][args.behavior_prefix + "_serialize_to_benchmark"] = \ + benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) + return result + def init(filename): global benchmark_dataset, message_class, message_list, counter @@ -66,11 +86,13 @@ def init(filename): temp.ParseFromString(one_payload) message_list.append(temp) + def parse_from_benchmark(): global counter, message_class, benchmark_dataset m = message_class().ParseFromString(benchmark_dataset.payload[counter % len(benchmark_dataset.payload)]) counter = counter + 1 + def serialize_to_benchmark(): global counter, message_list, message_class s = message_list[counter % len(benchmark_dataset.payload)].SerializeToString() @@ -108,10 +130,24 @@ class Benchmark: t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), setup=self.full_setup_code(setup_method_args), number=reps); - return "Average time for %s: %.2f ns" % \ - (self.test_method, 1.0 * t / reps * (10 ** 9)) - + return 1.0 * t / reps * (10 ** 9) + if __name__ == "__main__": - for i in range(2, len(sys.argv)): - run_one_test(sys.argv[i]) + results = [] + for file in args.data_files: + results.append(run_one_test(file)) + + if args.json != "no": + print(json.dumps(results)) + else: + for result in results: + print("Message %s of dataset file %s" % \ + (result["message_name"], result["filename"])) + print("Average time for parse_from_benchmark: %.2f ns" % \ + (result["benchmarks"][ \ + args.behavior_prefix + "_parse_from_benchmark"])) + print("Average time for serialize_to_benchmark: %.2f ns" % \ + (result["benchmarks"][ \ + args.behavior_prefix + "_serialize_to_benchmark"])) + print("") diff --git a/benchmarks/python_benchmark_messages.cc b/benchmarks/python/python_benchmark_messages.cc index 55242a2a..ded16fe9 100644 --- a/benchmarks/python_benchmark_messages.cc +++ b/benchmarks/python/python_benchmark_messages.cc @@ -1,8 +1,8 @@ #include <Python.h> #include "benchmarks.pb.h" -#include "datasets/google_message1/benchmark_message1_proto2.pb.h" -#include "datasets/google_message1/benchmark_message1_proto3.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" #include "datasets/google_message2/benchmark_message2.pb.h" #include "datasets/google_message3/benchmark_message3.pb.h" #include "datasets/google_message4/benchmark_message4.pb.h" diff --git a/benchmarks/util/__init__.py b/benchmarks/util/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/benchmarks/util/__init__.py diff --git a/benchmarks/util/big_query_utils.py b/benchmarks/util/big_query_utils.py new file mode 100755 index 00000000..aea55bbd --- /dev/null +++ b/benchmarks/util/big_query_utils.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function +import argparse +import json +import uuid +import httplib2 + +from apiclient import discovery +from apiclient.errors import HttpError +from oauth2client.client import GoogleCredentials + +# 30 days in milliseconds +_EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000 +NUM_RETRIES = 3 + + +def create_big_query(): + """Authenticates with cloud platform and gets a BiqQuery service object + """ + creds = GoogleCredentials.get_application_default() + return discovery.build( + 'bigquery', 'v2', credentials=creds, cache_discovery=False) + + +def create_dataset(biq_query, project_id, dataset_id): + is_success = True + body = { + 'datasetReference': { + 'projectId': project_id, + 'datasetId': dataset_id + } + } + + try: + dataset_req = biq_query.datasets().insert( + projectId=project_id, body=body) + dataset_req.execute(num_retries=NUM_RETRIES) + except HttpError as http_error: + if http_error.resp.status == 409: + print('Warning: The dataset %s already exists' % dataset_id) + else: + # Note: For more debugging info, print "http_error.content" + print('Error in creating dataset: %s. Err: %s' % (dataset_id, + http_error)) + is_success = False + return is_success + + +def create_table(big_query, project_id, dataset_id, table_id, table_schema, + description): + fields = [{ + 'name': field_name, + 'type': field_type, + 'description': field_description + } for (field_name, field_type, field_description) in table_schema] + return create_table2(big_query, project_id, dataset_id, table_id, fields, + description) + + +def create_partitioned_table(big_query, + project_id, + dataset_id, + table_id, + table_schema, + description, + partition_type='DAY', + expiration_ms=_EXPIRATION_MS): + """Creates a partitioned table. By default, a date-paritioned table is created with + each partition lasting 30 days after it was last modified. + """ + fields = [{ + 'name': field_name, + 'type': field_type, + 'description': field_description + } for (field_name, field_type, field_description) in table_schema] + return create_table2(big_query, project_id, dataset_id, table_id, fields, + description, partition_type, expiration_ms) + + +def create_table2(big_query, + project_id, + dataset_id, + table_id, + fields_schema, + description, + partition_type=None, + expiration_ms=None): + is_success = True + + body = { + 'description': description, + 'schema': { + 'fields': fields_schema + }, + 'tableReference': { + 'datasetId': dataset_id, + 'projectId': project_id, + 'tableId': table_id + } + } + + if partition_type and expiration_ms: + body["timePartitioning"] = { + "type": partition_type, + "expirationMs": expiration_ms + } + + try: + table_req = big_query.tables().insert( + projectId=project_id, datasetId=dataset_id, body=body) + res = table_req.execute(num_retries=NUM_RETRIES) + print('Successfully created %s "%s"' % (res['kind'], res['id'])) + except HttpError as http_error: + if http_error.resp.status == 409: + print('Warning: Table %s already exists' % table_id) + else: + print('Error in creating table: %s. Err: %s' % (table_id, + http_error)) + is_success = False + return is_success + + +def patch_table(big_query, project_id, dataset_id, table_id, fields_schema): + is_success = True + + body = { + 'schema': { + 'fields': fields_schema + }, + 'tableReference': { + 'datasetId': dataset_id, + 'projectId': project_id, + 'tableId': table_id + } + } + + try: + table_req = big_query.tables().patch( + projectId=project_id, + datasetId=dataset_id, + tableId=table_id, + body=body) + res = table_req.execute(num_retries=NUM_RETRIES) + print('Successfully patched %s "%s"' % (res['kind'], res['id'])) + except HttpError as http_error: + print('Error in creating table: %s. Err: %s' % (table_id, http_error)) + is_success = False + return is_success + + +def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): + is_success = True + body = {'rows': rows_list} + try: + insert_req = big_query.tabledata().insertAll( + projectId=project_id, + datasetId=dataset_id, + tableId=table_id, + body=body) + res = insert_req.execute(num_retries=NUM_RETRIES) + if res.get('insertErrors', None): + print('Error inserting rows! Response: %s' % res) + is_success = False + except HttpError as http_error: + print('Error inserting rows to the table %s' % table_id) + is_success = False + + return is_success + + +def sync_query_job(big_query, project_id, query, timeout=5000): + query_data = {'query': query, 'timeoutMs': timeout} + query_job = None + try: + query_job = big_query.jobs().query( + projectId=project_id, + body=query_data).execute(num_retries=NUM_RETRIES) + except HttpError as http_error: + print('Query execute job failed with error: %s' % http_error) + print(http_error.content) + return query_job + + + # List of (column name, column type, description) tuples +def make_row(unique_row_id, row_values_dict): + """row_values_dict is a dictionary of column name and column value. + """ + return {'insertId': unique_row_id, 'json': row_values_dict} diff --git a/benchmarks/util/data_proto2_to_proto3_util.h b/benchmarks/util/data_proto2_to_proto3_util.h new file mode 100644 index 00000000..5eea8509 --- /dev/null +++ b/benchmarks/util/data_proto2_to_proto3_util.h @@ -0,0 +1,64 @@ +#ifndef PROTOBUF_BENCHMARKS_UTIL_DATA_PROTO2_TO_PROTO3_UTIL_H_ +#define PROTOBUF_BENCHMARKS_UTIL_DATA_PROTO2_TO_PROTO3_UTIL_H_ + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" + +using google::protobuf::FieldDescriptor; +using google::protobuf::Message; +using google::protobuf::Reflection; + +namespace google { +namespace protobuf { +namespace util { + +class DataStripper { + public: + void StripMessage(Message *message) { + std::vector<const FieldDescriptor*> set_fields; + const Reflection* reflection = message->GetReflection(); + reflection->ListFields(*message, &set_fields); + + for (size_t i = 0; i < set_fields.size(); i++) { + const FieldDescriptor* field = set_fields[i]; + if (ShouldBeClear(field)) { + reflection->ClearField(message, field); + continue; + } + if (field->type() == FieldDescriptor::TYPE_MESSAGE) { + if (field->is_repeated()) { + for (int j = 0; j < reflection->FieldSize(*message, field); j++) { + StripMessage(reflection->MutableRepeatedMessage(message, field, j)); + } + } else { + StripMessage(reflection->MutableMessage(message, field)); + } + } + } + + reflection->MutableUnknownFields(message)->Clear(); + } + private: + virtual bool ShouldBeClear(const FieldDescriptor *field) = 0; +}; + +class GogoDataStripper : public DataStripper { + private: + virtual bool ShouldBeClear(const FieldDescriptor *field) { + return field->type() == FieldDescriptor::TYPE_GROUP; + } +}; + +class Proto3DataStripper : public DataStripper { + private: + virtual bool ShouldBeClear(const FieldDescriptor *field) { + return field->type() == FieldDescriptor::TYPE_GROUP || + field->is_extension(); + } +}; + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // PROTOBUF_BENCHMARKS_UTIL_DATA_PROTO2_TO_PROTO3_UTIL_H_ diff --git a/benchmarks/util/gogo_data_scrubber.cc b/benchmarks/util/gogo_data_scrubber.cc new file mode 100644 index 00000000..9ef57b0d --- /dev/null +++ b/benchmarks/util/gogo_data_scrubber.cc @@ -0,0 +1,74 @@ +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" +#include "data_proto2_to_proto3_util.h" + +#include <fstream> + +using google::protobuf::util::GogoDataStripper; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" + << name + << "', please make sure you are running this command from the benchmarks" + << " directory.\n"; + return std::string((std::istreambuf_iterator<char>(file)), + std::istreambuf_iterator<char>()); +} + +int main(int argc, char *argv[]) { + if (argc % 2 == 0 || argc == 1) { + std::cerr << "Usage: [input_files] [output_file_names] where " << + "input_files are one to one mapping to output_file_names." << + std::endl; + return 1; + } + + for (int i = argc / 2; i > 0; i--) { + const std::string &input_file = argv[i]; + const std::string &output_file = argv[i + argc / 2]; + + std::cerr << "Generating " << input_file + << " to " << output_file << std::endl; + benchmarks::BenchmarkDataset dataset; + Message* message; + std::string dataset_payload = ReadFile(input_file); + GOOGLE_CHECK(dataset.ParseFromString(dataset_payload)) + << "Can' t parse data file " << input_file; + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + message = new benchmarks::proto3::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + message = new benchmarks::proto2::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + message = new benchmarks::proto2::GoogleMessage2; + } else if (dataset.message_name() == + "benchmarks.google_message3.GoogleMessage3") { + message = new benchmarks::google_message3::GoogleMessage3; + } else if (dataset.message_name() == + "benchmarks.google_message4.GoogleMessage4") { + message = new benchmarks::google_message4::GoogleMessage4; + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } + + for (int i = 0; i < dataset.payload_size(); i++) { + message->ParseFromString(dataset.payload(i)); + GogoDataStripper stripper; + stripper.StripMessage(message); + dataset.set_payload(i, message->SerializeAsString()); + } + + std::ofstream ofs(output_file); + ofs << dataset.SerializeAsString(); + ofs.close(); + } + + + return 0; +} diff --git a/benchmarks/util/proto3_data_stripper.cc b/benchmarks/util/proto3_data_stripper.cc new file mode 100644 index 00000000..3096c4c1 --- /dev/null +++ b/benchmarks/util/proto3_data_stripper.cc @@ -0,0 +1,74 @@ +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" +#include "data_proto2_to_proto3_util.h" + +#include <fstream> + +using google::protobuf::util::Proto3DataStripper; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" + << name + << "', please make sure you are running this command from the benchmarks" + << " directory.\n"; + return std::string((std::istreambuf_iterator<char>(file)), + std::istreambuf_iterator<char>()); +} + +int main(int argc, char *argv[]) { + if (argc % 2 == 0 || argc == 1) { + std::cerr << "Usage: [input_files] [output_file_names] where " << + "input_files are one to one mapping to output_file_names." << + std::endl; + return 1; + } + + for (int i = argc / 2; i > 0; i--) { + const std::string &input_file = argv[i]; + const std::string &output_file = argv[i + argc / 2]; + + std::cerr << "Generating " << input_file + << " to " << output_file << std::endl; + benchmarks::BenchmarkDataset dataset; + Message* message; + std::string dataset_payload = ReadFile(input_file); + GOOGLE_CHECK(dataset.ParseFromString(dataset_payload)) + << "Can' t parse data file " << input_file; + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + message = new benchmarks::proto3::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + message = new benchmarks::proto2::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + message = new benchmarks::proto2::GoogleMessage2; + } else if (dataset.message_name() == + "benchmarks.google_message3.GoogleMessage3") { + message = new benchmarks::google_message3::GoogleMessage3; + } else if (dataset.message_name() == + "benchmarks.google_message4.GoogleMessage4") { + message = new benchmarks::google_message4::GoogleMessage4; + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } + + for (int i = 0; i < dataset.payload_size(); i++) { + message->ParseFromString(dataset.payload(i)); + Proto3DataStripper stripper; + stripper.StripMessage(message); + dataset.set_payload(i, message->SerializeAsString()); + } + + std::ofstream ofs(output_file); + ofs << dataset.SerializeAsString(); + ofs.close(); + } + + + return 0; +} diff --git a/benchmarks/util/protoc-gen-gogoproto.cc b/benchmarks/util/protoc-gen-gogoproto.cc new file mode 100644 index 00000000..9c1b3d04 --- /dev/null +++ b/benchmarks/util/protoc-gen-gogoproto.cc @@ -0,0 +1,103 @@ +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/printer.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "schema_proto2_to_proto3_util.h" + +#include "google/protobuf/compiler/plugin.h" + +using google::protobuf::FileDescriptorProto; +using google::protobuf::FileDescriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::io::Printer; +using google::protobuf::util::SchemaGroupStripper; +using google::protobuf::util::EnumScrubber; + +namespace google { +namespace protobuf { +namespace compiler { + +namespace { + +string StripProto(string filename) { + if (filename.substr(filename.size() - 11) == ".protodevel") { + // .protodevel + return filename.substr(0, filename.size() - 11); + } else { + // .proto + return filename.substr(0, filename.size() - 6); + } +} + +DescriptorPool new_pool_; + +} // namespace + +class GoGoProtoGenerator : public CodeGenerator { + public: + virtual bool GenerateAll(const std::vector<const FileDescriptor*>& files, + const string& parameter, + GeneratorContext* context, + string* error) const { + for (int i = 0; i < files.size(); i++) { + for (auto file : files) { + bool can_generate = + (new_pool_.FindFileByName(file->name()) == nullptr); + for (int j = 0; j < file->dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->dependency(j)->name()) != nullptr); + } + for (int j = 0; j < file->public_dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->public_dependency(j)->name()) != nullptr); + } + for (int j = 0; j < file->weak_dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->weak_dependency(j)->name()) != nullptr); + } + if (can_generate) { + Generate(file, parameter, context, error); + break; + } + } + } + + return true; + } + + virtual bool Generate(const FileDescriptor* file, + const string& parameter, + GeneratorContext* context, + string* error) const { + FileDescriptorProto new_file; + file->CopyTo(&new_file); + SchemaGroupStripper::StripFile(file, &new_file); + + EnumScrubber enum_scrubber; + enum_scrubber.ScrubFile(&new_file); + + string filename = file->name(); + string basename = StripProto(filename); + + std::vector<std::pair<string,string>> option_pairs; + ParseGeneratorParameter(parameter, &option_pairs); + + std::unique_ptr<google::protobuf::io::ZeroCopyOutputStream> output( + context->Open(basename + ".proto")); + string content = new_pool_.BuildFile(new_file)->DebugString(); + Printer printer(output.get(), '$'); + printer.WriteRaw(content.c_str(), content.size()); + + return true; + } +}; + +} // namespace compiler +} // namespace protobuf +} // namespace google + +int main(int argc, char* argv[]) { + google::protobuf::compiler::GoGoProtoGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/benchmarks/util/protoc-gen-proto2_to_proto3.cc b/benchmarks/util/protoc-gen-proto2_to_proto3.cc new file mode 100644 index 00000000..d0a89023 --- /dev/null +++ b/benchmarks/util/protoc-gen-proto2_to_proto3.cc @@ -0,0 +1,115 @@ +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/printer.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "schema_proto2_to_proto3_util.h" + +#include "google/protobuf/compiler/plugin.h" + +using google::protobuf::FileDescriptorProto; +using google::protobuf::FileDescriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::io::Printer; +using google::protobuf::util::SchemaGroupStripper; +using google::protobuf::util::EnumScrubber; +using google::protobuf::util::ExtensionStripper; +using google::protobuf::util::FieldScrubber; + +namespace google { +namespace protobuf { +namespace compiler { + +namespace { + +string StripProto(string filename) { + return filename.substr(0, filename.rfind(".proto")); +} + +DescriptorPool* GetPool() { + static DescriptorPool *pool = new DescriptorPool(); + return pool; +} + +} // namespace + +class Proto2ToProto3Generator final : public CodeGenerator { + public: + bool GenerateAll(const std::vector<const FileDescriptor*>& files, + const string& parameter, + GeneratorContext* context, + string* error) const { + for (int i = 0; i < files.size(); i++) { + for (auto file : files) { + if (CanGenerate(file)) { + Generate(file, parameter, context, error); + break; + } + } + } + + return true; + } + + bool Generate(const FileDescriptor* file, + const string& parameter, + GeneratorContext* context, + string* error) const { + FileDescriptorProto new_file; + file->CopyTo(&new_file); + SchemaGroupStripper::StripFile(file, &new_file); + + EnumScrubber enum_scrubber; + enum_scrubber.ScrubFile(&new_file); + ExtensionStripper::StripFile(&new_file); + FieldScrubber::ScrubFile(&new_file); + new_file.set_syntax("proto3"); + + string filename = file->name(); + string basename = StripProto(filename); + + std::vector<std::pair<string,string>> option_pairs; + ParseGeneratorParameter(parameter, &option_pairs); + + std::unique_ptr<google::protobuf::io::ZeroCopyOutputStream> output( + context->Open(basename + ".proto")); + string content = GetPool()->BuildFile(new_file)->DebugString(); + Printer printer(output.get(), '$'); + printer.WriteRaw(content.c_str(), content.size()); + + return true; + } + private: + bool CanGenerate(const FileDescriptor* file) const { + if (GetPool()->FindFileByName(file->name()) != nullptr) { + return false; + } + for (int j = 0; j < file->dependency_count(); j++) { + if (GetPool()->FindFileByName(file->dependency(j)->name()) == nullptr) { + return false; + } + } + for (int j = 0; j < file->public_dependency_count(); j++) { + if (GetPool()->FindFileByName( + file->public_dependency(j)->name()) == nullptr) { + return false; + } + } + for (int j = 0; j < file->weak_dependency_count(); j++) { + if (GetPool()->FindFileByName( + file->weak_dependency(j)->name()) == nullptr) { + return false; + } + } + return true; + } +}; + +} // namespace compiler +} // namespace protobuf +} // namespace google + +int main(int argc, char* argv[]) { + google::protobuf::compiler::Proto2ToProto3Generator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/benchmarks/util/run_and_upload.py b/benchmarks/util/run_and_upload.py new file mode 100755 index 00000000..43c9fa2d --- /dev/null +++ b/benchmarks/util/run_and_upload.py @@ -0,0 +1,292 @@ +from __future__ import print_function +from __future__ import absolute_import +import argparse +import os +import re +import copy +import uuid +import calendar +import time +from . import big_query_utils +import datetime +import json +# This import depends on the automake rule protoc_middleman, please make sure +# protoc_middleman has been built before run this file. +import os.path, sys +sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) +import tmp.benchmarks_pb2 as benchmarks_pb2 +from click.types import STRING + +_PROJECT_ID = 'grpc-testing' +_DATASET = 'protobuf_benchmark_result' +_TABLE = 'opensource_result_v1' +_NOW = "%d%02d%02d" % (datetime.datetime.now().year, + datetime.datetime.now().month, + datetime.datetime.now().day) + +file_size_map = {} + +def get_data_size(file_name): + if file_name in file_size_map: + return file_size_map[file_name] + benchmark_dataset = benchmarks_pb2.BenchmarkDataset() + benchmark_dataset.ParseFromString( + open(os.path.dirname(os.path.abspath(__file__)) + "/../" + file_name).read()) + size = 0 + count = 0 + for payload in benchmark_dataset.payload: + size += len(payload) + count += 1 + file_size_map[file_name] = (size, 1.0 * size / count) + return size, 1.0 * size / count + + +def extract_file_name(file_name): + name_list = re.split("[/\.]", file_name) + short_file_name = "" + for name in name_list: + if name[:14] == "google_message": + short_file_name = name + return short_file_name + + +cpp_result = [] +python_result = [] +java_result = [] +go_result = [] + + +# CPP results example: +# [ +# "benchmarks": [ +# { +# "bytes_per_second": int, +# "cpu_time": int, +# "name: string, +# "time_unit: string, +# ... +# }, +# ... +# ], +# ... +# ] +def parse_cpp_result(filename): + global cpp_result + if filename == "": + return + if filename[0] != '/': + filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename + with open(filename) as f: + results = json.loads(f.read()) + for benchmark in results["benchmarks"]: + data_filename = "".join( + re.split("(_parse_|_serialize)", benchmark["name"])[0]) + behavior = benchmark["name"][len(data_filename) + 1:] + cpp_result.append({ + "language": "cpp", + "dataFileName": data_filename, + "behavior": behavior, + "throughput": benchmark["bytes_per_second"] / 2.0 ** 20 + }) + + +# Python results example: +# [ +# [ +# { +# "filename": string, +# "benchmarks": { +# behavior: results, +# ... +# }, +# "message_name": STRING +# }, +# ... +# ], #pure-python +# ... +# ] +def parse_python_result(filename): + global python_result + if filename == "": + return + if filename[0] != '/': + filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename + with open(filename) as f: + results_list = json.loads(f.read()) + for results in results_list: + for result in results: + _, avg_size = get_data_size(result["filename"]) + for behavior in result["benchmarks"]: + python_result.append({ + "language": "python", + "dataFileName": extract_file_name(result["filename"]), + "behavior": behavior, + "throughput": avg_size / + result["benchmarks"][behavior] * 1e9 / 2 ** 20 + }) + + +# Java results example: +# [ +# { +# "id": string, +# "instrumentSpec": {...}, +# "measurements": [ +# { +# "weight": float, +# "value": { +# "magnitude": float, +# "unit": string +# }, +# ... +# }, +# ... +# ], +# "run": {...}, +# "scenario": { +# "benchmarkSpec": { +# "methodName": string, +# "parameters": { +# defined parameters in the benchmark: parameters value +# }, +# ... +# }, +# ... +# } +# +# }, +# ... +# ] +def parse_java_result(filename): + global average_bytes_per_message, java_result + if filename == "": + return + if filename[0] != '/': + filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename + with open(filename) as f: + results = json.loads(f.read()) + for result in results: + total_weight = 0 + total_value = 0 + for measurement in result["measurements"]: + total_weight += measurement["weight"] + total_value += measurement["value"]["magnitude"] + avg_time = total_value * 1.0 / total_weight + total_size, _ = get_data_size( + result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"]) + java_result.append({ + "language": "java", + "throughput": total_size / avg_time * 1e9 / 2 ** 20, + "behavior": result["scenario"]["benchmarkSpec"]["methodName"], + "dataFileName": extract_file_name( + result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"]) + }) + + +# Go benchmark results: +# +# goos: linux +# goarch: amd64 +# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Unmarshal-12 3000 705784 ns/op +# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Marshal-12 2000 634648 ns/op +# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Size-12 5000 244174 ns/op +# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Clone-12 300 4120954 ns/op +# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Merge-12 300 4108632 ns/op +# PASS +# ok _/usr/local/google/home/yilunchong/mygit/protobuf/benchmarks 124.173s +def parse_go_result(filename): + global go_result + if filename == "": + return + if filename[0] != '/': + filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename + with open(filename) as f: + for line in f: + result_list = re.split("[\ \t]+", line) + if result_list[0][:9] != "Benchmark": + continue + first_slash_index = result_list[0].find('/') + last_slash_index = result_list[0].rfind('/') + full_filename = result_list[0][first_slash_index+4:last_slash_index] # delete ../ prefix + total_bytes, _ = get_data_size(full_filename) + behavior_with_suffix = result_list[0][last_slash_index+1:] + last_dash = behavior_with_suffix.rfind("-") + if last_dash == -1: + behavior = behavior_with_suffix + else: + behavior = behavior_with_suffix[:last_dash] + go_result.append({ + "dataFilename": extract_file_name(full_filename), + "throughput": total_bytes / float(result_list[2]) * 1e9 / 2 ** 20, + "behavior": behavior, + "language": "go" + }) + + +def get_metadata(): + build_number = os.getenv('BUILD_NUMBER') + build_url = os.getenv('BUILD_URL') + job_name = os.getenv('JOB_NAME') + git_commit = os.getenv('GIT_COMMIT') + # actual commit is the actual head of PR that is getting tested + git_actual_commit = os.getenv('ghprbActualCommit') + + utc_timestamp = str(calendar.timegm(time.gmtime())) + metadata = {'created': utc_timestamp} + + if build_number: + metadata['buildNumber'] = build_number + if build_url: + metadata['buildUrl'] = build_url + if job_name: + metadata['jobName'] = job_name + if git_commit: + metadata['gitCommit'] = git_commit + if git_actual_commit: + metadata['gitActualCommit'] = git_actual_commit + + return metadata + + +def upload_result(result_list, metadata): + for result in result_list: + new_result = copy.deepcopy(result) + new_result['metadata'] = metadata + bq = big_query_utils.create_big_query() + row = big_query_utils.make_row(str(uuid.uuid4()), new_result) + if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET, + _TABLE + "$" + _NOW, + [row]): + print('Error when uploading result', new_result) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-cpp", "--cpp_input_file", + help="The CPP benchmark result file's name", + default="") + parser.add_argument("-java", "--java_input_file", + help="The Java benchmark result file's name", + default="") + parser.add_argument("-python", "--python_input_file", + help="The Python benchmark result file's name", + default="") + parser.add_argument("-go", "--go_input_file", + help="The golang benchmark result file's name", + default="") + args = parser.parse_args() + + parse_cpp_result(args.cpp_input_file) + parse_python_result(args.python_input_file) + parse_java_result(args.java_input_file) + parse_go_result(args.go_input_file) + + metadata = get_metadata() + print("uploading cpp results...") + upload_result(cpp_result, metadata) + print("uploading java results...") + upload_result(java_result, metadata) + print("uploading python results...") + upload_result(python_result, metadata) + print("uploading go results...") + upload_result(go_result, metadata) diff --git a/benchmarks/util/schema_proto2_to_proto3_util.h b/benchmarks/util/schema_proto2_to_proto3_util.h new file mode 100644 index 00000000..0079f6f1 --- /dev/null +++ b/benchmarks/util/schema_proto2_to_proto3_util.h @@ -0,0 +1,194 @@ +#ifndef PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ +#define PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" + +#include <sstream> +#include <algorithm> + +using google::protobuf::Descriptor; +using google::protobuf::DescriptorProto; +using google::protobuf::FileDescriptorProto; +using google::protobuf::FieldDescriptorProto; +using google::protobuf::Message; +using google::protobuf::EnumValueDescriptorProto; + +namespace google { +namespace protobuf { +namespace util { + +class SchemaGroupStripper { + + public: + static void StripFile(const FileDescriptor* old_file, + FileDescriptorProto *file) { + for (int i = file->mutable_message_type()->size() - 1; i >= 0; i--) { + if (IsMessageSet(old_file->message_type(i))) { + file->mutable_message_type()->DeleteSubrange(i, 1); + continue; + } + StripMessage(old_file->message_type(i), file->mutable_message_type(i)); + } + for (int i = file->mutable_extension()->size() - 1; i >= 0; i--) { + auto field = old_file->extension(i); + if (field->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(field->message_type()) || + IsMessageSet(field->containing_type())) { + file->mutable_extension()->DeleteSubrange(i, 1); + } + } + } + + private: + static bool IsMessageSet(const Descriptor *descriptor) { + if (descriptor != nullptr + && descriptor->options().message_set_wire_format()) { + return true; + } + return false; + } + + static void StripMessage(const Descriptor *old_message, + DescriptorProto *new_message) { + for (int i = new_message->mutable_field()->size() - 1; i >= 0; i--) { + if (old_message->field(i)->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(old_message->field(i)->message_type())) { + new_message->mutable_field()->DeleteSubrange(i, 1); + } + } + for (int i = new_message->mutable_extension()->size() - 1; i >= 0; i--) { + auto field_type_name = new_message->mutable_extension(i)->type_name(); + if (old_message->extension(i)->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(old_message->extension(i)->containing_type()) || + IsMessageSet(old_message->extension(i)->message_type())) { + new_message->mutable_extension()->DeleteSubrange(i, 1); + } + } + for (int i = 0; i < new_message->mutable_nested_type()->size(); i++) { + StripMessage(old_message->nested_type(i), + new_message->mutable_nested_type(i)); + } + } + +}; + +class EnumScrubber { + + public: + EnumScrubber() + : total_added_(0) { + } + + void ScrubFile(FileDescriptorProto *file) { + for (int i = 0; i < file->enum_type_size(); i++) { + ScrubEnum(file->mutable_enum_type(i)); + } + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + ScrubMessage(file->mutable_message_type(i)); + } + } + + private: + void ScrubEnum(EnumDescriptorProto *enum_type) { + if (enum_type->value(0).number() != 0) { + bool has_zero = false; + for (int j = 0; j < enum_type->value().size(); j++) { + if (enum_type->value(j).number() == 0) { + EnumValueDescriptorProto temp_enum_value; + temp_enum_value.CopyFrom(enum_type->value(j)); + enum_type->mutable_value(j)->CopyFrom(enum_type->value(0)); + enum_type->mutable_value(0)->CopyFrom(temp_enum_value); + has_zero = true; + break; + } + } + if (!has_zero) { + enum_type->mutable_value()->Add(); + for (int i = enum_type->mutable_value()->size() - 1; i > 0; i--) { + enum_type->mutable_value(i)->CopyFrom( + *enum_type->mutable_value(i - 1)); + } + enum_type->mutable_value(0)->set_number(0); + enum_type->mutable_value(0)->set_name("ADDED_ZERO_VALUE_" + + std::to_string(total_added_++)); + } + } + + } + + void ScrubMessage(DescriptorProto *message_type) { + for (int i = 0; i < message_type->mutable_enum_type()->size(); i++) { + ScrubEnum(message_type->mutable_enum_type(i)); + } + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + ScrubMessage(message_type->mutable_nested_type(i)); + } + } + + int total_added_; +}; + +class ExtensionStripper { + public: + static void StripFile(FileDescriptorProto *file) { + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + StripMessage(file->mutable_message_type(i)); + } + file->mutable_extension()->Clear(); + } + private: + static void StripMessage(DescriptorProto *message_type) { + message_type->mutable_extension()->Clear(); + message_type->clear_extension_range(); + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + StripMessage(message_type->mutable_nested_type(i)); + } + } +}; + + +class FieldScrubber { + public: + static void ScrubFile(FileDescriptorProto *file) { + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + ScrubMessage(file->mutable_message_type(i)); + } + for (int i = 0; i < file->mutable_extension()->size(); i++) { + file->mutable_extension(i)->clear_default_value(); + if (ShouldClearLabel(file->mutable_extension(i))) { + file->mutable_extension(i)->clear_label(); + } + } + } + private: + static bool ShouldClearLabel(const FieldDescriptorProto *field) { + return field->label() == FieldDescriptorProto::LABEL_REQUIRED; + } + + static void ScrubMessage(DescriptorProto *message_type) { + message_type->mutable_extension()->Clear(); + for (int i = 0; i < message_type->mutable_extension()->size(); i++) { + message_type->mutable_extension(i)->clear_default_value(); + if (ShouldClearLabel(message_type->mutable_extension(i))) { + message_type->mutable_extension(i)->clear_label(); + } + } + for (int i = 0; i < message_type->mutable_field()->size(); i++) { + message_type->mutable_field(i)->clear_default_value(); + if (ShouldClearLabel(message_type->mutable_field(i))) { + message_type->mutable_field(i)->clear_label(); + } + } + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + ScrubMessage(message_type->mutable_nested_type(i)); + } + } +}; + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ |