From c703061d4993904d1fb42e9ae88034112eceb261 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Tue, 10 Apr 2018 14:32:28 -0700 Subject: Add gogo benchmark --- benchmarks/Makefile.am | 378 ++++++++++++++++++------- benchmarks/cpp/cpp_benchmark.cc | 254 +++++++++++++++++ benchmarks/cpp_benchmark.cc | 254 ----------------- benchmarks/go/go_benchmark_test.go | 124 ++++++++ benchmarks/go_benchmark_test.go | 124 -------- benchmarks/py_benchmark.py | 118 -------- benchmarks/python/__init__.py | 0 benchmarks/python/py_benchmark.py | 150 ++++++++++ benchmarks/python/python_benchmark_messages.cc | 29 ++ benchmarks/python_benchmark_messages.cc | 29 -- benchmarks/util/gogo_data_scrubber.cc | 105 +++++++ benchmarks/util/protoc-gen-gogoproto.cc | 103 +++++++ benchmarks/util/schema_proto2_to_proto3_util.h | 137 +++++++++ 13 files changed, 1184 insertions(+), 621 deletions(-) create mode 100644 benchmarks/cpp/cpp_benchmark.cc delete mode 100644 benchmarks/cpp_benchmark.cc create mode 100644 benchmarks/go/go_benchmark_test.go delete mode 100644 benchmarks/go_benchmark_test.go delete mode 100755 benchmarks/py_benchmark.py create mode 100644 benchmarks/python/__init__.py create mode 100755 benchmarks/python/py_benchmark.py create mode 100644 benchmarks/python/python_benchmark_messages.cc delete mode 100644 benchmarks/python_benchmark_messages.cc create mode 100644 benchmarks/util/gogo_data_scrubber.cc create mode 100644 benchmarks/util/protoc-gen-gogoproto.cc create mode 100644 benchmarks/util/schema_proto2_to_proto3_util.h (limited to 'benchmarks') diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 3de0e380..5b257488 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -25,76 +25,62 @@ make_tmp_dir: mkdir -p 'tmp/java/src/main/java' touch make_tmp_dir -if USE_EXTERNAL_PROTOC - -protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) - touch protoc_middleman - -protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=./tmp $(benchmarks_protoc_inputs_proto2) - touch protoc_middleman2 - -else - # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) ) touch protoc_middleman protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd/cpp --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) touch protoc_middleman2 -endif - -all_data = `find . -type f -name "dataset.*.pb"` +all_data = $$(find $(srcdir) -type f -name "dataset.*.pb" -not -path "./tmp/*") ############# CPP RULES ############## benchmarks_protoc_outputs = \ - benchmarks.pb.cc \ - datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc - + cpp/benchmarks.pb.cc \ + cpp/datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc + benchmarks_protoc_outputs_header = \ - benchmarks.pb.h \ - datasets/google_message1/proto3/benchmark_message1_proto3.pb.h + cpp/benchmarks.pb.h \ + cpp/datasets/google_message1/proto3/benchmark_message1_proto3.pb.h benchmarks_protoc_outputs_proto2_header = \ - datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ - datasets/google_message2/benchmark_message2.pb.h \ - datasets/google_message3/benchmark_message3.pb.h \ - datasets/google_message3/benchmark_message3_1.pb.h \ - datasets/google_message3/benchmark_message3_2.pb.h \ - datasets/google_message3/benchmark_message3_3.pb.h \ - datasets/google_message3/benchmark_message3_4.pb.h \ - datasets/google_message3/benchmark_message3_5.pb.h \ - datasets/google_message3/benchmark_message3_6.pb.h \ - datasets/google_message3/benchmark_message3_7.pb.h \ - datasets/google_message3/benchmark_message3_8.pb.h \ - datasets/google_message4/benchmark_message4.pb.h \ - datasets/google_message4/benchmark_message4_1.pb.h \ - datasets/google_message4/benchmark_message4_2.pb.h \ - datasets/google_message4/benchmark_message4_3.pb.h + cpp/datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ + cpp/datasets/google_message2/benchmark_message2.pb.h \ + cpp/datasets/google_message3/benchmark_message3.pb.h \ + cpp/datasets/google_message3/benchmark_message3_1.pb.h \ + cpp/datasets/google_message3/benchmark_message3_2.pb.h \ + cpp/datasets/google_message3/benchmark_message3_3.pb.h \ + cpp/datasets/google_message3/benchmark_message3_4.pb.h \ + cpp/datasets/google_message3/benchmark_message3_5.pb.h \ + cpp/datasets/google_message3/benchmark_message3_6.pb.h \ + cpp/datasets/google_message3/benchmark_message3_7.pb.h \ + cpp/datasets/google_message3/benchmark_message3_8.pb.h \ + cpp/datasets/google_message4/benchmark_message4.pb.h \ + cpp/datasets/google_message4/benchmark_message4_1.pb.h \ + cpp/datasets/google_message4/benchmark_message4_2.pb.h \ + cpp/datasets/google_message4/benchmark_message4_3.pb.h benchmarks_protoc_outputs_proto2 = \ - datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ - datasets/google_message2/benchmark_message2.pb.cc \ - datasets/google_message3/benchmark_message3.pb.cc \ - datasets/google_message3/benchmark_message3_1.pb.cc \ - datasets/google_message3/benchmark_message3_2.pb.cc \ - datasets/google_message3/benchmark_message3_3.pb.cc \ - datasets/google_message3/benchmark_message3_4.pb.cc \ - datasets/google_message3/benchmark_message3_5.pb.cc \ - datasets/google_message3/benchmark_message3_6.pb.cc \ - datasets/google_message3/benchmark_message3_7.pb.cc \ - datasets/google_message3/benchmark_message3_8.pb.cc \ - datasets/google_message4/benchmark_message4.pb.cc \ - datasets/google_message4/benchmark_message4_1.pb.cc \ - datasets/google_message4/benchmark_message4_2.pb.cc \ - datasets/google_message4/benchmark_message4_3.pb.cc + cpp/datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ + cpp/datasets/google_message2/benchmark_message2.pb.cc \ + cpp/datasets/google_message3/benchmark_message3.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_1.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_2.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_3.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_4.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_5.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_6.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_7.pb.cc \ + cpp/datasets/google_message3/benchmark_message3_8.pb.cc \ + cpp/datasets/google_message4/benchmark_message4.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_1.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_2.pb.cc \ + cpp/datasets/google_message4/benchmark_message4_3.pb.cc $(benchmarks_protoc_outputs): protoc_middleman @@ -109,8 +95,7 @@ initialize_submodule: cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make cd $$oldpwd touch initialize_submodule - -$(top_srcdir)/src/libprotobuf.la: initialize_submodule + $(top_srcdir)/third_party/benchmark/src/libbenchmark.a: initialize_submodule AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare @@ -118,12 +103,12 @@ AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Wov bin_PROGRAMS = cpp-benchmark cpp_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a -cpp_benchmark_SOURCES = cpp_benchmark.cc -cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_party/benchmark/include +cpp_benchmark_SOURCES = cpp/cpp_benchmark.cc +cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(top_srcdir)/third_party/benchmark/include # Explicit deps because BUILT_SOURCES are only done before a "make all/check" # so a direct "make test_cpp" could fail if parallel enough. # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually -cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp/cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a nodist_cpp_benchmark_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ @@ -142,7 +127,7 @@ java_benchmark_testing_files = \ java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2 - cp -r java tmp && cd tmp/java && mvn clean compile assembly:single + cp -r $(srcdir)/java tmp && cd tmp/java && mvn clean compile assembly:single cd ../.. @touch javac_middleman @@ -174,11 +159,11 @@ python_add_init: protoc_middleman protoc_middleman2 python_cpp_pkg_flags = `pkg-config --cflags --libs python` lib_LTLIBRARIES = libbenchmark_messages.la -libbenchmark_messages_la_SOURCES = python_benchmark_messages.cc +libbenchmark_messages_la_SOURCES = python/python_benchmark_messages.cc libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la libbenchmark_messages_la_LDFLAGS = -version-info 1:0:0 -export-dynamic -libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) $(python_cpp_pkg_flags) -libbenchmark_messages_la-libbenchmark_messages_la.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) +libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp $(python_cpp_pkg_flags) +libbenchmark_messages_la-python_benchmark_messages.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) nodist_libbenchmark_messages_la_SOURCES = \ $(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs_proto2) \ @@ -191,7 +176,7 @@ python-pure-python-benchmark: python_add_init @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-pure-python-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'python\' >> python-pure-python-benchmark - @echo cp py_benchmark.py tmp >> python-pure-python-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-pure-python-benchmark @echo python tmp/py_benchmark.py false '$$@' >> python-pure-python-benchmark @chmod +x python-pure-python-benchmark @@ -201,7 +186,7 @@ python-cpp-reflection-benchmark: python_add_init @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-reflection-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-reflection-benchmark - @echo cp py_benchmark.py tmp >> python-cpp-reflection-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-reflection-benchmark @echo python tmp/py_benchmark.py false '$$@' >> python-cpp-reflection-benchmark @chmod +x python-cpp-reflection-benchmark @@ -211,7 +196,7 @@ python-cpp-generated-code-benchmark: python_add_init libbenchmark_messages.la @echo export LD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark @echo export DYLD_LIBRARY_PATH=$(top_srcdir)/src/libprotobuf.la >> python-cpp-generated-code-benchmark @echo export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=\'cpp\' >> python-cpp-generated-code-benchmark - @echo cp py_benchmark.py tmp >> python-cpp-generated-code-benchmark + @echo cp $(srcdir)/python/py_benchmark.py tmp >> python-cpp-generated-code-benchmark @echo python tmp/py_benchmark.py true '$$@' >> python-cpp-generated-code-benchmark @chmod +x python-cpp-generated-code-benchmark @@ -251,52 +236,230 @@ benchmarks_protoc_inputs_proto2_message4 = \ datasets/google_message4/benchmark_message4_2.proto \ datasets/google_message4/benchmark_message4_3.proto -if USE_EXTERNAL_PROTOC - -go_protoc_middleman: make_tmp_dir $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) - touch protoc_middleman - -go_protoc_middleman2: make_tmp_dir $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) - $(PROTOC) -I$(srcdir) -I$(top_srcdir) --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) - touch protoc_middleman2 - -else - -# We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is -# relative to srcdir, which may not be the same as the current directory when -# building out-of-tree. -go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) +go_protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_benchmark_wrapper) ) - touch protoc_middleman - -go_protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2_message1) $(benchmarks_protoc_inputs_proto2_message2) $(benchmarks_protoc_inputs_proto2_message3) $(benchmarks_protoc_inputs_proto2_message4) $(well_known_type_protoc_inputs) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message1) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message2) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message3) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --go_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2_message4) ) - touch protoc_middleman2 + touch go_protoc_middleman -endif - -go-benchmark: go_protoc_middleman go_protoc_middleman2 +go-benchmark: go_protoc_middleman @echo "Writing shortcut script go-benchmark..." @echo '#! /bin/sh' > go-benchmark - @echo 'mkdir tmp_cc && mv *.cc tmp_cc' >> go-benchmark - @echo 'go test -bench=. -- $$@' >> go-benchmark - @echo 'mv tmp_cc/* . && rm -rf tmp_cc' >> go-benchmark + @echo 'cd $(srcdir)/go' >> go-benchmark + @echo 'all_data=""' >> go-benchmark + @echo 'for data_file in $$@; do all_data="$$all_data ../$$data_file"; done' >> go-benchmark + @echo 'go test -bench=. -- $$all_data' >> go-benchmark + @echo 'cd ..' >> go-benchmark @chmod +x go-benchmark -go: go_protoc_middleman go_protoc_middleman2 go-benchmark +go: go_protoc_middleman go-benchmark ./go-benchmark $(all_data) ############# GO RULES END ############## +############# GOGO RULES BEGIN ############ + +cpp_no_group_benchmarks_protoc_outputs_header = \ + gogo/cpp_no_group/benchmarks.pb.h \ + gogo/cpp_no_group/datasets/google_message1/proto3/benchmark_message1_proto3.pb.h + +cpp_no_group_benchmarks_protoc_outputs = \ + gogo/cpp_no_group/benchmarks.pb.cc \ + gogo/cpp_no_group/datasets/google_message1/proto3/benchmark_message1_proto3.pb.cc + +cpp_no_group_benchmarks_protoc_outputs_proto2_header = \ + gogo/cpp_no_group/datasets/google_message1/proto2/benchmark_message1_proto2.pb.h \ + gogo/cpp_no_group/datasets/google_message2/benchmark_message2.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_1.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_2.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_3.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_4.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_5.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_6.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_7.pb.h \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_8.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_1.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_2.pb.h \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_3.pb.h + +cpp_no_group_benchmarks_protoc_outputs_proto2 = \ + gogo/cpp_no_group/datasets/google_message1/proto2/benchmark_message1_proto2.pb.cc \ + gogo/cpp_no_group/datasets/google_message2/benchmark_message2.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_1.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_2.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_3.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_4.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_5.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_6.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_7.pb.cc \ + gogo/cpp_no_group/datasets/google_message3/benchmark_message3_8.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_1.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_2.pb.cc \ + gogo/cpp_no_group/datasets/google_message4/benchmark_message4_3.pb.cc + +$(cpp_no_group_benchmarks_protoc_outputs): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_header): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_proto2): cpp_no_group_protoc_middleman +$(cpp_no_group_benchmarks_protoc_outputs_proto2_header): cpp_no_group_protoc_middleman + +generate_cpp_no_group_benchmark_code: + cp $(srcdir)/cpp/cpp_benchmark.cc gogo/cpp_no_group/cpp_benchmark.cc + sed -i -e "s/\#include \"datasets/\#include \"gogo\/cpp_no_group\/datasets/g" gogo/cpp_no_group/cpp_benchmark.cc + sed -i -e "s/\#include \"benchmarks.pb.h/\#include \"gogo\/cpp_no_group\/benchmarks.pb.h/g" gogo/cpp_no_group/cpp_benchmark.cc + touch generate_cpp_no_group_benchmark_code + +bin_PROGRAMS += cpp-no-group-benchmark +cpp_no_group_benchmark_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a +cpp_no_group_benchmark_SOURCES = gogo/cpp_no_group/cpp_benchmark.cc +cpp_no_group_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/gogo/cpp_no_group -I$(top_srcdir)/third_party/benchmark/include +# Explicit deps because BUILT_SOURCES are only done before a "make all/check" +# so a direct "make test_cpp" could fail if parallel enough. +# See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +gogo/cpp_no_group/cpp_no_group_benchmark-cpp_benchmark.$(OBJEXT): $(cpp_no_group_benchmarks_protoc_outputs) $(cpp_no_group_benchmarks_protoc_outputs_proto2) $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/third_party/benchmark/src/libbenchmark.a generate_cpp_no_group_benchmark_code +gogo/cpp_no_group/cpp_benchmark.cc: generate_cpp_no_group_benchmark_code +nodist_cpp_no_group_benchmark_SOURCES = \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2) \ + $(cpp_no_group_benchmarks_protoc_outputs) \ + $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) + +cpp_no_group: cpp_no_group_protoc_middleman generate_gogo_data cpp-no-group-benchmark + ./cpp-no-group-benchmark $(gogo_data) + +gogo_proto_middleman: protoc-gen-gogoproto + mkdir -p "tmp/gogo_proto" + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I$(srcdir) -I$(top_srcdir) --plugin=protoc-gen-gogoproto --gogoproto_out=$$oldpwd/tmp/gogo_proto $(benchmarks_protoc_inputs) $(benchmarks_protoc_inputs_benchmark_wrapper) $(benchmarks_protoc_inputs_proto2) ) + touch gogo_proto_middleman + +new_data = $$(for data in $(all_data); do echo "tmp$${data\#$(srcdir)}"; done | xargs) + +generate_gogo_data: protoc_middleman protoc_middleman2 gogo-data-scrubber + mkdir -p `dirname $(new_data)` + ./gogo-data-scrubber $(all_data) $(new_data) + touch generate_gogo_data + +make_tmp_dir_gogo: + mkdir -p tmp/go_no_group/benchmark_code + mkdir -p tmp/gogofast/benchmark_code + mkdir -p tmp/gogofaster/benchmark_code + mkdir -p tmp/gogoslick/benchmark_code + touch make_tmp_dir_gogo + +go_no_group_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --go_out=$$oldpwd/tmp/go_no_group $(benchmarks_protoc_inputs_proto2_message4) ) + touch go_no_group_protoc_middleman + +cpp_no_group_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --cpp_out=$$oldpwd/gogo/cpp_no_group $(benchmarks_protoc_inputs_proto2_message4) ) + touch cpp_no_group_protoc_middleman + +gogofast_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofast_out=$$oldpwd/tmp/gogofast $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogofast_protoc_middleman + +gogofaster_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogofaster_out=$$oldpwd/tmp/gogofaster $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogofaster_protoc_middleman + +gogoslick_protoc_middleman: make_tmp_dir_gogo $(top_srcdir)/src/protoc$(EXEEXT) gogo_proto_middleman $(well_known_type_protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_benchmark_wrapper) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message1) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message2) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message3) ) + oldpwd=`pwd` && ( cd $(srcdir)/tmp/gogo_proto && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$$oldpwd/$(top_srcdir)/src --gogoslick_out=$$oldpwd/tmp/gogoslick $(benchmarks_protoc_inputs_proto2_message4) ) + touch gogoslick_protoc_middleman + +gogo_data = $$(find . -type f -name "dataset.*.pb" -path "./tmp/*") + +generate-gogo-benchmark-code: + @echo '#! /bin/sh' > generate-gogo-benchmark-code + @echo 'cp $(srcdir)/go/go_benchmark_test.go tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'sed -i -e "s/\.\.\/tmp/../g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'sed -i -e "s/b\.Run(\"\(.*\)\"/b.Run(\"\1\_$$1\"/g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go' >> generate-gogo-benchmark-code + @echo 'if [[ $$2 == 1 ]]; then sed -i -e "s/github\.com\/golang/github.com\/gogo/g" tmp/$$1/benchmark_code/$$1_benchmark1_test.go; fi ' >> generate-gogo-benchmark-code + @chmod +x generate-gogo-benchmark-code + +generate_all_gogo_benchmark_code: generate-gogo-benchmark-code make_tmp_dir_gogo + ./generate-gogo-benchmark-code go_no_group 0 + ./generate-gogo-benchmark-code gogofast 1 + ./generate-gogo-benchmark-code gogofaster 1 + ./generate-gogo-benchmark-code gogoslick 1 + +gogo-benchmark: + @echo "Writing shortcut script gogo-benchmark..." + @echo '#! /bin/sh' > gogo-benchmark + @echo 'cd tmp/$$1/benchmark_code' >> gogo-benchmark + @echo 'shift' >> gogo-benchmark + @echo 'all_data=""' >> gogo-benchmark + @echo 'for data_file in $$@; do all_data="$$all_data ../../../$$data_file"; done' >> gogo-benchmark + @echo 'go test -bench=. -- $$all_data' >> gogo-benchmark + @echo 'cd ../..' >> gogo-benchmark + @chmod +x gogo-benchmark + +go_no_group: go_no_group_protoc_middleman generate_gogo_data generate_all_gogo_benchmark_code gogo-benchmark + ./gogo-benchmark go_no_group $(gogo_data) + +gogofast: gogofast_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogofast $(gogo_data) + +gogofaster: gogofaster_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogofaster $(gogo_data) + +gogoslick: gogoslick_protoc_middleman generate_gogo_data gogo-benchmark generate_all_gogo_benchmark_code + ./gogo-benchmark gogoslick $(gogo_data) + + +############# GOGO RULES END ############ + + +############ UTIL RULES BEGIN ############ + +bin_PROGRAMS += protoc-gen-gogoproto gogo-data-scrubber + +protoc_gen_gogoproto_LDADD = $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/src/libprotoc.la +protoc_gen_gogoproto_SOURCES = util/protoc-gen-gogoproto.cc +protoc_gen_gogoproto_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util + +gogo_data_scrubber_LDADD = $(top_srcdir)/src/libprotobuf.la +gogo_data_scrubber_SOURCES = util/gogo_data_scrubber.cc +gogo_data_scrubber_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir)/cpp -I$(srcdir)/util +util/gogo_data_scrubber-gogo_data_scrubber.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) +nodist_gogo_data_scrubber_SOURCES = \ + $(benchmarks_protoc_outputs) \ + $(benchmarks_protoc_outputs_proto2) \ + $(benchmarks_protoc_outputs_proto2_header) \ + $(benchmarks_protoc_outputs_header) + +############ UTIL RULES END ############ MAINTAINERCLEANFILES = \ Makefile.in @@ -318,7 +481,30 @@ CLEANFILES = \ python-cpp-generated-code-benchmark \ go-benchmark \ go_protoc_middleman \ - go_protoc_middleman2 + make_tmp_dir_gogo \ + gogo_proto_middleman \ + generate_gogo_data \ + go_no_group_protoc_middleman \ + go_no_group \ + go-no-group-benchmark \ + $(cpp_no_group_benchmarks_protoc_outputs_header) \ + $(cpp_no_group_benchmarks_protoc_outputs) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2_header) \ + $(cpp_no_group_benchmarks_protoc_outputs_proto2) \ + generate_all_gogo_benchmark_code \ + generate-gogo-benchmark-code \ + cpp_no_group_protoc_middleman \ + generate_cpp_no_group_benchmark_code \ + generate_gogo_benchmark_code \ + gogofast_protoc_middleman \ + gogofast \ + gogofaster_protoc_middleman \ + gogofaster \ + gogoslick_protoc_middleman \ + gogoslick \ + gogo-benchmark \ + gogo/cpp_no_group/cpp_benchmark.* + clean-local: -rm -rf tmp/* diff --git a/benchmarks/cpp/cpp_benchmark.cc b/benchmarks/cpp/cpp_benchmark.cc new file mode 100644 index 00000000..f8b55291 --- /dev/null +++ b/benchmarks/cpp/cpp_benchmark.cc @@ -0,0 +1,254 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include "benchmark/benchmark_api.h" +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" + + +#define PREFIX "dataset." +#define SUFFIX ".pb" + +using benchmarks::BenchmarkDataset; +using google::protobuf::Arena; +using google::protobuf::Descriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::Message; +using google::protobuf::MessageFactory; + +class Fixture : public benchmark::Fixture { + public: + Fixture(const BenchmarkDataset& dataset, const std::string& suffix) { + for (int i = 0; i < dataset.payload_size(); i++) { + payloads_.push_back(dataset.payload(i)); + } + + const Descriptor* d = + DescriptorPool::generated_pool()->FindMessageTypeByName( + dataset.message_name()); + + if (!d) { + std::cerr << "Couldn't find message named '" << dataset.message_name() + << "\n"; + } + + prototype_ = MessageFactory::generated_factory()->GetPrototype(d); + SetName((dataset.name() + suffix).c_str()); + } + + protected: + std::vector payloads_; + const Message* prototype_; +}; + +class WrappingCounter { + public: + WrappingCounter(size_t limit) : value_(0), limit_(limit) {} + + size_t Next() { + size_t ret = value_; + if (++value_ == limit_) { + value_ = 0; + } + return ret; + } + + private: + size_t value_; + size_t limit_; +}; + +template +class ParseNewFixture : public Fixture { + public: + ParseNewFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_parse_new") {} + + virtual void BenchmarkCase(benchmark::State& state) { + WrappingCounter i(payloads_.size()); + size_t total = 0; + + while (state.KeepRunning()) { + T m; + const std::string& payload = payloads_[i.Next()]; + total += payload.size(); + m.ParseFromString(payload); + } + + state.SetBytesProcessed(total); + } +}; + +template +class ParseNewArenaFixture : public Fixture { + public: + ParseNewArenaFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_parse_newarena") {} + + virtual void BenchmarkCase(benchmark::State& state) { + WrappingCounter i(payloads_.size()); + size_t total = 0; + Arena arena; + + while (state.KeepRunning()) { + arena.Reset(); + Message* m = Arena::CreateMessage(&arena); + const std::string& payload = payloads_[i.Next()]; + total += payload.size(); + m->ParseFromString(payload); + } + + state.SetBytesProcessed(total); + } +}; + +template +class ParseReuseFixture : public Fixture { + public: + ParseReuseFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_parse_reuse") {} + + virtual void BenchmarkCase(benchmark::State& state) { + T m; + WrappingCounter i(payloads_.size()); + size_t total = 0; + + while (state.KeepRunning()) { + const std::string& payload = payloads_[i.Next()]; + total += payload.size(); + m.ParseFromString(payload); + } + + state.SetBytesProcessed(total); + } +}; + +template +class SerializeFixture : public Fixture { + public: + SerializeFixture(const BenchmarkDataset& dataset) + : Fixture(dataset, "_serialize") { + for (size_t i = 0; i < payloads_.size(); i++) { + message_.push_back(new T); + message_.back()->ParseFromString(payloads_[i]); + } + } + + ~SerializeFixture() { + for (size_t i = 0; i < message_.size(); i++) { + delete message_[i]; + } + } + + virtual void BenchmarkCase(benchmark::State& state) { + size_t total = 0; + std::string str; + WrappingCounter i(payloads_.size()); + + while (state.KeepRunning()) { + str.clear(); + message_[i.Next()]->SerializeToString(&str); + total += str.size(); + } + + state.SetBytesProcessed(total); + } + + private: + std::vector message_; +}; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" << name << + "', please make sure you are running " + "this command from the benchmarks/ " + "directory.\n"; + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); +} + +template +void RegisterBenchmarksForType(const BenchmarkDataset& dataset) { + ::benchmark::internal::RegisterBenchmarkInternal( + new ParseNewFixture(dataset)); + ::benchmark::internal::RegisterBenchmarkInternal( + new ParseReuseFixture(dataset)); + ::benchmark::internal::RegisterBenchmarkInternal( + new ParseNewArenaFixture(dataset)); + ::benchmark::internal::RegisterBenchmarkInternal( + new SerializeFixture(dataset)); +} + +void RegisterBenchmarks(const std::string& dataset_bytes) { + BenchmarkDataset dataset; + GOOGLE_CHECK(dataset.ParseFromString(dataset_bytes)); + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + RegisterBenchmarksForType(dataset); + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + RegisterBenchmarksForType(dataset); + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + RegisterBenchmarksForType(dataset); + } else if (dataset.message_name() == + "benchmarks.google_message3.GoogleMessage3") { + RegisterBenchmarksForType + (dataset); + } else if (dataset.message_name() == + "benchmarks.google_message4.GoogleMessage4") { + RegisterBenchmarksForType + (dataset); + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } +} + +int main(int argc, char *argv[]) { + ::benchmark::Initialize(&argc, argv); + if (argc == 1) { + std::cerr << "Usage: ./cpp-benchmark " << std::endl; + std::cerr << "input data is in the format of \"benchmarks.proto\"" + << std::endl; + return 1; + } else { + for (int i = 1; i < argc; i++) { + RegisterBenchmarks(ReadFile(argv[i])); + } + } + + ::benchmark::RunSpecifiedBenchmarks(); +} diff --git a/benchmarks/cpp_benchmark.cc b/benchmarks/cpp_benchmark.cc deleted file mode 100644 index f8b55291..00000000 --- a/benchmarks/cpp_benchmark.cc +++ /dev/null @@ -1,254 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include -#include -#include "benchmark/benchmark_api.h" -#include "benchmarks.pb.h" -#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" -#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" -#include "datasets/google_message2/benchmark_message2.pb.h" -#include "datasets/google_message3/benchmark_message3.pb.h" -#include "datasets/google_message4/benchmark_message4.pb.h" - - -#define PREFIX "dataset." -#define SUFFIX ".pb" - -using benchmarks::BenchmarkDataset; -using google::protobuf::Arena; -using google::protobuf::Descriptor; -using google::protobuf::DescriptorPool; -using google::protobuf::Message; -using google::protobuf::MessageFactory; - -class Fixture : public benchmark::Fixture { - public: - Fixture(const BenchmarkDataset& dataset, const std::string& suffix) { - for (int i = 0; i < dataset.payload_size(); i++) { - payloads_.push_back(dataset.payload(i)); - } - - const Descriptor* d = - DescriptorPool::generated_pool()->FindMessageTypeByName( - dataset.message_name()); - - if (!d) { - std::cerr << "Couldn't find message named '" << dataset.message_name() - << "\n"; - } - - prototype_ = MessageFactory::generated_factory()->GetPrototype(d); - SetName((dataset.name() + suffix).c_str()); - } - - protected: - std::vector payloads_; - const Message* prototype_; -}; - -class WrappingCounter { - public: - WrappingCounter(size_t limit) : value_(0), limit_(limit) {} - - size_t Next() { - size_t ret = value_; - if (++value_ == limit_) { - value_ = 0; - } - return ret; - } - - private: - size_t value_; - size_t limit_; -}; - -template -class ParseNewFixture : public Fixture { - public: - ParseNewFixture(const BenchmarkDataset& dataset) - : Fixture(dataset, "_parse_new") {} - - virtual void BenchmarkCase(benchmark::State& state) { - WrappingCounter i(payloads_.size()); - size_t total = 0; - - while (state.KeepRunning()) { - T m; - const std::string& payload = payloads_[i.Next()]; - total += payload.size(); - m.ParseFromString(payload); - } - - state.SetBytesProcessed(total); - } -}; - -template -class ParseNewArenaFixture : public Fixture { - public: - ParseNewArenaFixture(const BenchmarkDataset& dataset) - : Fixture(dataset, "_parse_newarena") {} - - virtual void BenchmarkCase(benchmark::State& state) { - WrappingCounter i(payloads_.size()); - size_t total = 0; - Arena arena; - - while (state.KeepRunning()) { - arena.Reset(); - Message* m = Arena::CreateMessage(&arena); - const std::string& payload = payloads_[i.Next()]; - total += payload.size(); - m->ParseFromString(payload); - } - - state.SetBytesProcessed(total); - } -}; - -template -class ParseReuseFixture : public Fixture { - public: - ParseReuseFixture(const BenchmarkDataset& dataset) - : Fixture(dataset, "_parse_reuse") {} - - virtual void BenchmarkCase(benchmark::State& state) { - T m; - WrappingCounter i(payloads_.size()); - size_t total = 0; - - while (state.KeepRunning()) { - const std::string& payload = payloads_[i.Next()]; - total += payload.size(); - m.ParseFromString(payload); - } - - state.SetBytesProcessed(total); - } -}; - -template -class SerializeFixture : public Fixture { - public: - SerializeFixture(const BenchmarkDataset& dataset) - : Fixture(dataset, "_serialize") { - for (size_t i = 0; i < payloads_.size(); i++) { - message_.push_back(new T); - message_.back()->ParseFromString(payloads_[i]); - } - } - - ~SerializeFixture() { - for (size_t i = 0; i < message_.size(); i++) { - delete message_[i]; - } - } - - virtual void BenchmarkCase(benchmark::State& state) { - size_t total = 0; - std::string str; - WrappingCounter i(payloads_.size()); - - while (state.KeepRunning()) { - str.clear(); - message_[i.Next()]->SerializeToString(&str); - total += str.size(); - } - - state.SetBytesProcessed(total); - } - - private: - std::vector message_; -}; - -std::string ReadFile(const std::string& name) { - std::ifstream file(name.c_str()); - GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" << name << - "', please make sure you are running " - "this command from the benchmarks/ " - "directory.\n"; - return std::string((std::istreambuf_iterator(file)), - std::istreambuf_iterator()); -} - -template -void RegisterBenchmarksForType(const BenchmarkDataset& dataset) { - ::benchmark::internal::RegisterBenchmarkInternal( - new ParseNewFixture(dataset)); - ::benchmark::internal::RegisterBenchmarkInternal( - new ParseReuseFixture(dataset)); - ::benchmark::internal::RegisterBenchmarkInternal( - new ParseNewArenaFixture(dataset)); - ::benchmark::internal::RegisterBenchmarkInternal( - new SerializeFixture(dataset)); -} - -void RegisterBenchmarks(const std::string& dataset_bytes) { - BenchmarkDataset dataset; - GOOGLE_CHECK(dataset.ParseFromString(dataset_bytes)); - - if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { - RegisterBenchmarksForType(dataset); - } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { - RegisterBenchmarksForType(dataset); - } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { - RegisterBenchmarksForType(dataset); - } else if (dataset.message_name() == - "benchmarks.google_message3.GoogleMessage3") { - RegisterBenchmarksForType - (dataset); - } else if (dataset.message_name() == - "benchmarks.google_message4.GoogleMessage4") { - RegisterBenchmarksForType - (dataset); - } else { - std::cerr << "Unknown message type: " << dataset.message_name(); - exit(1); - } -} - -int main(int argc, char *argv[]) { - ::benchmark::Initialize(&argc, argv); - if (argc == 1) { - std::cerr << "Usage: ./cpp-benchmark " << std::endl; - std::cerr << "input data is in the format of \"benchmarks.proto\"" - << std::endl; - return 1; - } else { - for (int i = 1; i < argc; i++) { - RegisterBenchmarks(ReadFile(argv[i])); - } - } - - ::benchmark::RunSpecifiedBenchmarks(); -} diff --git a/benchmarks/go/go_benchmark_test.go b/benchmarks/go/go_benchmark_test.go new file mode 100644 index 00000000..8c741b71 --- /dev/null +++ b/benchmarks/go/go_benchmark_test.go @@ -0,0 +1,124 @@ +package main + +import ( + benchmarkWrapper "../tmp" + googleMessage1Proto2 "../tmp/datasets/google_message1/proto2" + googleMessage1Proto3 "../tmp/datasets/google_message1/proto3" + googleMessage2 "../tmp/datasets/google_message2" + googleMessage3 "../tmp/datasets/google_message3" + googleMessage4 "../tmp/datasets/google_message4" + "flag" + "github.com/golang/protobuf/proto" + "io/ioutil" + "testing" +) + +// Data is returned by the Load function. +type Dataset struct { + name string + newMessage func() proto.Message + marshaled [][]byte + unmarshaled []proto.Message +} + +var datasets []Dataset + +// This is used to getDefaultInstance for a message type. +func generateNewMessageFunction(dataset benchmarkWrapper.BenchmarkDataset) func() proto.Message { + switch dataset.MessageName { + case "benchmarks.proto3.GoogleMessage1": + return func() proto.Message { return new(googleMessage1Proto3.GoogleMessage1) } + case "benchmarks.proto2.GoogleMessage1": + return func() proto.Message { return new(googleMessage1Proto2.GoogleMessage1) } + case "benchmarks.proto2.GoogleMessage2": + return func() proto.Message { return new(googleMessage2.GoogleMessage2) } + case "benchmarks.google_message3.GoogleMessage3": + return func() proto.Message { return new(googleMessage3.GoogleMessage3) } + case "benchmarks.google_message4.GoogleMessage4": + return func() proto.Message { return new(googleMessage4.GoogleMessage4) } + default: + panic("Unknown message type: " + dataset.MessageName) + } +} + +func init() { + flag.Parse() + for _, f := range flag.Args() { + // Load the benchmark. + b, err := ioutil.ReadFile(f) + if err != nil { + panic(err) + } + + // Parse the benchmark. + var dm benchmarkWrapper.BenchmarkDataset + if err := proto.Unmarshal(b, &dm); err != nil { + panic(err) + } + + // Determine the concrete protobuf message type to use. + var ds Dataset + ds.newMessage = generateNewMessageFunction(dm) + + // Unmarshal each test message. + for _, payload := range dm.Payload { + ds.marshaled = append(ds.marshaled, payload) + m := ds.newMessage() + if err := proto.Unmarshal(payload, m); err != nil { + panic(err) + } + ds.unmarshaled = append(ds.unmarshaled, m) + } + ds.name = f + + datasets = append(datasets, ds) + } +} + +func Benchmark(b *testing.B) { + for _, ds := range datasets { + b.Run(ds.name, func(b *testing.B) { + b.Run("Unmarshal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for j, payload := range ds.marshaled { + out := ds.newMessage() + if err := proto.Unmarshal(payload, out); err != nil { + b.Fatalf("can't unmarshal message %d %v", j, err) + } + } + } + }) + b.Run("Marshal", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for j, m := range ds.unmarshaled { + if _, err := proto.Marshal(m); err != nil { + b.Fatalf("can't marshal message %d %+v: %v", j, m, err) + } + } + } + }) + b.Run("Size", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + proto.Size(m) + } + } + }) + b.Run("Clone", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + proto.Clone(m) + } + } + }) + b.Run("Merge", func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, m := range ds.unmarshaled { + out := ds.newMessage() + proto.Merge(out, m) + } + } + }) + }) + } +} diff --git a/benchmarks/go_benchmark_test.go b/benchmarks/go_benchmark_test.go deleted file mode 100644 index e747465e..00000000 --- a/benchmarks/go_benchmark_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - benchmarkWrapper "./tmp" - googleMessage1Proto2 "./tmp/datasets/google_message1/proto2" - googleMessage1Proto3 "./tmp/datasets/google_message1/proto3" - googleMessage2 "./tmp/datasets/google_message2" - googleMessage3 "./tmp/datasets/google_message3" - googleMessage4 "./tmp/datasets/google_message4" - "flag" - "github.com/golang/protobuf/proto" - "io/ioutil" - "testing" -) - -// Data is returned by the Load function. -type Dataset struct { - name string - newMessage func() proto.Message - marshaled [][]byte - unmarshaled []proto.Message -} - -var datasets []Dataset - -// This is used to getDefaultInstance for a message type. -func generateNewMessageFunction(dataset benchmarkWrapper.BenchmarkDataset) func() proto.Message { - switch dataset.MessageName { - case "benchmarks.proto3.GoogleMessage1": - return func() proto.Message { return new(googleMessage1Proto3.GoogleMessage1) } - case "benchmarks.proto2.GoogleMessage1": - return func() proto.Message { return new(googleMessage1Proto2.GoogleMessage1) } - case "benchmarks.proto2.GoogleMessage2": - return func() proto.Message { return new(googleMessage2.GoogleMessage2) } - case "benchmarks.google_message3.GoogleMessage3": - return func() proto.Message { return new(googleMessage3.GoogleMessage3) } - case "benchmarks.google_message4.GoogleMessage4": - return func() proto.Message { return new(googleMessage4.GoogleMessage4) } - default: - panic("Unknown message type: " + dataset.MessageName) - } -} - -func init() { - flag.Parse() - for _, f := range flag.Args() { - // Load the benchmark. - b, err := ioutil.ReadFile(f) - if err != nil { - panic(err) - } - - // Parse the benchmark. - var dm benchmarkWrapper.BenchmarkDataset - if err := proto.Unmarshal(b, &dm); err != nil { - panic(err) - } - - // Determine the concrete protobuf message type to use. - var ds Dataset - ds.newMessage = generateNewMessageFunction(dm) - - // Unmarshal each test message. - for _, payload := range dm.Payload { - ds.marshaled = append(ds.marshaled, payload) - m := ds.newMessage() - if err := proto.Unmarshal(payload, m); err != nil { - panic(err) - } - ds.unmarshaled = append(ds.unmarshaled, m) - } - ds.name = f - - datasets = append(datasets, ds) - } -} - -func Benchmark(b *testing.B) { - for _, ds := range datasets { - b.Run(ds.name, func(b *testing.B) { - b.Run("Unmarshal", func(b *testing.B) { - for i := 0; i < b.N; i++ { - for j, payload := range ds.marshaled { - out := ds.newMessage() - if err := proto.Unmarshal(payload, out); err != nil { - b.Fatalf("can't unmarshal message %d %v", j, err) - } - } - } - }) - b.Run("Marshal", func(b *testing.B) { - for i := 0; i < b.N; i++ { - for j, m := range ds.unmarshaled { - if _, err := proto.Marshal(m); err != nil { - b.Fatalf("can't marshal message %d %+v: %v", j, m, err) - } - } - } - }) - b.Run("Size", func(b *testing.B) { - for i := 0; i < b.N; i++ { - for _, m := range ds.unmarshaled { - proto.Size(m) - } - } - }) - b.Run("Clone", func(b *testing.B) { - for i := 0; i < b.N; i++ { - for _, m := range ds.unmarshaled { - proto.Clone(m) - } - } - }) - b.Run("Merge", func(b *testing.B) { - for i := 0; i < b.N; i++ { - for _, m := range ds.unmarshaled { - out := ds.newMessage() - proto.Merge(out, m) - } - } - }) - }) - } -} diff --git a/benchmarks/py_benchmark.py b/benchmarks/py_benchmark.py deleted file mode 100755 index ba7a3470..00000000 --- a/benchmarks/py_benchmark.py +++ /dev/null @@ -1,118 +0,0 @@ -import sys -import os -import timeit -import math -import fnmatch - -# BEGIN CPP GENERATED MESSAGE -# CPP generated code must be linked before importing the generated Python code -# for the descriptor can be found in the pool -if len(sys.argv) < 2: - raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") -if sys.argv[1] == "true": - sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/.libs" ) - import libbenchmark_messages - sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/tmp" ) -elif sys.argv[1] != "false": - raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code") -# END CPP GENERATED MESSAGE - -import datasets.google_message1.proto2.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 -import datasets.google_message1.proto3.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 -import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2 -import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2 -import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2 -import benchmarks_pb2 as benchmarks_pb2 - - -def run_one_test(filename): - data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() - benchmark_dataset = benchmarks_pb2.BenchmarkDataset() - benchmark_dataset.ParseFromString(data) - benchmark_util = Benchmark(full_iteration=len(benchmark_dataset.payload), - module="py_benchmark", - setup_method="init") - print "Message %s of dataset file %s" % \ - (benchmark_dataset.message_name, filename) - benchmark_util.set_test_method("parse_from_benchmark") - print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) - benchmark_util.set_test_method("serialize_to_benchmark") - print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) - print "" - -def init(filename): - global benchmark_dataset, message_class, message_list, counter - message_list=[] - counter = 0 - data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() - benchmark_dataset = benchmarks_pb2.BenchmarkDataset() - benchmark_dataset.ParseFromString(data) - - if benchmark_dataset.message_name == "benchmarks.proto3.GoogleMessage1": - message_class = benchmark_message1_proto3_pb2.GoogleMessage1 - elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage1": - message_class = benchmark_message1_proto2_pb2.GoogleMessage1 - elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage2": - message_class = benchmark_message2_pb2.GoogleMessage2 - elif benchmark_dataset.message_name == "benchmarks.google_message3.GoogleMessage3": - message_class = benchmark_message3_pb2.GoogleMessage3 - elif benchmark_dataset.message_name == "benchmarks.google_message4.GoogleMessage4": - message_class = benchmark_message4_pb2.GoogleMessage4 - else: - raise IOError("Message %s not found!" % (benchmark_dataset.message_name)) - - for one_payload in benchmark_dataset.payload: - temp = message_class() - temp.ParseFromString(one_payload) - message_list.append(temp) - -def parse_from_benchmark(): - global counter, message_class, benchmark_dataset - m = message_class().ParseFromString(benchmark_dataset.payload[counter % len(benchmark_dataset.payload)]) - counter = counter + 1 - -def serialize_to_benchmark(): - global counter, message_list, message_class - s = message_list[counter % len(benchmark_dataset.payload)].SerializeToString() - counter = counter + 1 - - -class Benchmark: - def __init__(self, module=None, test_method=None, - setup_method=None, full_iteration = 1): - self.full_iteration = full_iteration - self.module = module - self.test_method = test_method - self.setup_method = setup_method - - def set_test_method(self, test_method): - self.test_method = test_method - - def full_setup_code(self, setup_method_args=''): - setup_code = "" - setup_code += "from %s import %s\n" % (self.module, self.test_method) - setup_code += "from %s import %s\n" % (self.module, self.setup_method) - setup_code += "%s(%s)\n" % (self.setup_method, setup_method_args) - return setup_code - - def dry_run(self, test_method_args='', setup_method_args=''): - return timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), - setup=self.full_setup_code(setup_method_args), - number=self.full_iteration); - - def run_benchmark(self, test_method_args='', setup_method_args=''): - reps = self.full_iteration; - t = self.dry_run(test_method_args, setup_method_args); - if t < 3 : - reps = int(math.ceil(3 / t)) * self.full_iteration - t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), - setup=self.full_setup_code(setup_method_args), - number=reps); - return "Average time for %s: %.2f ns" % \ - (self.test_method, 1.0 * t / reps * (10 ** 9)) - - -if __name__ == "__main__": - for i in range(2, len(sys.argv)): - run_one_test(sys.argv[i]) - diff --git a/benchmarks/python/__init__.py b/benchmarks/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/python/py_benchmark.py b/benchmarks/python/py_benchmark.py new file mode 100755 index 00000000..a551ba6d --- /dev/null +++ b/benchmarks/python/py_benchmark.py @@ -0,0 +1,150 @@ +import sys +import os +import timeit +import math +import argparse +import fnmatch +import json + +parser = argparse.ArgumentParser(description="Python protobuf benchmark") +parser.add_argument("data_files", metavar="dataFile", nargs="+", + help="testing data files.") +parser.add_argument("--json", action="store_const", dest="json", + const="yes", default="no", + help="Whether to output json results") +parser.add_argument("--behavior_prefix", dest="behavior_prefix", + help="The output json format's behavior's name's prefix", + default="") +# BEGIN CPP GENERATED MESSAGE +parser.add_argument("--cpp_generated", action="store_const", + dest="cpp_generated", const="yes", default="no", + help="Whether to link generated code library") +# END CPP GENERATED MESSAGE +args = parser.parse_args() +# BEGIN CPP GENERATED MESSAGE +# CPP generated code must be linked before importing the generated Python code +# for the descriptor can be found in the pool +if args.cpp_generated != "no": + sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/.libs" ) + import libbenchmark_messages + sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/tmp" ) +# END CPP GENERATED MESSAGE + + +import datasets.google_message1.proto2.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2 +import datasets.google_message1.proto3.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2 +import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2 +import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2 +import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2 +import benchmarks_pb2 as benchmarks_pb2 + + +def run_one_test(filename): + data = open(filename).read() + benchmark_dataset = benchmarks_pb2.BenchmarkDataset() + benchmark_dataset.ParseFromString(data) + benchmark_util = Benchmark(full_iteration=len(benchmark_dataset.payload), + module="py_benchmark", + setup_method="init") + result={} + result["filename"] = filename + result["message_name"] = benchmark_dataset.message_name + result["benchmarks"] = {} + benchmark_util.set_test_method("parse_from_benchmark") + result["benchmarks"][args.behavior_prefix + "_parse_from_benchmark"] = \ + benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) + benchmark_util.set_test_method("serialize_to_benchmark") + result["benchmarks"][args.behavior_prefix + "_serialize_to_benchmark"] = \ + benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename)) + return result + + +def init(filename): + global benchmark_dataset, message_class, message_list, counter + message_list=[] + counter = 0 + data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read() + benchmark_dataset = benchmarks_pb2.BenchmarkDataset() + benchmark_dataset.ParseFromString(data) + + if benchmark_dataset.message_name == "benchmarks.proto3.GoogleMessage1": + message_class = benchmark_message1_proto3_pb2.GoogleMessage1 + elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage1": + message_class = benchmark_message1_proto2_pb2.GoogleMessage1 + elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage2": + message_class = benchmark_message2_pb2.GoogleMessage2 + elif benchmark_dataset.message_name == "benchmarks.google_message3.GoogleMessage3": + message_class = benchmark_message3_pb2.GoogleMessage3 + elif benchmark_dataset.message_name == "benchmarks.google_message4.GoogleMessage4": + message_class = benchmark_message4_pb2.GoogleMessage4 + else: + raise IOError("Message %s not found!" % (benchmark_dataset.message_name)) + + for one_payload in benchmark_dataset.payload: + temp = message_class() + temp.ParseFromString(one_payload) + message_list.append(temp) + + +def parse_from_benchmark(): + global counter, message_class, benchmark_dataset + m = message_class().ParseFromString(benchmark_dataset.payload[counter % len(benchmark_dataset.payload)]) + counter = counter + 1 + + +def serialize_to_benchmark(): + global counter, message_list, message_class + s = message_list[counter % len(benchmark_dataset.payload)].SerializeToString() + counter = counter + 1 + + +class Benchmark: + def __init__(self, module=None, test_method=None, + setup_method=None, full_iteration = 1): + self.full_iteration = full_iteration + self.module = module + self.test_method = test_method + self.setup_method = setup_method + + def set_test_method(self, test_method): + self.test_method = test_method + + def full_setup_code(self, setup_method_args=''): + setup_code = "" + setup_code += "from %s import %s\n" % (self.module, self.test_method) + setup_code += "from %s import %s\n" % (self.module, self.setup_method) + setup_code += "%s(%s)\n" % (self.setup_method, setup_method_args) + return setup_code + + def dry_run(self, test_method_args='', setup_method_args=''): + return timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), + setup=self.full_setup_code(setup_method_args), + number=self.full_iteration); + + def run_benchmark(self, test_method_args='', setup_method_args=''): + reps = self.full_iteration; + t = self.dry_run(test_method_args, setup_method_args); + if t < 3 : + reps = int(math.ceil(3 / t)) * self.full_iteration + t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args), + setup=self.full_setup_code(setup_method_args), + number=reps); + return 1.0 * t / reps * (10 ** 9) + + +if __name__ == "__main__": + results = [] + for file in args.data_files: + results.append(run_one_test(file)) + + if args.json != "no": + print json.dumps(results) + else: + for result in results: + print "Message %s of dataset file %s" % \ + (result["message_name"], result["filename"]) + print "Average time for parse_from_benchmark: %.2f ns" % \ + (result["benchmarks"]["parse_from_benchmark"]) + print "Average time for serialize_to_benchmark: %.2f ns" % \ + (result["benchmarks"]["serialize_to_benchmark"]) + print "" diff --git a/benchmarks/python/python_benchmark_messages.cc b/benchmarks/python/python_benchmark_messages.cc new file mode 100644 index 00000000..ded16fe9 --- /dev/null +++ b/benchmarks/python/python_benchmark_messages.cc @@ -0,0 +1,29 @@ +#include + +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" + +static PyMethodDef python_benchmark_methods[] = { + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + + +PyMODINIT_FUNC +initlibbenchmark_messages() { + benchmarks::BenchmarkDataset().descriptor(); + benchmarks::proto3::GoogleMessage1().descriptor(); + benchmarks::proto2::GoogleMessage1().descriptor(); + benchmarks::proto2::GoogleMessage2().descriptor(); + benchmarks::google_message3::GoogleMessage3().descriptor(); + benchmarks::google_message4::GoogleMessage4().descriptor(); + + PyObject *m; + + m = Py_InitModule("libbenchmark_messages", python_benchmark_methods); + if (m == NULL) + return; +} diff --git a/benchmarks/python_benchmark_messages.cc b/benchmarks/python_benchmark_messages.cc deleted file mode 100644 index 55242a2a..00000000 --- a/benchmarks/python_benchmark_messages.cc +++ /dev/null @@ -1,29 +0,0 @@ -#include - -#include "benchmarks.pb.h" -#include "datasets/google_message1/benchmark_message1_proto2.pb.h" -#include "datasets/google_message1/benchmark_message1_proto3.pb.h" -#include "datasets/google_message2/benchmark_message2.pb.h" -#include "datasets/google_message3/benchmark_message3.pb.h" -#include "datasets/google_message4/benchmark_message4.pb.h" - -static PyMethodDef python_benchmark_methods[] = { - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - - -PyMODINIT_FUNC -initlibbenchmark_messages() { - benchmarks::BenchmarkDataset().descriptor(); - benchmarks::proto3::GoogleMessage1().descriptor(); - benchmarks::proto2::GoogleMessage1().descriptor(); - benchmarks::proto2::GoogleMessage2().descriptor(); - benchmarks::google_message3::GoogleMessage3().descriptor(); - benchmarks::google_message4::GoogleMessage4().descriptor(); - - PyObject *m; - - m = Py_InitModule("libbenchmark_messages", python_benchmark_methods); - if (m == NULL) - return; -} diff --git a/benchmarks/util/gogo_data_scrubber.cc b/benchmarks/util/gogo_data_scrubber.cc new file mode 100644 index 00000000..fb9af6e2 --- /dev/null +++ b/benchmarks/util/gogo_data_scrubber.cc @@ -0,0 +1,105 @@ +#include "benchmarks.pb.h" +#include "datasets/google_message1/proto2/benchmark_message1_proto2.pb.h" +#include "datasets/google_message1/proto3/benchmark_message1_proto3.pb.h" +#include "datasets/google_message2/benchmark_message2.pb.h" +#include "datasets/google_message3/benchmark_message3.pb.h" +#include "datasets/google_message4/benchmark_message4.pb.h" + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" + +#include + +using google::protobuf::FieldDescriptor; +using google::protobuf::Message; +using google::protobuf::Reflection; + + +class DataGroupStripper { + public: + static void StripMessage(Message *message) { + std::vector set_fields; + const Reflection* reflection = message->GetReflection(); + reflection->ListFields(*message, &set_fields); + + for (size_t i = 0; i < set_fields.size(); i++) { + const FieldDescriptor* field = set_fields[i]; + if (field->type() == FieldDescriptor::TYPE_GROUP) { + reflection->ClearField(message, field); + } + if (field->type() == FieldDescriptor::TYPE_MESSAGE) { + if (field->is_repeated()) { + for (int j = 0; j < reflection->FieldSize(*message, field); j++) { + StripMessage(reflection->MutableRepeatedMessage(message, field, j)); + } + } else { + StripMessage(reflection->MutableMessage(message, field)); + } + } + } + + reflection->MutableUnknownFields(message)->Clear(); + } +}; + +std::string ReadFile(const std::string& name) { + std::ifstream file(name.c_str()); + GOOGLE_CHECK(file.is_open()) << "Couldn't find file '" + << name + << "', please make sure you are running this command from the benchmarks" + << " directory.\n"; + return std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); +} + +int main(int argc, char *argv[]) { + if (argc % 2 == 0 || argc == 1) { + std::cerr << "Usage: [input_files] [output_file_names] where " << + "input_files are one to one mapping to output_file_names." << + std::endl; + return 1; + } + + for (int i = argc / 2; i > 0; i--) { + const std::string &input_file = argv[i]; + const std::string &output_file = argv[i + argc / 2]; + + std::cerr << "Generating " << input_file + << " to " << output_file << std::endl; + benchmarks::BenchmarkDataset dataset; + Message* message; + std::string dataset_payload = ReadFile(input_file); + GOOGLE_CHECK(dataset.ParseFromString(dataset_payload)) + << "Can' t parse data file " << input_file; + + if (dataset.message_name() == "benchmarks.proto3.GoogleMessage1") { + message = new benchmarks::proto3::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage1") { + message = new benchmarks::proto2::GoogleMessage1; + } else if (dataset.message_name() == "benchmarks.proto2.GoogleMessage2") { + message = new benchmarks::proto2::GoogleMessage2; + } else if (dataset.message_name() == + "benchmarks.google_message3.GoogleMessage3") { + message = new benchmarks::google_message3::GoogleMessage3; + } else if (dataset.message_name() == + "benchmarks.google_message4.GoogleMessage4") { + message = new benchmarks::google_message4::GoogleMessage4; + } else { + std::cerr << "Unknown message type: " << dataset.message_name(); + exit(1); + } + + for (int i = 0; i < dataset.payload_size(); i++) { + message->ParseFromString(dataset.payload(i)); + DataGroupStripper::StripMessage(message); + dataset.set_payload(i, message->SerializeAsString()); + } + + std::ofstream ofs(output_file); + ofs << dataset.SerializeAsString(); + ofs.close(); + } + + + return 0; +} diff --git a/benchmarks/util/protoc-gen-gogoproto.cc b/benchmarks/util/protoc-gen-gogoproto.cc new file mode 100644 index 00000000..bfa6a5e5 --- /dev/null +++ b/benchmarks/util/protoc-gen-gogoproto.cc @@ -0,0 +1,103 @@ +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/io/zero_copy_stream.h" +#include "google/protobuf/io/printer.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "schema_proto2_to_proto3_util.h" + +#include "google/protobuf/compiler/plugin.h" + +using google::protobuf::FileDescriptorProto; +using google::protobuf::FileDescriptor; +using google::protobuf::DescriptorPool; +using google::protobuf::io::Printer; +using google::protobuf::util::SchemaGroupStripper; +using google::protobuf::util::SchemaAddZeroEnumValue; + +namespace google { +namespace protobuf { +namespace compiler { + +namespace { + +string StripProto(string filename) { + if (filename.substr(filename.size() - 11) == ".protodevel") { + // .protodevel + return filename.substr(0, filename.size() - 11); + } else { + // .proto + return filename.substr(0, filename.size() - 6); + } +} + +DescriptorPool new_pool_; + +} // namespace + +class GoGoProtoGenerator : public CodeGenerator { + public: + virtual bool GenerateAll(const std::vector& files, + const string& parameter, + GeneratorContext* context, + string* error) const { + for (int i = 0; i < files.size(); i++) { + for (auto file : files) { + bool can_generate = + (new_pool_.FindFileByName(file->name()) == nullptr); + for (int j = 0; j < file->dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->dependency(j)->name()) != nullptr); + } + for (int j = 0; j < file->public_dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->public_dependency(j)->name()) != nullptr); + } + for (int j = 0; j < file->weak_dependency_count(); j++) { + can_generate &= (new_pool_.FindFileByName( + file->weak_dependency(j)->name()) != nullptr); + } + if (can_generate) { + Generate(file, parameter, context, error); + break; + } + } + } + + return true; + } + + virtual bool Generate(const FileDescriptor* file, + const string& parameter, + GeneratorContext* context, + string* error) const { + FileDescriptorProto new_file; + file->CopyTo(&new_file); + SchemaGroupStripper::StripFile(file, &new_file); + + SchemaAddZeroEnumValue enum_scrubber; + enum_scrubber.ScrubFile(&new_file); + + string filename = file->name(); + string basename = StripProto(filename); + + std::vector> option_pairs; + ParseGeneratorParameter(parameter, &option_pairs); + + std::unique_ptr output( + context->Open(basename + ".proto")); + string content = new_pool_.BuildFile(new_file)->DebugString(); + Printer printer(output.get(), '$'); + printer.WriteRaw(content.c_str(), content.size()); + + return true; + } +}; + +} // namespace compiler +} // namespace protobuf +} // namespace google + +int main(int argc, char* argv[]) { + google::protobuf::compiler::GoGoProtoGenerator generator; + return google::protobuf::compiler::PluginMain(argc, argv, &generator); +} diff --git a/benchmarks/util/schema_proto2_to_proto3_util.h b/benchmarks/util/schema_proto2_to_proto3_util.h new file mode 100644 index 00000000..089012dd --- /dev/null +++ b/benchmarks/util/schema_proto2_to_proto3_util.h @@ -0,0 +1,137 @@ +#ifndef PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ +#define PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ + +#include "google/protobuf/message.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" + +#include +#include + +using google::protobuf::Descriptor; +using google::protobuf::DescriptorProto; +using google::protobuf::FileDescriptorProto; +using google::protobuf::FieldDescriptorProto; +using google::protobuf::Message; +using google::protobuf::EnumValueDescriptorProto; + +namespace google { +namespace protobuf { +namespace util { + +class SchemaGroupStripper { + + public: + static void StripFile(const FileDescriptor* old_file, + FileDescriptorProto *file) { + for (int i = file->mutable_message_type()->size() - 1; i >= 0; i--) { + if (IsMessageSet(old_file->message_type(i))) { + file->mutable_message_type()->DeleteSubrange(i, 1); + continue; + } + StripMessage(old_file->message_type(i), file->mutable_message_type(i)); + } + for (int i = file->mutable_extension()->size() - 1; i >= 0; i--) { + auto field = old_file->extension(i); + if (field->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(field->message_type()) || + IsMessageSet(field->containing_type())) { + file->mutable_extension()->DeleteSubrange(i, 1); + } + } + } + + private: + static bool IsMessageSet(const Descriptor *descriptor) { + if (descriptor != nullptr + && descriptor->options().message_set_wire_format()) { + return true; + } + return false; + } + + static void StripMessage(const Descriptor *old_message, + DescriptorProto *new_message) { + for (int i = new_message->mutable_field()->size() - 1; i >= 0; i--) { + if (old_message->field(i)->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(old_message->field(i)->message_type())) { + new_message->mutable_field()->DeleteSubrange(i, 1); + } + } + for (int i = new_message->mutable_extension()->size() - 1; i >= 0; i--) { + auto field_type_name = new_message->mutable_extension(i)->type_name(); + if (old_message->extension(i)->type() == FieldDescriptor::TYPE_GROUP || + IsMessageSet(old_message->extension(i)->containing_type()) || + IsMessageSet(old_message->extension(i)->message_type())) { + new_message->mutable_extension()->DeleteSubrange(i, 1); + } + } + for (int i = 0; i < new_message->mutable_nested_type()->size(); i++) { + StripMessage(old_message->nested_type(i), + new_message->mutable_nested_type(i)); + } + } + +}; + +class SchemaAddZeroEnumValue { + + public: + SchemaAddZeroEnumValue() + : total_added_(0) { + } + + void ScrubFile(FileDescriptorProto *file) { + for (int i = 0; i < file->enum_type_size(); i++) { + ScrubEnum(file->mutable_enum_type(i)); + } + for (int i = 0; i < file->mutable_message_type()->size(); i++) { + ScrubMessage(file->mutable_message_type(i)); + } + } + + private: + void ScrubEnum(EnumDescriptorProto *enum_type) { + if (enum_type->value(0).number() != 0) { + bool has_zero = false; + for (int j = 0; j < enum_type->value().size(); j++) { + if (enum_type->value(j).number() == 0) { + EnumValueDescriptorProto temp_enum_value; + temp_enum_value.CopyFrom(enum_type->value(j)); + enum_type->mutable_value(j)->CopyFrom(enum_type->value(0)); + enum_type->mutable_value(0)->CopyFrom(temp_enum_value); + has_zero = true; + break; + } + } + if (!has_zero) { + enum_type->mutable_value()->Add(); + for (int i = enum_type->mutable_value()->size() - 1; i > 0; i--) { + enum_type->mutable_value(i)->CopyFrom( + *enum_type->mutable_value(i - 1)); + } + enum_type->mutable_value(0)->set_number(0); + enum_type->mutable_value(0)->set_name("ADDED_ZERO_VALUE_" + + std::to_string(total_added_++)); + } + } + + } + + void ScrubMessage(DescriptorProto *message_type) { + for (int i = 0; i < message_type->mutable_enum_type()->size(); i++) { + ScrubEnum(message_type->mutable_enum_type(i)); + } + for (int i = 0; i < message_type->mutable_nested_type()->size(); i++) { + ScrubMessage(message_type->mutable_nested_type(i)); + } + } + + int total_added_; +}; + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // PROTOBUF_BENCHMARKS_UTIL_SCHEMA_PROTO2_TO_PROTO3_UTIL_H_ -- cgit v1.2.3