From 98e097233f1cb2978ee2e57671bcd8e0e65e0dd2 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Mon, 21 May 2018 14:56:56 -0700 Subject: Fix java benchmark bug, fix python library_path --- benchmarks/Makefile.am | 14 ++++++---- benchmarks/java/pom.xml | 5 +++- .../com/google/protobuf/ProtoCaliperBenchmark.java | 32 +++++----------------- kokoro/linux/benchmark/build.sh | 12 +++++--- 4 files changed, 27 insertions(+), 36 deletions(-) diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index 564fdab3..ed893457 100644 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -126,8 +126,10 @@ java_benchmark_testing_files = \ java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2 - cp -r $(srcdir)/java tmp && cd tmp/java && mvn clean compile assembly:single - cd ../.. + cp -r $(srcdir)/java tmp + mkdir -p tmp/java/lib + cp $(top_srcdir)/java/core/target/*.jar tmp/java/lib/protobuf-java.jar + cd tmp/java && mvn clean compile assembly:single -Dprotobuf.version=$(PACKAGE_VERSION) && cd ../.. @touch javac_middleman java-benchmark: javac_middleman @@ -137,10 +139,10 @@ java-benchmark: javac_middleman @echo 'conf=()' >> java-benchmark @echo 'data_files=""' >> java-benchmark @echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+="$$arg,"; fi; done' >> java-benchmark - @echo 'java -cp '"tmp/java/target/*.jar"' com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\" >> java-benchmark - @echo '-b serializeToByteString,serializeToByteArray,serializeToMemoryStream,'"\\" >> java-benchmark - @echo 'deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark - @echo '-DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark + @echo 'java -cp '\"tmp/java/target/*:$(top_srcdir)/java/core/target/*:$(top_srcdir)/java/util/target/*\"" \\" >>java-benchmark + @echo ' com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\" >> java-benchmark + @echo ' -b serializeToByteArray,serializeToMemoryStream,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark + @echo ' -DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark @chmod +x java-benchmark java: protoc_middleman protoc_middleman2 java-benchmark diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index c2cd78a1..570bd664 100755 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -14,7 +14,10 @@ com.google.protobuf protobuf-java - 3.5.0 + ${protobuf.version} + jar + system + ${project.basedir}/lib/protobuf-java.jar com.google.caliper diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java index 94568aea..c766d74e 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -5,6 +5,7 @@ import com.google.caliper.BeforeExperiment; import com.google.caliper.AfterExperiment; import com.google.caliper.Benchmark; import com.google.caliper.Param; +import com.google.caliper.api.VmOptions; import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import com.google.protobuf.ExtensionRegistry; @@ -22,6 +23,12 @@ import java.io.RandomAccessFile; import java.util.ArrayList; import java.util.List; +// Caliper set CICompilerCount to 1 for making sure compilation doesn't run in parallel with itself, +// This makes TieredCompilation not working. We just disable TieredCompilation by default. In master +// branch this has been disabled by default in caliper: +// https://github.com/google/caliper/blob/master/caliper-runner/src/main/java/com/google/caliper/runner/target/Jvm.java#L38:14 +// But this haven't been added into most recent release. +@VmOptions("-XX:-TieredCompilation") public class ProtoCaliperBenchmark { public enum BenchmarkMessageType { GOOGLE_MESSAGE1_PROTO3 { @@ -151,18 +158,6 @@ public class ProtoCaliperBenchmark { } - @Benchmark - void serializeToByteString(int reps) throws IOException { - if (sampleMessageList.size() == 0) { - return; - } - for (int i = 0; i < reps; i++) { - for (int j = 0; j < sampleMessageList.size(); j++) { - sampleMessageList.get(j).toByteString(); - } - } - } - @Benchmark void serializeToByteArray(int reps) throws IOException { if (sampleMessageList.size() == 0) { @@ -188,19 +183,6 @@ public class ProtoCaliperBenchmark { } } - @Benchmark - void deserializeFromByteString(int reps) throws IOException { - if (inputStringList.size() == 0) { - return; - } - for (int i = 0; i < reps; i++) { - for (int j = 0; j < inputStringList.size(); j++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStringList.get(j), extensions); - } - } - } - @Benchmark void deserializeFromByteArray(int reps) throws IOException { if (inputDataList.size() == 0) { diff --git a/kokoro/linux/benchmark/build.sh b/kokoro/linux/benchmark/build.sh index f29a16fd..af5b299e 100755 --- a/kokoro/linux/benchmark/build.sh +++ b/kokoro/linux/benchmark/build.sh @@ -41,10 +41,10 @@ echo "benchmarking pure python..." ./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets >> tmp/python_result.json echo "," >> "tmp/python_result.json" echo "benchmarking python cpp reflection..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets >> tmp/python_result.json echo "," >> "tmp/python_result.json" echo "benchmarking python cpp generated code..." -env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json +env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json echo "]" >> "tmp/python_result.json" cd $oldpwd @@ -52,6 +52,11 @@ cd $oldpwd ./configure make clean && make -j8 +# build Java protobuf +cd java +mvn package +cd .. + # build CPP benchmark cd benchmarks mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp @@ -78,11 +83,10 @@ echo "benchmarking go..." make java-benchmark echo "benchmarking java..." ./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets -cat $(find /tmp -name "trail-1.log") # upload result to bq make python_add_init -python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \ +env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \ -python="../tmp/python_result.json" -go="../tmp/go_result.txt" cd $oldpwd -- cgit v1.2.3