From 5e732e35c1aea3a00903aae50f63cf1cf9166833 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Tue, 12 Dec 2017 12:05:29 -0800 Subject: Add caliper supported to java benchmark --- benchmarks/Makefile.am | 21 +- benchmarks/ProtoBench.java | 319 --------------------- benchmarks/README.md | 23 +- benchmarks/initialize_submodule.sh | 7 + benchmarks/java/pom.xml | 148 ++++++++++ .../com/google/protocolbuffers/ProtoBench.java | 155 ++++++++++ .../google/protocolbuffers/ProtoBenchCaliper.java | 210 ++++++++++++++ 7 files changed, 552 insertions(+), 331 deletions(-) mode change 100644 => 100755 benchmarks/Makefile.am delete mode 100644 benchmarks/ProtoBench.java create mode 100755 benchmarks/initialize_submodule.sh create mode 100755 benchmarks/java/pom.xml create mode 100755 benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java create mode 100755 benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java (limited to 'benchmarks') diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am old mode 100644 new mode 100755 index e9e08a0d..e69d9fb7 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 = \ datasets/google_message4/benchmark_message4_3.proto make_tmp_dir: - mkdir -p 'tmp' + mkdir -p 'tmp/java/src/main/java' touch make_tmp_dir if USE_EXTERNAL_PROTOC @@ -39,11 +39,11 @@ else # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) ) touch protoc_middleman protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) ) touch protoc_middleman2 endif @@ -124,18 +124,19 @@ cpp: protoc_middleman protoc_middleman2 cpp-benchmark ############# JAVA RULES ############## -javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2 - jar=`ls $(top_srcdir)/java/util/target/*.jar` && \ - jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \ - javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"` +java_benchmark_testing_files = + java/src/main/java/com/google/protocolbuffers/ProtoBench.java \ + java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java + +javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2 + cp -r java tmp && cd tmp/java && mvn clean compile assembly:single + cd ../.. @touch javac_middleman java-benchmark: javac_middleman @echo "Writing shortcut script java-benchmark..." @echo '#! /bin/sh' > java-benchmark - @jar=`ls $(top_srcdir)/java/util/target/*.jar` && \ - jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \ - echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark + @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark @chmod +x java-benchmark java: protoc_middleman protoc_middleman2 java-benchmark diff --git a/benchmarks/ProtoBench.java b/benchmarks/ProtoBench.java deleted file mode 100644 index cde9095d..00000000 --- a/benchmarks/ProtoBench.java +++ /dev/null @@ -1,319 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2009 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package com.google.protocolbuffers; - -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.Message; -import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.List; - -public class ProtoBench { - - private static final long MIN_SAMPLE_TIME_MS = 1 * 1000; - private static final long TARGET_TIME_MS = 5 * 1000; - - private ProtoBench() { - // Prevent instantiation - } - - public static void main(String[] args) { - if (args.length < 1) { - System.err.println("Usage: ./java-benchmark "); - System.err.println("input data is in the format of \"benchmarks.proto\""); - System.exit(1); - } - boolean success = true; - for (int i = 0; i < args.length; i++) { - success &= runTest(args[i]); - } - System.exit(success ? 0 : 1); - } - - public static ExtensionRegistry getExtensionsRegistry(BenchmarkDataset benchmarkDataset) { - ExtensionRegistry extensions = ExtensionRegistry.newInstance(); - if (benchmarkDataset.getMessageName().equals("benchmarks.google_message3.GoogleMessage3")) { - benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions); - } else if (benchmarkDataset.getMessageName().equals( - "benchmarks.google_message4.GoogleMessage4")) { - benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions); - } - - return extensions; - } - - /** - * Return an message instance for one specific dataset, and register the extensions for that - * message. - */ - public static Message registerBenchmarks(BenchmarkDataset benchmarkDataset) { - if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { - return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 - .getDefaultInstance(); - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { - return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 - .getDefaultInstance(); - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { - return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message3.GoogleMessage3")) { - return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message4.GoogleMessage4")) { - return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); - } else { - return null; - } - } - - /** - * Runs a single test. Error messages are displayed to stderr, and the return value indicates - * general success/failure. - */ - public static boolean runTest(String file) { - final Message defaultMessage; - BenchmarkDataset benchmarkDataset; - ExtensionRegistry extensions; - final byte[] inputData; - - try { - inputData = readAllBytes(file); - benchmarkDataset = BenchmarkDataset.parseFrom(inputData); - } catch (IOException e) { - System.err.println("Unable to get input data"); - return false; - } - defaultMessage = registerBenchmarks(benchmarkDataset); - extensions = getExtensionsRegistry(benchmarkDataset); - if (defaultMessage == null) { - System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); - return false; - } - System.out.println("Benchmarking " + benchmarkDataset.getMessageName() + " with file " + file); - - try { - List inputDataList = new ArrayList(); - List inputStreamList = new ArrayList(); - List inputStringList = new ArrayList(); - List sampleMessageList = new ArrayList(); - - for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { - byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); - inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); - inputStreamList.add(new ByteArrayInputStream(benchmarkDataset.getPayload(i).toByteArray())); - inputStringList.add(benchmarkDataset.getPayload(i)); - sampleMessageList.add( - defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); - } - - FileOutputStream devNullTemp = null; - CodedOutputStream reuseDevNullTemp = null; - try { - devNullTemp = new FileOutputStream("/dev/null"); - reuseDevNullTemp = CodedOutputStream.newInstance(devNullTemp); - } catch (FileNotFoundException e) { - // ignore: this is probably Windows, where /dev/null does not exist - } - final FileOutputStream devNull = devNullTemp; - final CodedOutputStream reuseDevNull = reuseDevNullTemp; - benchmark( - "Serialize to byte string", - inputData.length, - new Action() { - public void execute() { - for (int i = 0; i < sampleMessageList.size(); i++) { - sampleMessageList.get(i).toByteString(); - } - } - }); - benchmark( - "Serialize to byte array", - inputData.length, - new Action() { - public void execute() { - for (int i = 0; i < sampleMessageList.size(); i++) { - sampleMessageList.get(i).toByteString(); - } - } - }); - benchmark( - "Serialize to memory stream", - inputData.length, - new Action() { - public void execute() throws IOException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - for (int i = 0; i < sampleMessageList.size(); i++) { - sampleMessageList.get(i).writeTo(output); - } - } - }); - if (devNull != null) { - benchmark( - "Serialize to /dev/null with FileOutputStream", - inputData.length, - new Action() { - public void execute() throws IOException { - for (int i = 0; i < sampleMessageList.size(); i++) { - sampleMessageList.get(i).writeTo(devNull); - } - } - }); - benchmark( - "Serialize to /dev/null reusing FileOutputStream", - inputData.length, - new Action() { - public void execute() throws IOException { - for (int i = 0; i < sampleMessageList.size(); i++) { - sampleMessageList.get(i).writeTo(reuseDevNull); - reuseDevNull.flush(); // force the write to the OutputStream - } - } - }); - } - benchmark( - "Deserialize from byte string", - inputData.length, - new Action() { - public void execute() throws IOException { - for (int i = 0; i < inputStringList.size(); i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputStringList.get(i), extensions) - .build(); - } - } - }); - benchmark( - "Deserialize from byte array", - inputData.length, - new Action() { - public void execute() throws IOException { - for (int i = 0; i < inputDataList.size(); i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(CodedInputStream.newInstance(inputDataList.get(i)), extensions) - .build(); - } - } - }); - benchmark( - "Deserialize from memory stream", - inputData.length, - new Action() { - public void execute() throws IOException { - for (int i = 0; i < inputStreamList.size(); i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(CodedInputStream.newInstance(inputStreamList.get(i)), extensions) - .build(); - inputStreamList.get(i).reset(); - } - } - }); - System.out.println(); - return true; - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - System.err.println("Detailed exception information:"); - e.printStackTrace(System.err); - return false; - } - } - - private static void benchmark(String name, long dataSize, Action action) throws IOException { - // Make sure it's JITted "reasonably" hard before running the first progress test - for (int i=0; i < 100; i++) { - action.execute(); - } - - // Run it progressively more times until we've got a reasonable sample - int iterations = 1; - long elapsed = timeAction(action, iterations); - while (elapsed < MIN_SAMPLE_TIME_MS) { - iterations *= 2; - elapsed = timeAction(action, iterations); - } - - // Upscale the sample to the target time. Do this in floating point arithmetic - // to avoid overflow issues. - iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations); - elapsed = timeAction(action, iterations); - System.out.println(name + ": " + iterations + " iterations in " - + (elapsed/1000f) + "s; " - + (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f) - + "MB/s"); - } - - private static long timeAction(Action action, int iterations) throws IOException { - System.gc(); - long start = System.currentTimeMillis(); - for (int i = 0; i < iterations; i++) { - action.execute(); - } - long end = System.currentTimeMillis(); - return end - start; - } - - private static byte[] readAllBytes(String filename) throws IOException { - RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); - byte[] content = new byte[(int) file.length()]; - file.readFully(content); - return content; - } - - /** - * Interface used to capture a single action to benchmark. - */ - interface Action { - void execute() throws IOException; - } -} diff --git a/benchmarks/README.md b/benchmarks/README.md index 0273e389..00a63704 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,6 +7,18 @@ protobuf language runtime. The schema for the datasets is described in `benchmarks.proto`. +The benchmark is based on some submodules. To initialize the submodues: + +For java: +``` +$ ./initialize_submodule.sh java +``` + +For java: +``` +$ ./initialize_submodule.sh cpp +``` + To run all the benchmark dataset: For java: @@ -26,17 +38,24 @@ To run a specific dataset: For java: ``` -$ make java +$ make java-benchmark $ ./java-benchmark $(specific generated dataset file name) ``` For cpp: ``` -$ make cpp +$ make cpp-benchmark $ ./cpp-benchmark $(specific generated dataset file name) ``` +There's some big testing data not included in the directory initially, you need to +run the following command to download the testing data: + +``` +$ ./download_data.sh +``` + Each data set is in the format of benchmarks.proto: 1. name is the benchmark dataset's name. 2. message_name is the benchmark's message type full name (including package and message name) diff --git a/benchmarks/initialize_submodule.sh b/benchmarks/initialize_submodule.sh new file mode 100755 index 00000000..f3456adf --- /dev/null +++ b/benchmarks/initialize_submodule.sh @@ -0,0 +1,7 @@ +#! /bin/sh + +oldpwd=`pwd` +cd "../third_party" +git submodule update --init -r +cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd .. +cd "$oldpwd" diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml new file mode 100755 index 00000000..e19f3f59 --- /dev/null +++ b/benchmarks/java/pom.xml @@ -0,0 +1,148 @@ + + + + + + 4.0.0 + + protobuf-java-benchmark + com.google.protocolbuffers + 1.0.0 + Protocol Buffers [Benchmark] + + The benchmark tools for Protobuf Java. + + + + + com.google.protobuf + protobuf-java + 3.5.0 + + + com.google.caliper + caliper + 1.0-beta-2 + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4.1 + + + + jar-with-dependencies + + + + + com.mkyong.core.utils.App + + + + + + + make-assembly + + package + + single + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + + true + com.google.protocolbuffers.ProtoBench + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.4 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + true + all + + + + com.google.monitoring.runtime.instrumentation.AllocationInstrumenter + true + true + + + + + + + + + + + + + diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java new file mode 100755 index 00000000..72f2d629 --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java @@ -0,0 +1,155 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2009 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +package com.google.protocolbuffers; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.caliper.runner.CaliperMain; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; + + +public class ProtoBench { + + private ProtoBench() { + // Prevent instantiation + } + + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Usage: ./java-benchmark "); + System.err.println("input data is in the format of \"benchmarks.proto\""); + System.exit(1); + } + boolean success = true; + for (int i = 0; i < args.length; i++) { + success &= runTest(args[i]); + } + System.exit(success ? 0 : 1); + } + + /** + * Runs a single test with specific test data. Error messages are displayed to stderr, + * and the return value indicates general success/failure. + */ + public static boolean runTest(String file) { + byte[] inputData; + BenchmarkDataset benchmarkDataset; + try { + inputData = readAllBytes(file); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + } catch (IOException e) { + System.err.println("Unable to get input data"); + return false; + } + + List argsList = getCaliperOption(benchmarkDataset); + if (argsList == null) { + System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); + return false; + } + argsList.add("-DdataFile=" + file); + argsList.add("com.google.protocolbuffers.ProtoBenchCaliper"); + + try { + String args[] = new String[argsList.size()]; + argsList.toArray(args); + CaliperMain.exitlessMain(args, + new PrintWriter(System.out, true), new PrintWriter(System.err, true)); + return true; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + System.err.println("Detailed exception information:"); + e.printStackTrace(System.err); + return false; + } + } + + + private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) { + List temp = new ArrayList(); + if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3"); + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2"); + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2"); + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message3.GoogleMessage3")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3"); + for (String opt : ProtoBenchCaliper + .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) { + temp.add(opt); + } + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message4.GoogleMessage4")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4"); + for (String opt : ProtoBenchCaliper + .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) { + temp.add(opt); + } + } else { + return null; + } + + temp.add("-i"); + temp.add("runtime"); + temp.add("-b"); + String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" + + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; + temp.add(benchmarkNames); + + return temp; + } + + public static byte[] readAllBytes(String filename) throws IOException { + RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); + byte[] content = new byte[(int) file.length()]; + file.readFully(content); + return content; + } +} diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java new file mode 100755 index 00000000..4045b56a --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java @@ -0,0 +1,210 @@ + +package com.google.protocolbuffers; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ProtoBenchCaliper { + public enum BenchmarkMessageType { + GOOGLE_MESSAGE1_PROTO3 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 + .getDefaultInstance(); + } + @Override String[] getSpecificCaliperOption() { return null; } + }, + GOOGLE_MESSAGE1_PROTO2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 + .getDefaultInstance(); + } + @Override String[] getSpecificCaliperOption() { return null; } + }, + GOOGLE_MESSAGE2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); + } + @Override String[] getSpecificCaliperOption() { return null; } + }, + GOOGLE_MESSAGE3 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); + } + @Override + String[] getSpecificCaliperOption() { + String[] opt = new String[1]; + opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms"; + return opt; + } + }, + GOOGLE_MESSAGE4 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); + } + @Override + String[] getSpecificCaliperOption() { + String[] opt = new String[1]; + opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms"; + return opt; + } + }; + + abstract ExtensionRegistry getExtensionRegistry(); + abstract Message getDefaultInstance(); + abstract String[] getSpecificCaliperOption(); + } + + @Param + private BenchmarkMessageType benchmarkMessageType; + @Param + private String dataFile; + + private byte[] inputData; + private BenchmarkDataset benchmarkDataset; + private Message defaultMessage; + private ExtensionRegistry extensions; + private List inputDataList; + private List inputStreamList; + private List inputStringList; + private List sampleMessageList; + private int counter; + private FileOutputStream devNull; + private CodedOutputStream reuseDevNull; + + @BeforeExperiment + void setUp() throws IOException { + inputData = ProtoBench.readAllBytes(dataFile); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + defaultMessage = benchmarkMessageType.getDefaultInstance(); + extensions = benchmarkMessageType.getExtensionRegistry(); + inputDataList = new ArrayList(); + inputStreamList = new ArrayList(); + inputStringList = new ArrayList(); + sampleMessageList = new ArrayList(); + + for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { + byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); + inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); + inputStreamList.add(new ByteArrayInputStream( + benchmarkDataset.getPayload(i).toByteArray())); + inputStringList.add(benchmarkDataset.getPayload(i)); + sampleMessageList.add( + defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); + } + devNull = null; + reuseDevNull = null; + + try { + devNull = new FileOutputStream("/dev/null"); + reuseDevNull = CodedOutputStream.newInstance(devNull); + } catch (FileNotFoundException e) { + // ignore: this is probably Windows, where /dev/null does not exist + } + + counter = 0; + } + + + @Benchmark + void serializeToByteString(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + sampleMessageList.get(counter % sampleMessageList.size()).toByteString(); + counter++; + } + } + + @Benchmark + void serializeToByteArray(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + sampleMessageList.get(counter % sampleMessageList.size()).toByteArray(); + counter++; + } + } + + @Benchmark + void serializeToMemoryStream(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output); + counter++; + } + } + + @Benchmark + void deserializeFromByteString(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions) + .build(); + counter++; + } + } + + @Benchmark + void deserializeFromByteArray(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions) + .build(); + counter++; + } + } + + @Benchmark + void deserializeFromMemoryStream(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions) + .build(); + inputStreamList.get(counter % inputStreamList.size()).reset(); + counter++; + } + } +} -- cgit v1.2.3