From 5e732e35c1aea3a00903aae50f63cf1cf9166833 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Tue, 12 Dec 2017 12:05:29 -0800 Subject: Add caliper supported to java benchmark --- benchmarks/java/pom.xml | 148 +++++++++++++++ .../com/google/protocolbuffers/ProtoBench.java | 155 +++++++++++++++ .../google/protocolbuffers/ProtoBenchCaliper.java | 210 +++++++++++++++++++++ 3 files changed, 513 insertions(+) create mode 100755 benchmarks/java/pom.xml create mode 100755 benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java create mode 100755 benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java (limited to 'benchmarks/java') diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml new file mode 100755 index 00000000..e19f3f59 --- /dev/null +++ b/benchmarks/java/pom.xml @@ -0,0 +1,148 @@ + + + + + + 4.0.0 + + protobuf-java-benchmark + com.google.protocolbuffers + 1.0.0 + Protocol Buffers [Benchmark] + + The benchmark tools for Protobuf Java. + + + + + com.google.protobuf + protobuf-java + 3.5.0 + + + com.google.caliper + caliper + 1.0-beta-2 + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4.1 + + + + jar-with-dependencies + + + + + com.mkyong.core.utils.App + + + + + + + make-assembly + + package + + single + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + + true + com.google.protocolbuffers.ProtoBench + + + + + + org.apache.maven.plugins + maven-source-plugin + 2.4 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + true + all + + + + com.google.monitoring.runtime.instrumentation.AllocationInstrumenter + true + true + + + + + + + + + + + + + diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java new file mode 100755 index 00000000..72f2d629 --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java @@ -0,0 +1,155 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2009 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +package com.google.protocolbuffers; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.caliper.runner.CaliperMain; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; + + +public class ProtoBench { + + private ProtoBench() { + // Prevent instantiation + } + + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Usage: ./java-benchmark "); + System.err.println("input data is in the format of \"benchmarks.proto\""); + System.exit(1); + } + boolean success = true; + for (int i = 0; i < args.length; i++) { + success &= runTest(args[i]); + } + System.exit(success ? 0 : 1); + } + + /** + * Runs a single test with specific test data. Error messages are displayed to stderr, + * and the return value indicates general success/failure. + */ + public static boolean runTest(String file) { + byte[] inputData; + BenchmarkDataset benchmarkDataset; + try { + inputData = readAllBytes(file); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + } catch (IOException e) { + System.err.println("Unable to get input data"); + return false; + } + + List argsList = getCaliperOption(benchmarkDataset); + if (argsList == null) { + System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); + return false; + } + argsList.add("-DdataFile=" + file); + argsList.add("com.google.protocolbuffers.ProtoBenchCaliper"); + + try { + String args[] = new String[argsList.size()]; + argsList.toArray(args); + CaliperMain.exitlessMain(args, + new PrintWriter(System.out, true), new PrintWriter(System.err, true)); + return true; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + System.err.println("Detailed exception information:"); + e.printStackTrace(System.err); + return false; + } + } + + + private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) { + List temp = new ArrayList(); + if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3"); + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2"); + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2"); + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message3.GoogleMessage3")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3"); + for (String opt : ProtoBenchCaliper + .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) { + temp.add(opt); + } + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message4.GoogleMessage4")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4"); + for (String opt : ProtoBenchCaliper + .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) { + temp.add(opt); + } + } else { + return null; + } + + temp.add("-i"); + temp.add("runtime"); + temp.add("-b"); + String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" + + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; + temp.add(benchmarkNames); + + return temp; + } + + public static byte[] readAllBytes(String filename) throws IOException { + RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); + byte[] content = new byte[(int) file.length()]; + file.readFully(content); + return content; + } +} diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java new file mode 100755 index 00000000..4045b56a --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java @@ -0,0 +1,210 @@ + +package com.google.protocolbuffers; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ProtoBenchCaliper { + public enum BenchmarkMessageType { + GOOGLE_MESSAGE1_PROTO3 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 + .getDefaultInstance(); + } + @Override String[] getSpecificCaliperOption() { return null; } + }, + GOOGLE_MESSAGE1_PROTO2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 + .getDefaultInstance(); + } + @Override String[] getSpecificCaliperOption() { return null; } + }, + GOOGLE_MESSAGE2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); + } + @Override String[] getSpecificCaliperOption() { return null; } + }, + GOOGLE_MESSAGE3 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); + } + @Override + String[] getSpecificCaliperOption() { + String[] opt = new String[1]; + opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms"; + return opt; + } + }, + GOOGLE_MESSAGE4 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); + } + @Override + String[] getSpecificCaliperOption() { + String[] opt = new String[1]; + opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms"; + return opt; + } + }; + + abstract ExtensionRegistry getExtensionRegistry(); + abstract Message getDefaultInstance(); + abstract String[] getSpecificCaliperOption(); + } + + @Param + private BenchmarkMessageType benchmarkMessageType; + @Param + private String dataFile; + + private byte[] inputData; + private BenchmarkDataset benchmarkDataset; + private Message defaultMessage; + private ExtensionRegistry extensions; + private List inputDataList; + private List inputStreamList; + private List inputStringList; + private List sampleMessageList; + private int counter; + private FileOutputStream devNull; + private CodedOutputStream reuseDevNull; + + @BeforeExperiment + void setUp() throws IOException { + inputData = ProtoBench.readAllBytes(dataFile); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + defaultMessage = benchmarkMessageType.getDefaultInstance(); + extensions = benchmarkMessageType.getExtensionRegistry(); + inputDataList = new ArrayList(); + inputStreamList = new ArrayList(); + inputStringList = new ArrayList(); + sampleMessageList = new ArrayList(); + + for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { + byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); + inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); + inputStreamList.add(new ByteArrayInputStream( + benchmarkDataset.getPayload(i).toByteArray())); + inputStringList.add(benchmarkDataset.getPayload(i)); + sampleMessageList.add( + defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); + } + devNull = null; + reuseDevNull = null; + + try { + devNull = new FileOutputStream("/dev/null"); + reuseDevNull = CodedOutputStream.newInstance(devNull); + } catch (FileNotFoundException e) { + // ignore: this is probably Windows, where /dev/null does not exist + } + + counter = 0; + } + + + @Benchmark + void serializeToByteString(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + sampleMessageList.get(counter % sampleMessageList.size()).toByteString(); + counter++; + } + } + + @Benchmark + void serializeToByteArray(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + sampleMessageList.get(counter % sampleMessageList.size()).toByteArray(); + counter++; + } + } + + @Benchmark + void serializeToMemoryStream(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output); + counter++; + } + } + + @Benchmark + void deserializeFromByteString(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions) + .build(); + counter++; + } + } + + @Benchmark + void deserializeFromByteArray(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions) + .build(); + counter++; + } + } + + @Benchmark + void deserializeFromMemoryStream(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions) + .build(); + inputStreamList.get(counter % inputStreamList.size()).reset(); + counter++; + } + } +} -- cgit v1.2.3