From 6cd4ec497239070c1513c9dd05fc88cd3ecc57fd Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Fri, 9 Feb 2018 11:47:00 -0800 Subject: Sync internal benchmark changes --- .../main/java/com/google/protobuf/ProtoBench.java | 10 +- .../com/google/protobuf/ProtoBenchCaliper.java | 197 ----------------- .../com/google/protobuf/ProtoCaliperBenchmark.java | 245 +++++++++++++++++++++ 3 files changed, 249 insertions(+), 203 deletions(-) delete mode 100755 benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java create mode 100755 benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java (limited to 'benchmarks/java') diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java index f8d3b797..f35b180a 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java @@ -97,7 +97,7 @@ public class ProtoBench { return false; } argsList.add("-DdataFile=" + file); - argsList.add("com.google.protobuf.ProtoBenchCaliper"); + argsList.add("com.google.protobuf.ProtoCaliperBenchmark"); try { String args[] = new String[argsList.size()]; @@ -142,19 +142,14 @@ public class ProtoBench { private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) { List temp = new ArrayList(); if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3"); } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2"); } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2"); } else if (benchmarkDataset.getMessageName(). equals("benchmarks.google_message3.GoogleMessage3")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3"); temp.add("-Cinstrument.runtime.options.timingInterval=3000ms"); temp.add("-Cinstrument.runtime.options.measurements=20"); } else if (benchmarkDataset.getMessageName(). equals("benchmarks.google_message4.GoogleMessage4")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4"); temp.add("-Cinstrument.runtime.options.timingInterval=1500ms"); temp.add("-Cinstrument.runtime.options.measurements=20"); } else { @@ -172,6 +167,9 @@ public class ProtoBench { } public static byte[] readAllBytes(String filename) throws IOException { + if (filename.equals("")) { + return new byte[0]; + } RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); byte[] content = new byte[(int) file.length()]; file.readFully(content); diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java deleted file mode 100755 index a7027586..00000000 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java +++ /dev/null @@ -1,197 +0,0 @@ - -package com.google.protobuf; - -import com.google.caliper.BeforeExperiment; -import com.google.caliper.AfterExperiment; -import com.google.caliper.Benchmark; -import com.google.caliper.Param; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.Message; -import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.BufferedWriter; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class ProtoBenchCaliper { - public enum BenchmarkMessageType { - GOOGLE_MESSAGE1_PROTO3 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } - @Override - Message getDefaultInstance() { - return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 - .getDefaultInstance(); - } - }, - GOOGLE_MESSAGE1_PROTO2 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } - @Override - Message getDefaultInstance() { - return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 - .getDefaultInstance(); - } - }, - GOOGLE_MESSAGE2 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } - @Override - Message getDefaultInstance() { - return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); - } - }, - GOOGLE_MESSAGE3 { - @Override - ExtensionRegistry getExtensionRegistry() { - ExtensionRegistry extensions = ExtensionRegistry.newInstance(); - benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions); - return extensions; - } - @Override - Message getDefaultInstance() { - return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); - } - }, - GOOGLE_MESSAGE4 { - @Override - ExtensionRegistry getExtensionRegistry() { - ExtensionRegistry extensions = ExtensionRegistry.newInstance(); - benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions); - return extensions; - } - @Override - Message getDefaultInstance() { - return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); - } - }; - - abstract ExtensionRegistry getExtensionRegistry(); - abstract Message getDefaultInstance(); - } - - @Param - private BenchmarkMessageType benchmarkMessageType; - @Param - private String dataFile; - - private byte[] inputData; - private BenchmarkDataset benchmarkDataset; - private Message defaultMessage; - private ExtensionRegistry extensions; - private List inputDataList; - private List inputStreamList; - private List inputStringList; - private List sampleMessageList; - private long counter; - - @BeforeExperiment - void setUp() throws IOException { - inputData = ProtoBench.readAllBytes(dataFile); - benchmarkDataset = BenchmarkDataset.parseFrom(inputData); - defaultMessage = benchmarkMessageType.getDefaultInstance(); - extensions = benchmarkMessageType.getExtensionRegistry(); - inputDataList = new ArrayList(); - inputStreamList = new ArrayList(); - inputStringList = new ArrayList(); - sampleMessageList = new ArrayList(); - - for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { - byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); - inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); - inputStreamList.add(new ByteArrayInputStream( - benchmarkDataset.getPayload(i).toByteArray())); - inputStringList.add(benchmarkDataset.getPayload(i)); - sampleMessageList.add( - defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); - } - - counter = 0; - } - - - @Benchmark - void serializeToByteString(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString(); - counter++; - } - } - - @Benchmark - void serializeToByteArray(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray(); - counter++; - } - } - - @Benchmark - void serializeToMemoryStream(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output); - counter++; - } - } - - @Benchmark - void deserializeFromByteString(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStringList.get((int) (counter % inputStringList.size())), extensions); - counter++; - } - } - - @Benchmark - void deserializeFromByteArray(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputDataList.get((int) (counter % inputDataList.size())), extensions); - counter++; - } - } - - @Benchmark - void deserializeFromMemoryStream(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStreamList.get((int) (counter % inputStreamList.size())), extensions); - inputStreamList.get((int) (counter % inputStreamList.size())).reset(); - counter++; - } - } - - @AfterExperiment - void checkCounter() throws IOException { - if (counter == 1) { - // Dry run - return; - } - if (benchmarkDataset.getPayloadCount() != 1 - && counter < benchmarkDataset.getPayloadCount() * 10L) { - BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true)); - // If the total number of non-warmup reps is smaller than 100 times of the total number of - // datasets, then output the scale that need to multiply to the configuration (either extend - // the running time for one timingInterval or run for more measurements). - writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " "); - writer.close(); - } - } -} diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java new file mode 100755 index 00000000..7dff16dc --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -0,0 +1,245 @@ + +package com.google.protobuf; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.AfterExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; + +public class ProtoCaliperBenchmark { + public enum BenchmarkMessageType { + GOOGLE_MESSAGE1_PROTO3 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 + .getDefaultInstance(); + } + }, + GOOGLE_MESSAGE1_PROTO2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 + .getDefaultInstance(); + } + }, + GOOGLE_MESSAGE2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); + } + }, + GOOGLE_MESSAGE3 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + com.google.protobuf.benchmarks.BenchmarkMessage38.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage37.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage36.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage35.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage34.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage33.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage32.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage31.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage3.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); + } + }, + GOOGLE_MESSAGE4 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + com.google.protobuf.benchmarks.BenchmarkMessage43.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage42.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage41.registerAllExtensions(extensions); + com.google.protobuf.benchmarks.BenchmarkMessage4.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); + } + }; + + abstract ExtensionRegistry getExtensionRegistry(); + abstract Message getDefaultInstance(); + } + + private BenchmarkMessageType benchmarkMessageType; + @Param("") + private String dataFile; + + private byte[] inputData; + private BenchmarkDataset benchmarkDataset; + private Message defaultMessage; + private ExtensionRegistry extensions; + private List inputDataList; + private List inputStreamList; + private List inputStringList; + private List sampleMessageList; + private long counter; + + private BenchmarkMessageType getMessageType() throws IOException { + if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { + return BenchmarkMessageType.GOOGLE_MESSAGE1_PROTO3; + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { + return BenchmarkMessageType.GOOGLE_MESSAGE1_PROTO2; + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { + return BenchmarkMessageType.GOOGLE_MESSAGE2; + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message3.GoogleMessage3")) { + return BenchmarkMessageType.GOOGLE_MESSAGE3; + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message4.GoogleMessage4")) { + return BenchmarkMessageType.GOOGLE_MESSAGE4; + } else { + throw new IllegalStateException("Invalid DataFile! There's no testing message named " + + benchmarkDataset.getMessageName()); + } + } + + @BeforeExperiment + void setUp() throws IOException { + if (!dataFile.equals("")) { + RandomAccessFile file = new RandomAccessFile(new File(dataFile), "r"); + inputData = new byte[(int) file.length()]; + file.readFully(inputData); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + benchmarkMessageType = getMessageType(); + } else { + inputData = new byte[0]; + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + benchmarkMessageType = BenchmarkMessageType.GOOGLE_MESSAGE2; + } + defaultMessage = benchmarkMessageType.getDefaultInstance(); + extensions = benchmarkMessageType.getExtensionRegistry(); + inputDataList = new ArrayList(); + inputStreamList = new ArrayList(); + inputStringList = new ArrayList(); + sampleMessageList = new ArrayList(); + + for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { + byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); + inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); + inputStreamList.add(new ByteArrayInputStream( + benchmarkDataset.getPayload(i).toByteArray())); + inputStringList.add(benchmarkDataset.getPayload(i)); + sampleMessageList.add( + defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); + } + + counter = 0; + } + + + @Benchmark + void serializeToByteString(int reps) throws IOException { + if (sampleMessageList.size() == 0) { + return; + } + for (int i = 0; i < reps; i++) { + sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString(); + counter++; + } + } + + @Benchmark + void serializeToByteArray(int reps) throws IOException { + if (sampleMessageList.size() == 0) { + return; + } + for (int i = 0; i < reps; i++) { + sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray(); + counter++; + } + } + + @Benchmark + void serializeToMemoryStream(int reps) throws IOException { + if (sampleMessageList.size() == 0) { + return; + } + for (int i = 0; i < reps; i++) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output); + counter++; + } + } + + @Benchmark + void deserializeFromByteString(int reps) throws IOException { + if (inputStringList.size() == 0) { + return; + } + for (int i = 0; i < reps; i++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStringList.get((int) (counter % inputStringList.size())), extensions); + counter++; + } + } + + @Benchmark + void deserializeFromByteArray(int reps) throws IOException { + if (inputDataList.size() == 0) { + return; + } + for (int i = 0; i < reps; i++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputDataList.get((int) (counter % inputDataList.size())), extensions); + counter++; + } + } + + @Benchmark + void deserializeFromMemoryStream(int reps) throws IOException { + if (inputStreamList.size() == 0) { + return; + } + for (int i = 0; i < reps; i++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStreamList.get((int) (counter % inputStreamList.size())), extensions); + inputStreamList.get((int) (counter % inputStreamList.size())).reset(); + counter++; + } + } + + @AfterExperiment + void checkCounter() throws IOException { + if (counter == 1) { + // Dry run + return; + } + if (benchmarkDataset.getPayloadCount() != 1 + && counter < benchmarkDataset.getPayloadCount() * 10L) { + BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true)); + // If the total number of non-warmup reps is smaller than 100 times of the total number of + // datasets, then output the scale that need to multiply to the configuration (either extend + // the running time for one timingInterval or run for more measurements). + writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " "); + writer.close(); + } + } +} + -- cgit v1.2.3