From 34843eddfe65686a86d12780d1ca709a997d83ad Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Wed, 13 Dec 2017 14:34:52 -0800 Subject: Fix bugs --- benchmarks/Makefile.am | 2 +- benchmarks/README.md | 32 +++- benchmarks/initialize_submodule.sh | 7 - benchmarks/java/pom.xml | 150 +++++---------- .../main/java/com/google/protobuf/ProtoBench.java | 148 +++++++++++++++ .../com/google/protobuf/ProtoBenchCaliper.java | 183 ++++++++++++++++++ .../com/google/protocolbuffers/ProtoBench.java | 155 --------------- .../google/protocolbuffers/ProtoBenchCaliper.java | 210 --------------------- tests.sh | 5 +- 9 files changed, 408 insertions(+), 484 deletions(-) delete mode 100755 benchmarks/initialize_submodule.sh create mode 100755 benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java create mode 100755 benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java delete mode 100755 benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java delete mode 100755 benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java diff --git a/benchmarks/Makefile.am b/benchmarks/Makefile.am index e69d9fb7..aa65c02f 100755 --- a/benchmarks/Makefile.am +++ b/benchmarks/Makefile.am @@ -136,7 +136,7 @@ javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middlem java-benchmark: javac_middleman @echo "Writing shortcut script java-benchmark..." @echo '#! /bin/sh' > java-benchmark - @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark + @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark @chmod +x java-benchmark java: protoc_middleman protoc_middleman2 java-benchmark diff --git a/benchmarks/README.md b/benchmarks/README.md index 00a63704..f266d41e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -5,20 +5,31 @@ This directory contains benchmarking schemas and data sets that you can use to test a variety of performance scenarios against your protobuf language runtime. -The schema for the datasets is described in `benchmarks.proto`. +## Benchmark tools and build instructions -The benchmark is based on some submodules. To initialize the submodues: +First, you need to follow the instruction in the root directory's README to +build your language's protobuf, then: -For java: -``` -$ ./initialize_submodule.sh java -``` +### CPP +We are using [google/benchmark](https://github.com/google/benchmark) as the +benchmark tool for testing cpp. This is included as submodule under third_party +directory. To init and build this tools, you need to do this under root dirctory: -For java: ``` -$ ./initialize_submodule.sh cpp +$ cd third_party +$ git submodule update --init -r +$ cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../.. ``` +### JAVA +We're using maven to build the java benchmarks, which is the same as to build +the Java protobuf. There're no other tools need to install. We're using +[google/caliper](https://github.com/google/caliper) as benchmark tool, which +can be automaticly included by maven. + + +## Run instructions + To run all the benchmark dataset: For java: @@ -49,6 +60,8 @@ $ make cpp-benchmark $ ./cpp-benchmark $(specific generated dataset file name) ``` +## Benchmark datasets + There's some big testing data not included in the directory initially, you need to run the following command to download the testing data: @@ -57,10 +70,13 @@ $ ./download_data.sh ``` Each data set is in the format of benchmarks.proto: + 1. name is the benchmark dataset's name. 2. message_name is the benchmark's message type full name (including package and message name) 3. payload is the list of raw data. +The schema for the datasets is described in `benchmarks.proto`. + Benchmark likely want to run several benchmarks against each data set (parse, serialize, possibly JSON, possibly using different APIs, etc). diff --git a/benchmarks/initialize_submodule.sh b/benchmarks/initialize_submodule.sh deleted file mode 100755 index f3456adf..00000000 --- a/benchmarks/initialize_submodule.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /bin/sh - -oldpwd=`pwd` -cd "../third_party" -git submodule update --init -r -cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd .. -cd "$oldpwd" diff --git a/benchmarks/java/pom.xml b/benchmarks/java/pom.xml index e19f3f59..3afeebca 100755 --- a/benchmarks/java/pom.xml +++ b/benchmarks/java/pom.xml @@ -1,85 +1,59 @@ - - - - - 4.0.0 - + + + 4.0.0 protobuf-java-benchmark - com.google.protocolbuffers + com.google.protobuf 1.0.0 Protocol Buffers [Benchmark] - - The benchmark tools for Protobuf Java. - - + The benchmark tools for Protobuf Java. + - com.google.protobuf - protobuf-java - 3.5.0 - - - com.google.caliper - caliper - 1.0-beta-2 - + com.google.protobuf + protobuf-java + 3.5.0 + + + com.google.caliper + caliper + 1.0-beta-2 + - + - - - org.apache.maven.plugins - maven-assembly-plugin - 2.4.1 - - - - jar-with-dependencies - - - - - com.mkyong.core.utils.App - - - - - - - make-assembly - - package - - single - - - - - + + + org.apache.maven.plugins + maven-assembly-plugin + 2.4.1 + + + + jar-with-dependencies + + + + + com.mkyong.core.utils.App + + + + + + make-assembly + + package + + single + + + + + org.apache.maven.plugins maven-compiler-plugin 3.5.1 @@ -114,35 +88,7 @@ - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - package - - shade - - - true - all - - - - com.google.monitoring.runtime.instrumentation.AllocationInstrumenter - true - true - - - - - - - - - - + \ No newline at end of file diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java new file mode 100755 index 00000000..abd9200e --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java @@ -0,0 +1,148 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2009 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +package com.google.protobuf; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.caliper.runner.CaliperMain; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedInputStream; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.List; + + +public class ProtoBench { + + private ProtoBench() { + // Prevent instantiation + } + + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Usage: ./java-benchmark "); + System.err.println("input data is in the format of \"benchmarks.proto\""); + System.exit(1); + } + boolean success = true; + for (int i = 0; i < args.length; i++) { + success &= runTest(args[i]); + } + System.exit(success ? 0 : 1); + } + + /** + * Runs a single test with specific test data. Error messages are displayed to stderr, + * and the return value indicates general success/failure. + */ + public static boolean runTest(String file) { + byte[] inputData; + BenchmarkDataset benchmarkDataset; + try { + inputData = readAllBytes(file); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + } catch (IOException e) { + System.err.println("Unable to get input data"); + return false; + } + + List argsList = getCaliperOption(benchmarkDataset); + if (argsList == null) { + System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); + return false; + } + argsList.add("-DdataFile=" + file); + argsList.add("com.google.protobuf.ProtoBenchCaliper"); + + try { + String args[] = new String[argsList.size()]; + argsList.toArray(args); + CaliperMain.exitlessMain(args, + new PrintWriter(System.out, true), new PrintWriter(System.err, true)); + return true; + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + System.err.println("Detailed exception information:"); + e.printStackTrace(System.err); + return false; + } + } + + + private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) { + List temp = new ArrayList(); + if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3"); + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2"); + } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2"); + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message3.GoogleMessage3")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3"); + } else if (benchmarkDataset.getMessageName(). + equals("benchmarks.google_message4.GoogleMessage4")) { + temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4"); + } else { + return null; + } + + temp.add("-i"); + temp.add("runtime"); + temp.add("-b"); + String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" + + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; + temp.add(benchmarkNames); + temp.add("-Cinstrument.runtime.options.timingInterval=3000ms"); + + return temp; + } + + public static byte[] readAllBytes(String filename) throws IOException { + RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); + byte[] content = new byte[(int) file.length()]; + file.readFully(content); + return content; + } +} diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java new file mode 100755 index 00000000..546c25a1 --- /dev/null +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java @@ -0,0 +1,183 @@ + +package com.google.protobuf; + +import com.google.caliper.BeforeExperiment; +import com.google.caliper.Benchmark; +import com.google.caliper.Param; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ExtensionRegistry; +import com.google.protobuf.Message; +import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ProtoBenchCaliper { + public enum BenchmarkMessageType { + GOOGLE_MESSAGE1_PROTO3 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 + .getDefaultInstance(); + } + }, + GOOGLE_MESSAGE1_PROTO2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 + .getDefaultInstance(); + } + }, + GOOGLE_MESSAGE2 { + @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } + @Override + Message getDefaultInstance() { + return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); + } + }, + GOOGLE_MESSAGE3 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions); + benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); + } + }, + GOOGLE_MESSAGE4 { + @Override + ExtensionRegistry getExtensionRegistry() { + ExtensionRegistry extensions = ExtensionRegistry.newInstance(); + benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions); + benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions); + return extensions; + } + @Override + Message getDefaultInstance() { + return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); + } + }; + + abstract ExtensionRegistry getExtensionRegistry(); + abstract Message getDefaultInstance(); + } + + @Param + private BenchmarkMessageType benchmarkMessageType; + @Param + private String dataFile; + + private byte[] inputData; + private BenchmarkDataset benchmarkDataset; + private Message defaultMessage; + private ExtensionRegistry extensions; + private List inputDataList; + private List inputStreamList; + private List inputStringList; + private List sampleMessageList; + private int counter; + + @BeforeExperiment + void setUp() throws IOException { + inputData = ProtoBench.readAllBytes(dataFile); + benchmarkDataset = BenchmarkDataset.parseFrom(inputData); + defaultMessage = benchmarkMessageType.getDefaultInstance(); + extensions = benchmarkMessageType.getExtensionRegistry(); + inputDataList = new ArrayList(); + inputStreamList = new ArrayList(); + inputStringList = new ArrayList(); + sampleMessageList = new ArrayList(); + + for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { + byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); + inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); + inputStreamList.add(new ByteArrayInputStream( + benchmarkDataset.getPayload(i).toByteArray())); + inputStringList.add(benchmarkDataset.getPayload(i)); + sampleMessageList.add( + defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); + } + + counter = 0; + } + + + @Benchmark + void serializeToByteString(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + sampleMessageList.get(counter % sampleMessageList.size()).toByteString(); + counter++; + } + } + + @Benchmark + void serializeToByteArray(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + sampleMessageList.get(counter % sampleMessageList.size()).toByteArray(); + counter++; + } + } + + @Benchmark + void serializeToMemoryStream(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output); + counter++; + } + } + + @Benchmark + void deserializeFromByteString(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions) + .build(); + counter++; + } + } + + @Benchmark + void deserializeFromByteArray(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions) + .build(); + counter++; + } + } + + @Benchmark + void deserializeFromMemoryStream(int reps) throws IOException { + for (int i = 0; i < reps; i++) { + defaultMessage + .newBuilderForType() + .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions) + .build(); + inputStreamList.get(counter % inputStreamList.size()).reset(); + counter++; + } + } +} diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java deleted file mode 100755 index 72f2d629..00000000 --- a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBench.java +++ /dev/null @@ -1,155 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2009 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -package com.google.protocolbuffers; - -import com.google.caliper.BeforeExperiment; -import com.google.caliper.Benchmark; -import com.google.caliper.Param; -import com.google.caliper.runner.CaliperMain; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedInputStream; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.Message; -import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.List; - - -public class ProtoBench { - - private ProtoBench() { - // Prevent instantiation - } - - public static void main(String[] args) { - if (args.length < 1) { - System.err.println("Usage: ./java-benchmark "); - System.err.println("input data is in the format of \"benchmarks.proto\""); - System.exit(1); - } - boolean success = true; - for (int i = 0; i < args.length; i++) { - success &= runTest(args[i]); - } - System.exit(success ? 0 : 1); - } - - /** - * Runs a single test with specific test data. Error messages are displayed to stderr, - * and the return value indicates general success/failure. - */ - public static boolean runTest(String file) { - byte[] inputData; - BenchmarkDataset benchmarkDataset; - try { - inputData = readAllBytes(file); - benchmarkDataset = BenchmarkDataset.parseFrom(inputData); - } catch (IOException e) { - System.err.println("Unable to get input data"); - return false; - } - - List argsList = getCaliperOption(benchmarkDataset); - if (argsList == null) { - System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); - return false; - } - argsList.add("-DdataFile=" + file); - argsList.add("com.google.protocolbuffers.ProtoBenchCaliper"); - - try { - String args[] = new String[argsList.size()]; - argsList.toArray(args); - CaliperMain.exitlessMain(args, - new PrintWriter(System.out, true), new PrintWriter(System.err, true)); - return true; - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - System.err.println("Detailed exception information:"); - e.printStackTrace(System.err); - return false; - } - } - - - private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) { - List temp = new ArrayList(); - if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3"); - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2"); - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2"); - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message3.GoogleMessage3")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3"); - for (String opt : ProtoBenchCaliper - .BenchmarkMessageType.GOOGLE_MESSAGE3.getSpecificCaliperOption()) { - temp.add(opt); - } - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message4.GoogleMessage4")) { - temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4"); - for (String opt : ProtoBenchCaliper - .BenchmarkMessageType.GOOGLE_MESSAGE4.getSpecificCaliperOption()) { - temp.add(opt); - } - } else { - return null; - } - - temp.add("-i"); - temp.add("runtime"); - temp.add("-b"); - String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" - + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; - temp.add(benchmarkNames); - - return temp; - } - - public static byte[] readAllBytes(String filename) throws IOException { - RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); - byte[] content = new byte[(int) file.length()]; - file.readFully(content); - return content; - } -} diff --git a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java b/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java deleted file mode 100755 index 4045b56a..00000000 --- a/benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java +++ /dev/null @@ -1,210 +0,0 @@ - -package com.google.protocolbuffers; - -import com.google.caliper.BeforeExperiment; -import com.google.caliper.Benchmark; -import com.google.caliper.Param; -import com.google.protobuf.ByteString; -import com.google.protobuf.CodedOutputStream; -import com.google.protobuf.ExtensionRegistry; -import com.google.protobuf.Message; -import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class ProtoBenchCaliper { - public enum BenchmarkMessageType { - GOOGLE_MESSAGE1_PROTO3 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } - @Override - Message getDefaultInstance() { - return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1 - .getDefaultInstance(); - } - @Override String[] getSpecificCaliperOption() { return null; } - }, - GOOGLE_MESSAGE1_PROTO2 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } - @Override - Message getDefaultInstance() { - return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1 - .getDefaultInstance(); - } - @Override String[] getSpecificCaliperOption() { return null; } - }, - GOOGLE_MESSAGE2 { - @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); } - @Override - Message getDefaultInstance() { - return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance(); - } - @Override String[] getSpecificCaliperOption() { return null; } - }, - GOOGLE_MESSAGE3 { - @Override - ExtensionRegistry getExtensionRegistry() { - ExtensionRegistry extensions = ExtensionRegistry.newInstance(); - benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions); - benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions); - return extensions; - } - @Override - Message getDefaultInstance() { - return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance(); - } - @Override - String[] getSpecificCaliperOption() { - String[] opt = new String[1]; - opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms"; - return opt; - } - }, - GOOGLE_MESSAGE4 { - @Override - ExtensionRegistry getExtensionRegistry() { - ExtensionRegistry extensions = ExtensionRegistry.newInstance(); - benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions); - benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions); - return extensions; - } - @Override - Message getDefaultInstance() { - return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance(); - } - @Override - String[] getSpecificCaliperOption() { - String[] opt = new String[1]; - opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms"; - return opt; - } - }; - - abstract ExtensionRegistry getExtensionRegistry(); - abstract Message getDefaultInstance(); - abstract String[] getSpecificCaliperOption(); - } - - @Param - private BenchmarkMessageType benchmarkMessageType; - @Param - private String dataFile; - - private byte[] inputData; - private BenchmarkDataset benchmarkDataset; - private Message defaultMessage; - private ExtensionRegistry extensions; - private List inputDataList; - private List inputStreamList; - private List inputStringList; - private List sampleMessageList; - private int counter; - private FileOutputStream devNull; - private CodedOutputStream reuseDevNull; - - @BeforeExperiment - void setUp() throws IOException { - inputData = ProtoBench.readAllBytes(dataFile); - benchmarkDataset = BenchmarkDataset.parseFrom(inputData); - defaultMessage = benchmarkMessageType.getDefaultInstance(); - extensions = benchmarkMessageType.getExtensionRegistry(); - inputDataList = new ArrayList(); - inputStreamList = new ArrayList(); - inputStringList = new ArrayList(); - sampleMessageList = new ArrayList(); - - for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) { - byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray(); - inputDataList.add(benchmarkDataset.getPayload(i).toByteArray()); - inputStreamList.add(new ByteArrayInputStream( - benchmarkDataset.getPayload(i).toByteArray())); - inputStringList.add(benchmarkDataset.getPayload(i)); - sampleMessageList.add( - defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); - } - devNull = null; - reuseDevNull = null; - - try { - devNull = new FileOutputStream("/dev/null"); - reuseDevNull = CodedOutputStream.newInstance(devNull); - } catch (FileNotFoundException e) { - // ignore: this is probably Windows, where /dev/null does not exist - } - - counter = 0; - } - - - @Benchmark - void serializeToByteString(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - sampleMessageList.get(counter % sampleMessageList.size()).toByteString(); - counter++; - } - } - - @Benchmark - void serializeToByteArray(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - sampleMessageList.get(counter % sampleMessageList.size()).toByteArray(); - counter++; - } - } - - @Benchmark - void serializeToMemoryStream(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output); - counter++; - } - } - - @Benchmark - void deserializeFromByteString(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions) - .build(); - counter++; - } - } - - @Benchmark - void deserializeFromByteArray(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions) - .build(); - counter++; - } - } - - @Benchmark - void deserializeFromMemoryStream(int reps) throws IOException { - for (int i = 0; i < reps; i++) { - defaultMessage - .newBuilderForType() - .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions) - .build(); - inputStreamList.get(counter % inputStreamList.size()).reset(); - counter++; - } - } -} diff --git a/tests.sh b/tests.sh index 2a0d203b..0a91e4ce 100755 --- a/tests.sh +++ b/tests.sh @@ -44,7 +44,10 @@ build_cpp() { # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996 if [[ $(type cmake 2>/dev/null) ]]; then # Verify benchmarking code can build successfully. - cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd .. + git submodule init + git submodule update + cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../.. + cd benchmarks && make cpp-benchmark && cd .. else echo "" echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed." -- cgit v1.2.3