From 805174eda2356df1b01752c8bc57019e696e0a75 Mon Sep 17 00:00:00 2001 From: Yilun Chong Date: Tue, 10 Apr 2018 13:26:17 -0700 Subject: Add script for run and upload the benchmark result to bq --- .../main/java/com/google/protobuf/ProtoBench.java | 98 ++++------------------ .../com/google/protobuf/ProtoCaliperBenchmark.java | 60 +++++-------- 2 files changed, 40 insertions(+), 118 deletions(-) (limited to 'benchmarks/java') diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java index f35b180a..02503905 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java @@ -69,40 +69,18 @@ public class ProtoBench { System.err.println("input data is in the format of \"benchmarks.proto\""); System.exit(1); } - boolean success = true; - for (int i = 0; i < args.length; i++) { - success &= runTest(args[i]); - } - System.exit(success ? 0 : 1); + + System.exit(runTest(args) ? 0 : 1); } - - /** - * Runs a single test with specific test data. Error messages are displayed to stderr, - * and the return value indicates general success/failure. - */ - public static boolean runTest(String file) { - byte[] inputData; - BenchmarkDataset benchmarkDataset; - try { - inputData = readAllBytes(file); - benchmarkDataset = BenchmarkDataset.parseFrom(inputData); - } catch (IOException e) { - System.err.println("Unable to get input data"); - return false; - } - List argsList = getCaliperOption(benchmarkDataset); - if (argsList == null) { - System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); - return false; - } - argsList.add("-DdataFile=" + file); + public static boolean runTest(String args[]) { + List argsList = getCaliperOption(args); argsList.add("com.google.protobuf.ProtoCaliperBenchmark"); try { - String args[] = new String[argsList.size()]; - argsList.toArray(args); - CaliperMain.exitlessMain(args, + String newArgs[] = new String[argsList.size()]; + argsList.toArray(newArgs); + CaliperMain.exitlessMain(newArgs, new PrintWriter(System.out, true), new PrintWriter(System.err, true)); } catch (Exception e) { System.err.println("Error: " + e.getMessage()); @@ -110,54 +88,22 @@ public class ProtoBench { e.printStackTrace(System.err); return false; } - try { - double mininumScale = 0; - // If the file not exist, this will throw IOException, which won't print the warning - // information below. - Scanner scanner = new Scanner(new String(readAllBytes("JavaBenchmarkWarning.txt"))); - while (scanner.hasNext()) { - mininumScale = Math.max(mininumScale, scanner.nextDouble()); - } - scanner.close(); - - System.out.println( - "WARNING: This benchmark's whole iterations are not enough, consider to config caliper to " - + "run for more time to make the result more convincing. You may change the configure " - + "code in com.google.protobuf.ProtoBench.getCaliperOption() of benchmark " - + benchmarkDataset.getMessageName() - + " to run for more time. e.g. Change the value of " - + "instrument.runtime.options.timingInterval or value of " - + "instrument.runtime.options.measurements to be at least " - + Math.round(mininumScale * 10 + 1) / 10.0 - + " times of before, then build and run the benchmark again\n"); - Files.deleteIfExists(Paths.get("JavaBenchmarkWarning.txt")); - } catch (IOException e) { - // The IOException here should be file not found, which means there's no warning generated by - // The benchmark, so this IOException should be discarded. - } return true; } - - private static List getCaliperOption(final BenchmarkDataset benchmarkDataset) { + private static List getCaliperOption(String args[]) { List temp = new ArrayList(); - if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) { - } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) { - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message3.GoogleMessage3")) { - temp.add("-Cinstrument.runtime.options.timingInterval=3000ms"); - temp.add("-Cinstrument.runtime.options.measurements=20"); - } else if (benchmarkDataset.getMessageName(). - equals("benchmarks.google_message4.GoogleMessage4")) { - temp.add("-Cinstrument.runtime.options.timingInterval=1500ms"); - temp.add("-Cinstrument.runtime.options.measurements=20"); - } else { - return null; - } - temp.add("-i"); temp.add("runtime"); + String files = ""; + for (int i = 0; i < args.length; i++) { + if (args[i].charAt(0) == '-') { + temp.add(args[i]); + } else { + files += (files.equals("") ? "" : ",") + args[i]; + } + } + temp.add("-DdataFile=" + files); temp.add("-b"); String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream" + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream"; @@ -165,14 +111,4 @@ public class ProtoBench { return temp; } - - public static byte[] readAllBytes(String filename) throws IOException { - if (filename.equals("")) { - return new byte[0]; - } - RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); - byte[] content = new byte[(int) file.length()]; - file.readFully(content); - return content; - } } diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java index b50c6585..94568aea 100755 --- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java +++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java @@ -99,7 +99,6 @@ public class ProtoCaliperBenchmark { private List inputStreamList; private List inputStringList; private List sampleMessageList; - private long counter; private BenchmarkMessageType getMessageType() throws IOException { if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) { @@ -149,8 +148,6 @@ public class ProtoCaliperBenchmark { sampleMessageList.add( defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build()); } - - counter = 0; } @@ -160,8 +157,9 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString(); - counter++; + for (int j = 0; j < sampleMessageList.size(); j++) { + sampleMessageList.get(j).toByteString(); + } } } @@ -171,8 +169,9 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray(); - counter++; + for (int j = 0; j < sampleMessageList.size(); j++) { + sampleMessageList.get(j).toByteArray(); + } } } @@ -182,9 +181,10 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output); - counter++; + for (int j = 0; j < sampleMessageList.size(); j++) { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + sampleMessageList.get(j).writeTo(output); + } } } @@ -194,9 +194,10 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStringList.get((int) (counter % inputStringList.size())), extensions); - counter++; + for (int j = 0; j < inputStringList.size(); j++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStringList.get(j), extensions); + } } } @@ -206,9 +207,10 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputDataList.get((int) (counter % inputDataList.size())), extensions); - counter++; + for (int j = 0; j < inputDataList.size(); j++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputDataList.get(j), extensions); + } } } @@ -218,27 +220,11 @@ public class ProtoCaliperBenchmark { return; } for (int i = 0; i < reps; i++) { - benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( - inputStreamList.get((int) (counter % inputStreamList.size())), extensions); - inputStreamList.get((int) (counter % inputStreamList.size())).reset(); - counter++; - } - } - - @AfterExperiment - void checkCounter() throws IOException { - if (counter == 1) { - // Dry run - return; - } - if (benchmarkDataset.getPayloadCount() != 1 - && counter < benchmarkDataset.getPayloadCount() * 10L) { - BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true)); - // If the total number of non-warmup reps is smaller than 100 times of the total number of - // datasets, then output the scale that need to multiply to the configuration (either extend - // the running time for one timingInterval or run for more measurements). - writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " "); - writer.close(); + for (int j = 0; j < inputStreamList.size(); j++) { + benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom( + inputStreamList.get(j), extensions); + inputStreamList.get(j).reset(); + } } } } -- cgit v1.2.3