Add script for run and upload the benchmark result to bq

2 files changed, 40 insertions, 118 deletions

diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
index f35b180a..02503905 100755
--- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
+++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java
@@ -69,40 +69,18 @@ public class ProtoBench {
       System.err.println("input data is in the format of \"benchmarks.proto\"");
       System.exit(1);
     }
-    boolean success = true;
-    for (int i = 0; i < args.length; i++) {
-      success &= runTest(args[i]);
-    }
-    System.exit(success ? 0 : 1);
+    
+    System.exit(runTest(args) ? 0 : 1);
   }
   
- 
-  /**
-   * Runs a single test with specific test data. Error messages are displayed to stderr, 
-   * and the return value indicates general success/failure.
-   */
-  public static boolean runTest(String file) {
-    byte[] inputData;
-    BenchmarkDataset benchmarkDataset;
-    try {
-      inputData = readAllBytes(file);
-      benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
-    } catch (IOException e) {
-      System.err.println("Unable to get input data");
-      return false;
-    }
-    List<String> argsList = getCaliperOption(benchmarkDataset);
-    if (argsList == null) {
-      System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
-      return false;
-    }
-    argsList.add("-DdataFile=" + file);
+  public static boolean runTest(String args[]) {
+    List<String> argsList = getCaliperOption(args);
     argsList.add("com.google.protobuf.ProtoCaliperBenchmark");
     
     try {
-      String args[] = new String[argsList.size()];
-      argsList.toArray(args);
-      CaliperMain.exitlessMain(args,
+      String newArgs[] = new String[argsList.size()];
+      argsList.toArray(newArgs);
+      CaliperMain.exitlessMain(newArgs,
           new PrintWriter(System.out, true), new PrintWriter(System.err, true));
     } catch (Exception e) {
       System.err.println("Error: " + e.getMessage());
@@ -110,54 +88,22 @@ public class ProtoBench {
       e.printStackTrace(System.err);
       return false;
     }
-    try {
-      double mininumScale = 0;
-      // If the file not exist, this will throw IOException, which won't print the warning 
-      // information below.
-      Scanner scanner = new Scanner(new String(readAllBytes("JavaBenchmarkWarning.txt")));
-      while (scanner.hasNext()) {
-        mininumScale = Math.max(mininumScale, scanner.nextDouble());
-      }
-      scanner.close();
-      
-      System.out.println(
-         "WARNING: This benchmark's whole iterations are not enough, consider to config caliper to "
-        + "run for more time to make the result more convincing. You may change the configure "
-        + "code in com.google.protobuf.ProtoBench.getCaliperOption() of benchmark " 
-        + benchmarkDataset.getMessageName()
-        + " to run for more time. e.g. Change the value of "
-        + "instrument.runtime.options.timingInterval or value of "
-        + "instrument.runtime.options.measurements to be at least "
-        + Math.round(mininumScale * 10 + 1) / 10.0 
-        + " times of before, then build and run the benchmark again\n");
-      Files.deleteIfExists(Paths.get("JavaBenchmarkWarning.txt"));
-    } catch (IOException e) {
-      // The IOException here should be file not found, which means there's no warning generated by
-      // The benchmark, so this IOException should be discarded.
-    }
     return true;
   }
   
-  
-  private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDataset) {
+  private static List<String> getCaliperOption(String args[]) {
     List<String> temp = new ArrayList<String>();
-    if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
-    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
-    } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
-    } else if (benchmarkDataset.getMessageName().
-        equals("benchmarks.google_message3.GoogleMessage3")) {
-      temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
-      temp.add("-Cinstrument.runtime.options.measurements=20");
-    } else if (benchmarkDataset.getMessageName().
-        equals("benchmarks.google_message4.GoogleMessage4")) {
-      temp.add("-Cinstrument.runtime.options.timingInterval=1500ms");
-      temp.add("-Cinstrument.runtime.options.measurements=20");
-    } else {
-      return null;
-    }
-    
     temp.add("-i");
     temp.add("runtime");
+    String files = "";
+    for (int i = 0; i < args.length; i++) {
+      if (args[i].charAt(0) == '-') {
+        temp.add(args[i]);
+      } else {
+        files += (files.equals("") ? "" : ",") + args[i];
+      }
+    }
+    temp.add("-DdataFile=" + files);
     temp.add("-b");
     String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
        + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
@@ -165,14 +111,4 @@ public class ProtoBench {
     
     return temp;
   }
-  
-  public static byte[] readAllBytes(String filename) throws IOException {
-    if (filename.equals("")) {
-      return new byte[0];
-    }
-    RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
-    byte[] content = new byte[(int) file.length()];
-    file.readFully(content);
-    return content;
-  }
 }
diff --git a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
index b50c6585..94568aea 100755
--- a/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
+++ b/benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
@@ -99,7 +99,6 @@ public class ProtoCaliperBenchmark {
   private List<ByteArrayInputStream> inputStreamList;
   private List<ByteString> inputStringList;
   private List<Message> sampleMessageList;
-  private long counter;
 
   private BenchmarkMessageType getMessageType() throws IOException {
     if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
@@ -149,8 +148,6 @@ public class ProtoCaliperBenchmark {
       sampleMessageList.add(
           defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
     }
-    
-    counter = 0;
   }
   
   
@@ -160,8 +157,9 @@ public class ProtoCaliperBenchmark {
       return;
     }
     for (int i = 0; i < reps; i++) {
-      sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteString();
-      counter++;
+      for (int j = 0; j < sampleMessageList.size(); j++) {
+        sampleMessageList.get(j).toByteString();        
+      }
     }
   }
   
@@ -171,8 +169,9 @@ public class ProtoCaliperBenchmark {
       return;
     }
     for (int i = 0; i < reps; i++) {
-      sampleMessageList.get((int) (counter % sampleMessageList.size())).toByteArray();
-      counter++;
+      for (int j = 0; j < sampleMessageList.size(); j++) {
+        sampleMessageList.get(j).toByteArray();  
+      }
     }
   }
   
@@ -182,9 +181,10 @@ public class ProtoCaliperBenchmark {
       return;
     }
     for (int i = 0; i < reps; i++) {
-      ByteArrayOutputStream output = new ByteArrayOutputStream();
-      sampleMessageList.get((int) (counter % sampleMessageList.size())).writeTo(output);
-      counter++;
+      for (int j = 0; j < sampleMessageList.size(); j++) {
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        sampleMessageList.get(j).writeTo(output); 
+      }
     }
   }
   
@@ -194,9 +194,10 @@ public class ProtoCaliperBenchmark {
       return;
     }
     for (int i = 0; i < reps; i++) {
-      benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
-          inputStringList.get((int) (counter % inputStringList.size())), extensions);
-      counter++;
+      for (int j = 0; j < inputStringList.size(); j++) {
+        benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
+          inputStringList.get(j), extensions);
+      }
     }
   }
   
@@ -206,9 +207,10 @@ public class ProtoCaliperBenchmark {
       return;
     }
     for (int i = 0; i < reps; i++) {
-      benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
-          inputDataList.get((int) (counter % inputDataList.size())), extensions);
-      counter++;
+      for (int j = 0; j < inputDataList.size(); j++) {
+        benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
+          inputDataList.get(j), extensions);
+      }
     }
   }
   
@@ -218,27 +220,11 @@ public class ProtoCaliperBenchmark {
       return;
     }
     for (int i = 0; i < reps; i++) {
-      benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
-          inputStreamList.get((int) (counter % inputStreamList.size())), extensions);
-      inputStreamList.get((int) (counter % inputStreamList.size())).reset();
-      counter++;
-    }
-  }
-  
-  @AfterExperiment
-  void checkCounter() throws IOException {
-    if (counter == 1) {
-      // Dry run
-      return;
-    }
-    if (benchmarkDataset.getPayloadCount() != 1 
-        && counter < benchmarkDataset.getPayloadCount() * 10L) {
-      BufferedWriter writer = new BufferedWriter(new FileWriter("JavaBenchmarkWarning.txt", true));
-      // If the total number of non-warmup reps is smaller than 100 times of the total number of 
-      // datasets, then output the scale that need to multiply to the configuration (either extend
-      // the running time for one timingInterval or run for more measurements). 
-      writer.append(1.0 * benchmarkDataset.getPayloadCount() * 10L / counter + " ");
-      writer.close();
+      for (int j = 0; j < inputStreamList.size(); j++) {
+        benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
+            inputStreamList.get(j), extensions);
+        inputStreamList.get(j).reset();
+      }
     }
   }
 }