Benchmark part 2 (preparation for benchmark)

Builders: tools to benchmark the performance.
BuildCases: provide all targets for Builders to build.

--
PiperOrigin-RevId: 146226852
MOS_MIGRATED_REVID=146226852

11 files changed, 761 insertions, 0 deletions

diff --git a/src/tools/benchmark/BUILD b/src/tools/benchmark/BUILD
index c2678f81ce..bf798ab992 100644
--- a/src/tools/benchmark/BUILD
+++ b/src/tools/benchmark/BUILD
@@ -1,7 +1,9 @@
 filegroup(
     name = "srcs",
     srcs = glob(["**"]) + [
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark:srcs",
         "//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:srcs",
+        "//src/tools/benchmark/javatests/com/google/devtools/build/benchmark:srcs",
         "//src/tools/benchmark/javatests/com/google/devtools/build/benchmark/codegenerator:srcs",
     ],
     visibility = ["//src:__pkg__"],
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD
new file mode 100644
index 0000000000..8d433d9cd9
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BUILD
@@ -0,0 +1,28 @@
+package(default_visibility = ["//src/tools/benchmark:__subpackages__"])
+
+load("//tools/build_rules:genproto.bzl", "java_proto_library")
+
+java_proto_library(
+    name = "build_data_proto",
+    src = "build_data.proto",
+)
+
+java_library(
+    name = "benchmark_lib",
+    testonly = 1,
+    srcs = glob(["*.java"]),
+    deps = [
+        ":build_data_proto",
+        "//src/main/java/com/google/devtools/build/lib:shell",
+        "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/common/options",
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark/codegenerator:codegenerator_lib",
+        "//third_party:guava",
+        "//third_party/protobuf",
+    ],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+)
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BazelBuildCase.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BazelBuildCase.java
new file mode 100644
index 0000000000..6ee57967ea
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BazelBuildCase.java
@@ -0,0 +1,124 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.benchmark.codegenerator.JavaCodeGenerator;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.FileSystems;
+import java.io.IOException;
+import java.nio.file.Path;
+
+/** Provides all build target information for Bazel. */
+final class BazelBuildCase implements BuildCase {
+
+  private static final ImmutableMap<String, String> BUILD_TARGET_NAME_TO_DESCRIPTION =
+      ImmutableMap.of(
+          "AFewFiles", "Target: A Few Files",
+          "ManyFiles", "Target: Many Files",
+          "LongChainedDeps", "Target: Long Chained Deps",
+          "ParallelDeps", "Target: Parallel Deps");
+  private static final String WORKSPACE_FILE_NAME = "WORKSPACE";
+  private static final ImmutableList<BuildTargetConfig> defaultBuildTargetConfigs =
+      getDefaultBuildTargetConfigs();
+  private static final boolean INCLUDE_TARGET_A_FEW_FILES = true;
+  private static final boolean INCLUDE_TARGET_MANY_FILES = true;
+  private static final boolean INCLUDE_TARGET_LONG_CHAINED_DEPS = true;
+  private static final boolean INCLUDE_TARGET_PARALLEL_DEPS = true;
+
+  private static final BuildEnvConfig FULL_CLEAN_BUILD_CONFIG =
+      BuildEnvConfig.newBuilder()
+          .setDescription("Full clean build")
+          .setCleanBeforeBuild(true)
+          .setIncremental(false)
+          .build();
+  private static final BuildEnvConfig INCREMENTAL_BUILD_CONFIG =
+      BuildEnvConfig.newBuilder()
+          .setDescription("Incremental build")
+          .setCleanBeforeBuild(false)
+          .setIncremental(true)
+          .build();
+  private static final ImmutableList<BuildEnvConfig> BUILD_ENV_CONFIGS =
+      ImmutableList.of(FULL_CLEAN_BUILD_CONFIG, INCREMENTAL_BUILD_CONFIG);
+
+  private static final FileSystem fileSystem = FileSystems.initDefaultAsJavaIo();
+
+  @Override
+  public ImmutableList<BuildTargetConfig> getBuildTargetConfigs() {
+    return defaultBuildTargetConfigs;
+  }
+
+  @Override
+  public ImmutableList<String> getCodeVersions(Builder builder, String from, String to)
+      throws IOException, CommandException {
+    return builder.getCodeVersionsBetween(from, to);
+  }
+
+  @Override
+  public ImmutableList<BuildEnvConfig> getBuildEnvConfigs() {
+    return BUILD_ENV_CONFIGS;
+  }
+
+  // TODO(yueg): configurable target, we may not want to run benchmark for all kinds of target
+  @Override
+  public void prepareGeneratedCode(Path copyDir, Path generatedCodePath) throws IOException {
+    // Prepare generated code for copy
+    if (!copyDir.toFile().exists()) {
+      JavaCodeGenerator.generateNewProject(
+          copyDir.toString(),
+          INCLUDE_TARGET_A_FEW_FILES,
+          INCLUDE_TARGET_MANY_FILES,
+          INCLUDE_TARGET_LONG_CHAINED_DEPS,
+          INCLUDE_TARGET_PARALLEL_DEPS);
+    }
+
+    // Clean generated code path
+    if (generatedCodePath.toFile().exists()) {
+      try {
+        FileSystemUtils.deleteTreesBelow(fileSystem.getPath(generatedCodePath.toString()));
+      } catch (IOException e) {
+        throw new IOException("Failed to clean directory for generated code", e);
+      }
+    } else {
+      generatedCodePath.toFile().mkdirs();
+    }
+
+    // Copy
+    try {
+      FileSystemUtils.copyTreesBelow(
+          fileSystem.getPath(copyDir.toString()), fileSystem.getPath(generatedCodePath.toString()));
+    } catch (IOException e) {
+      throw new IOException("Failed to copy generated code", e);
+    }
+    if (!generatedCodePath.resolve(WORKSPACE_FILE_NAME).toFile().createNewFile()) {
+      throw new IOException("Failed to create workspace file");
+    }
+  }
+
+  private static ImmutableList<BuildTargetConfig> getDefaultBuildTargetConfigs() {
+    ImmutableList.Builder<BuildTargetConfig> builder = ImmutableList.builder();
+    for (ImmutableMap.Entry<String, String> entry : BUILD_TARGET_NAME_TO_DESCRIPTION.entrySet()) {
+      builder.add(
+          BuildTargetConfig.newBuilder()
+              .setBuildTarget(entry.getKey())
+              .setDescription(entry.getValue())
+              .build());
+    }
+    return builder.build();
+  }
+}
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BazelBuilder.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BazelBuilder.java
new file mode 100644
index 0000000000..a228143d90
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BazelBuilder.java
@@ -0,0 +1,154 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.shell.CommandResult;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** Class that provides all needed feature of Bazel for benchmark. */
+class BazelBuilder implements Builder {
+
+  private static final Logger logger = Logger.getLogger(BazelBuilder.class.getName());
+
+  private static final String BAZEL_BINARY_PATH = "bazel-bin/src/bazel";
+  private static final Pattern ELAPSED_TIME_PATTERN = Pattern.compile("(?<=Elapsed time: )[0-9.]+");
+
+  private final Path generatedCodeDir;
+  private final Path builderDir;
+  private Path buildBinary = null;
+  private String currentCodeVersion = "";
+
+  BazelBuilder(Path generatedCodeDir, Path builderDir) {
+    this.generatedCodeDir = generatedCodeDir;
+    this.builderDir = builderDir;
+  }
+
+  @Override
+  public ImmutableList<String> getCodeVersionsBetween(String from, String to)
+      throws CommandException {
+    String[] gitLogCommand = {"git", "log", from + ".." + to, "--pretty=format:%H", "--reverse"};
+    Command cmd = new Command(gitLogCommand, null, builderDir.toFile());
+    CommandResult result = cmd.execute();
+    String output = new String(result.getStdout(), UTF_8).trim();
+    return ImmutableList.copyOf(output.split("\n"));
+  }
+
+  @Override
+  public Path getBuildBinary(String codeVersion) throws IOException, CommandException {
+    if (buildBinary != null && currentCodeVersion.equals(codeVersion)) {
+      return buildBinary;
+    }
+
+    // git checkout codeVersion
+    String[] checkoutCommand = {"git", "checkout", codeVersion};
+    Command cmd = new Command(checkoutCommand, null, builderDir.toFile());
+    cmd.execute();
+
+    // bazel build src:bazel
+    String[] buildBazelCommand = {"bazel", "build", "src:bazel"};
+    cmd = new Command(buildBazelCommand, null, builderDir.toFile());
+    CommandResult result = cmd.execute();
+
+    // Get binary path, bazel output is in stderr
+    String output = new String(result.getStderr(), UTF_8).trim();
+    if (!output.contains(BAZEL_BINARY_PATH)) {
+      throw new IOException("Bazel binary " + BAZEL_BINARY_PATH + " is not in output of build.");
+    }
+    buildBinary = builderDir.resolve(BAZEL_BINARY_PATH);
+    currentCodeVersion = codeVersion;
+    return buildBinary;
+  }
+
+  @Override
+  public ImmutableList<String> getCommandFromConfig(
+      BuildTargetConfig targetConfig, BuildEnvConfig envConfig) {
+    return ImmutableList.<String>builder()
+        .add("build")
+        .add(targetConfig.getBuildTarget())
+        .addAll(envConfig.getBuildArgsList())
+        .build();
+  }
+
+  @Override
+  public double buildAndGetElapsedTime(Path buildBinary, ImmutableList<String> args)
+      throws CommandException {
+    List<String> cmdList = new ArrayList<>();
+    cmdList.add(buildBinary.toString());
+    cmdList.addAll(args);
+    String[] cmdArr = new String[cmdList.size()];
+    cmdArr = cmdList.toArray(cmdArr);
+
+    // Run build command
+    Command cmd = new Command(cmdArr, null, generatedCodeDir.toFile());
+    CommandResult result = cmd.execute();
+
+    // Get elapsed time from output
+    String output = new String(result.getStderr(), UTF_8).trim();
+    Matcher m = ELAPSED_TIME_PATTERN.matcher(output);
+
+    if (m.find()) {
+      try {
+        return (Double.parseDouble(m.group(0)));
+      } catch (NumberFormatException e) {
+        // Should not be here since we look for [0-9.]+
+        logger.log(Level.SEVERE, "Cannot parse " + m.group(0));
+      }
+    }
+    throw new CommandException(cmd, "Command didn't provide parsable output.");
+  }
+
+  @Override
+  public void clean() throws CommandException {
+    String[] cleanCommand = {"bazel", "clean"};
+    Command cmd = new Command(cleanCommand, null, generatedCodeDir.toFile());
+    cmd.execute();
+  }
+
+  @Override
+  public void prepare() throws IOException, CommandException {
+    if (builderDir.toFile().exists() && !builderDir.toFile().isDirectory()) {
+      try {
+        Files.delete(builderDir);
+      } catch (IOException e) {
+        throw new IOException(builderDir + " is a file and cannot be deleted", e);
+      }
+    }
+    if (Files.notExists(builderDir)) {
+      try {
+        Files.createDirectories(builderDir);
+      } catch (IOException e) {
+        throw new IOException("Failed to create directory for bazel", e);
+      }
+
+      String[] gitCloneCommand = {"git", "clone", "https://github.com/bazelbuild/bazel.git", "."};
+      Command cmd = new Command(gitCloneCommand, null, builderDir.toFile());
+      cmd.execute();
+    }
+    // Assume the directory is what we need if not empty
+  }
+}
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BenchmarkOptions.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BenchmarkOptions.java
new file mode 100644
index 0000000000..54e8d2facb
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BenchmarkOptions.java
@@ -0,0 +1,58 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import com.google.devtools.common.options.Option;
+import com.google.devtools.common.options.OptionsBase;
+
+/** Class that contains arguments for running the benchmark. */
+public class BenchmarkOptions extends OptionsBase {
+
+  @Option(
+    name = "workspace",
+    defaultValue = "",
+    category = "benchmark",
+    valueHelp = "path",
+    help = "Directory where we put all the code and results."
+  )
+  public String workspace;
+
+  @Option(
+    name = "output",
+    defaultValue = "",
+    category = "benchmark",
+    valueHelp = "path",
+    help = "Path to put benchmark result (json format)."
+  )
+  public String output;
+
+  @Option(
+    name = "from",
+    defaultValue = "",
+    category = "benchmark",
+    valueHelp = "string",
+    help = "Use code versions from this (not included)."
+  )
+  public String from;
+
+  @Option(
+    name = "to",
+    defaultValue = "",
+    category = "benchmark",
+    valueHelp = "string",
+    help = "Use code versions to this (included)."
+  )
+  public String to;
+}
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildCase.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildCase.java
new file mode 100644
index 0000000000..f29ce89448
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/BuildCase.java
@@ -0,0 +1,45 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.shell.CommandException;
+import java.io.IOException;
+import java.nio.file.Path;
+
+/** Interface that includes methods for a build case including all build target information. */
+interface BuildCase {
+
+  /** Returns a list of build environment configs. */
+  ImmutableList<BuildEnvConfig> getBuildEnvConfigs();
+
+  /** Returns a list of build target configs. */
+  ImmutableList<BuildTargetConfig> getBuildTargetConfigs();
+
+  /**
+   * Returns a list of code versions (can be anything you specified) in {@code (from, to]} (can be
+   * any interval you specified) of {@code builder}.
+   */
+  ImmutableList<String> getCodeVersions(Builder builder, String from, String to)
+      throws IOException, CommandException;
+
+  /**
+   * Prepares generated code for build.
+   *
+   * @param copyDir the source code path for copy
+   * @param generatedCodePath the path to put generated code
+   */
+  void prepareGeneratedCode(Path copyDir, Path generatedCodePath) throws IOException;
+}
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/Builder.java b/src/tools/benchmark/java/com/google/devtools/build/benchmark/Builder.java
new file mode 100644
index 0000000000..603fd526d1
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/Builder.java
@@ -0,0 +1,48 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.shell.CommandException;
+import java.io.IOException;
+import java.nio.file.Path;
+
+/** Interface that includes methods for a building tool. */
+interface Builder {
+
+  /** Prepare anything the build needs. */
+  void prepare() throws IOException, CommandException;
+
+  /** Returns the binary path of the build tool of a specific {@code codeVersion}. */
+  Path getBuildBinary(String codeVersion) throws IOException, CommandException;
+
+  /** Return a list of code versions in {@code (from, to]}. */
+  ImmutableList<String> getCodeVersionsBetween(String from, String to) throws CommandException;
+
+  /** Returns a command for build under specific config. */
+  ImmutableList<String> getCommandFromConfig(
+      BuildTargetConfig targetConfig, BuildEnvConfig envConfig);
+
+  /**
+   * Build the given buildConfig using the given binary.
+   *
+   * @return elapsed time of the build
+   */
+  double buildAndGetElapsedTime(Path buildBinary, ImmutableList<String> args)
+      throws CommandException;
+
+  /** Clean the previous build results. */
+  void clean() throws CommandException;
+}
diff --git a/src/tools/benchmark/java/com/google/devtools/build/benchmark/build_data.proto b/src/tools/benchmark/java/com/google/devtools/build/benchmark/build_data.proto
new file mode 100644
index 0000000000..f43ecdfea1
--- /dev/null
+++ b/src/tools/benchmark/java/com/google/devtools/build/benchmark/build_data.proto
@@ -0,0 +1,57 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+syntax = "proto3";
+
+package benchmark;
+
+option java_package = "com.google.devtools.build.benchmark";
+option java_multiple_files = true;
+
+// Environment config for a build (how we build a target)
+message BuildEnvConfig {
+  string description = 1;
+  repeated string build_args = 2;
+  bool clean_before_build = 3;
+  bool incremental = 4;
+}
+
+// Build results (elapsed time) for a specific code version
+// results are used for running benchmark for several times for now
+message SingleBuildResult {
+  string code_version = 1;
+  repeated double results = 2;
+}
+
+// Build results for a specific environment config
+message BuildEnvResult {
+  BuildEnvConfig config = 1;
+  repeated SingleBuildResult results = 2;
+}
+
+// Target config for a build (what to build)
+message BuildTargetConfig {
+  string description = 1;
+  string build_target = 2;
+}
+
+// Build results for a specific target config
+message BuildTargetResult {
+  BuildTargetConfig build_target_config = 1;
+  repeated BuildEnvResult build_env_results = 2;
+}
+
+// Build results for a group (environment * target)
+message BuildGroupResult {
+  repeated BuildTargetResult build_target_results = 1;
+}
diff --git a/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD
new file mode 100644
index 0000000000..f2bc4e85b4
--- /dev/null
+++ b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BUILD
@@ -0,0 +1,31 @@
+package(default_visibility = ["//src/tools/benchmark:__subpackages__"])
+
+java_test(
+    name = "BazelBuildCaseTest",
+    srcs = ["BazelBuildCaseTest.java"],
+    deps = [
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark:benchmark_lib",
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark:build_data_proto",
+        "//third_party:guava",
+        "//third_party:junit4",
+        "//third_party:truth",
+    ],
+)
+
+java_test(
+    name = "BazelBuilderTest",
+    srcs = ["BazelBuilderTest.java"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib:shell",
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark:benchmark_lib",
+        "//src/tools/benchmark/java/com/google/devtools/build/benchmark:build_data_proto",
+        "//third_party:guava",
+        "//third_party:junit4",
+        "//third_party:truth",
+    ],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+)
diff --git a/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BazelBuildCaseTest.java b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BazelBuildCaseTest.java
new file mode 100644
index 0000000000..7d4c3d8451
--- /dev/null
+++ b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BazelBuildCaseTest.java
@@ -0,0 +1,87 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableSet;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Path;
+import java.util.Scanner;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BazelBuildCaseTest {
+
+  @Rule public TemporaryFolder folder = new TemporaryFolder();
+
+  @Test
+  public void testPrepareGeneratedCode_Copy() throws IOException {
+    Path root = folder.newFolder("PrepareGeneratedCodeCopy").toPath();
+    // Prepare source
+    Path source = root.resolve("source");
+    source.toFile().mkdir();
+    try (PrintWriter writer = new PrintWriter(source.resolve("file").toFile(), UTF_8.name())) {
+      writer.println("content");
+    }
+    // Prepare destination
+    Path destination = root.resolve("destination");
+    destination.toFile().mkdir();
+
+    new BazelBuildCase().prepareGeneratedCode(source, destination);
+
+    File[] filesList = destination.toFile().listFiles();
+    assertThat(filesList).isNotNull();
+    assertThat(filesList).hasLength(2);
+    assertThat(filesList[0].getName()).isEqualTo("WORKSPACE");
+    assertThat(new Scanner(filesList[1]).useDelimiter("\\Z").next()).isEqualTo("content");
+  }
+
+  @Test
+  public void testPrepareGeneratedCode_Generate() throws IOException {
+    Path root = folder.newFolder("PrepareGeneratedCodeGenerate").toPath();
+    // Prepare source, don't mkdir
+    Path source = root.resolve("source");
+    // Prepare destination
+    Path destination = root.resolve("destination");
+    destination.toFile().mkdir();
+
+    new BazelBuildCase().prepareGeneratedCode(source, destination);
+
+    // Check both source and destination directory include generated code
+    ImmutableSet<String> sourceList = fileArrayToImmutableSet(source.toFile().listFiles());
+    ImmutableSet<String> destinationList =
+        fileArrayToImmutableSet(destination.toFile().listFiles());
+    assertThat(sourceList)
+        .containsExactly("AFewFiles", "LongChainedDeps", "ManyFiles", "ParallelDeps");
+    assertThat(destinationList)
+        .containsExactly("AFewFiles", "LongChainedDeps", "ManyFiles", "ParallelDeps", "WORKSPACE");
+  }
+
+  private static ImmutableSet<String> fileArrayToImmutableSet(File[] files) {
+    ImmutableSet.Builder<String> fileNames = ImmutableSet.builder();
+    for (File file : files) {
+      fileNames.add(file.getName());
+    }
+    return fileNames.build();
+  }
+}
diff --git a/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BazelBuilderTest.java b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BazelBuilderTest.java
new file mode 100644
index 0000000000..f8a1c2003d
--- /dev/null
+++ b/src/tools/benchmark/javatests/com/google/devtools/build/benchmark/BazelBuilderTest.java
@@ -0,0 +1,127 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.benchmark;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.shell.CommandException;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BazelBuilderTest {
+
+  private static final double EPSILON = 1e-4;
+
+  @Rule public TemporaryFolder folder = new TemporaryFolder();
+
+  @Test
+  public void testGetCommandFromConfig() {
+    BuildTargetConfig targetConfig = BuildTargetConfig.newBuilder().setBuildTarget("foo").build();
+    BuildEnvConfig envConfig =
+        BuildEnvConfig.newBuilder().addBuildArgs("apple").addBuildArgs("mango").build();
+
+    ImmutableList<String> command =
+        new BazelBuilder(null, null).getCommandFromConfig(targetConfig, envConfig);
+
+    assertThat(command).containsExactly("build", "foo", "apple", "mango");
+  }
+
+  @Test
+  public void testBuildAndGetElapsedTime() throws IOException, CommandException {
+    Path root = folder.newFolder("BuildAndGetElapsedTime").toPath();
+    Path generatedCode = root.resolve("GeneratedCode");
+    Files.createDirectories(generatedCode);
+    // Prepare binary
+    Path buildBinary = root.resolve("binary.sh");
+    Files.createFile(buildBinary);
+    if (!buildBinary.toFile().setExecutable(true)) {
+      fail("Failed to set executable");
+    }
+    double expectedValue = 10.42;
+    try (PrintWriter writer = new PrintWriter(buildBinary.toFile())) {
+      writer.format(
+          "#!/bin/bash\n>&2 echo 'blah blah Elapsed time: %.2f blah blah'", expectedValue);
+    }
+
+    double result = -1;
+    try {
+      result =
+          new BazelBuilder(generatedCode, null)
+              .buildAndGetElapsedTime(buildBinary, ImmutableList.<String>of());
+    } catch (IOException e) {
+      fail("Should not fail to build and get elapsed time.");
+    }
+
+    assertThat(result).isWithin(EPSILON).of(expectedValue);
+  }
+
+  @Test
+  public void testClean() throws IOException, CommandException {
+    // Prepare workspace
+    Path root = folder.newFolder("Clean").toPath();
+    Path generatedCode = root.resolve("GeneratedCode");
+    Files.createDirectories(generatedCode);
+    Files.createDirectories(generatedCode.resolve("bazel-bin"));
+    Files.createDirectories(generatedCode.resolve("bazel-out"));
+    Files.createFile(generatedCode.resolve("BUILD"));
+    Files.createFile(generatedCode.resolve("WORKSPACE"));
+
+    BazelBuilder builder = new BazelBuilder(generatedCode, root.resolve("Builder"));
+    try {
+      builder.clean();
+    } catch (IOException e) {
+      fail("Should not throw IOException in clean()");
+    }
+
+    ImmutableSet<String> fileList = fileArrayToImmutableSet(generatedCode.toFile().listFiles());
+    assertThat(fileList).containsExactly("BUILD", "WORKSPACE");
+  }
+
+  @Test
+  public void testPrepare() throws IOException, CommandException {
+    Path root = folder.newFolder("Prepare").toPath();
+
+    BazelBuilder builder = new BazelBuilder(root.resolve("GeneratedCode"), root.resolve("Builder"));
+    try {
+      builder.prepare();
+    } catch (IOException e) {
+      fail("Should not fail to prepare builder");
+    }
+
+    ImmutableSet<String> fileList =
+        fileArrayToImmutableSet(root.resolve("Builder").toFile().listFiles());
+    assertThat(fileList).containsAllOf("BUILD", "WORKSPACE");
+  }
+
+  private static ImmutableSet<String> fileArrayToImmutableSet(File[] files) {
+    ImmutableSet.Builder<String> fileNames = ImmutableSet.builder();
+    for (File file : files) {
+      fileNames.add(file.getName());
+    }
+    return fileNames.build();
+  }
+}