process runner for junit integration test framework

Closes #5435.

PiperOrigin-RevId: 202100672

5 files changed, 537 insertions, 0 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 1322ac9838..a5944fac08 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -1544,6 +1544,24 @@ java_test(
     ],
 )
 
+java_test(
+    name = "java-integration-tests",
+    srcs = glob(["integration/blackbox/**/*.java"]),
+    test_class = "com.google.devtools.build.lib.AllTests",
+    deps = [
+        ":guava_junit_truth",
+        ":test_runner",
+        ":testutil",
+        "//src/main/java/com/google/devtools/build/lib:io",
+        "//src/main/java/com/google/devtools/build/lib:os_util",
+        "//src/main/java/com/google/devtools/build/lib:util",
+        "//third_party:auto_value",
+        "//third_party:guava",
+        "//third_party:jsr305",
+        "//third_party:junit4",
+    ],
+)
+
 java_library(
     name = "guava_junit_truth",
     testonly = 1,
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessParameters.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessParameters.java
new file mode 100644
index 0000000000..d9f728aa20
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessParameters.java
@@ -0,0 +1,90 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.integration.blackbox.framework;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.io.File;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/** {@link ProcessRunner} parameters */
+@AutoValue
+public abstract class ProcessParameters {
+  abstract String name();
+
+  abstract ImmutableList<String> arguments();
+
+  abstract File workingDirectory();
+
+  abstract int expectedExitCode();
+
+  abstract boolean expectedEmptyError();
+
+  abstract Optional<ImmutableMap<String, String>> environment();
+
+  abstract long timeoutMillis();
+
+  abstract Optional<Path> redirectOutput();
+
+  abstract Optional<Path> redirectError();
+
+  public static Builder builder() {
+    return new AutoValue_ProcessParameters.Builder()
+        .setExpectedExitCode(0)
+        .setExpectedEmptyError(true)
+        .setTimeoutMillis(30 * 1000)
+        .setArguments();
+  }
+
+  /** Builder class */
+  @AutoValue.Builder
+  public abstract static class Builder {
+    public abstract Builder setName(String value);
+
+    public abstract Builder setArguments(String... args);
+
+    public abstract Builder setArguments(ImmutableList<String> args);
+
+    public Builder setArguments(List<String> args) {
+      setArguments(ImmutableList.copyOf(args));
+      return this;
+    }
+
+    public abstract Builder setWorkingDirectory(File value);
+
+    public abstract Builder setExpectedExitCode(int value);
+
+    public abstract Builder setExpectedEmptyError(boolean value);
+
+    public abstract Builder setEnvironment(ImmutableMap<String, String> map);
+
+    public Builder setEnvironment(Map<String, String> map) {
+      setEnvironment(ImmutableMap.copyOf(map));
+      return this;
+    }
+
+    public abstract Builder setTimeoutMillis(long millis);
+
+    public abstract Builder setRedirectOutput(Path path);
+
+    public abstract Builder setRedirectError(Path path);
+
+    public abstract ProcessParameters build();
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessResult.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessResult.java
new file mode 100644
index 0000000000..9fccf9f616
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessResult.java
@@ -0,0 +1,42 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.integration.blackbox.framework;
+
+import com.google.auto.value.AutoValue;
+import com.google.devtools.build.lib.util.StringUtilities;
+import java.util.List;
+
+/** Result of the external process execution, see {@link ProcessRunner} */
+@AutoValue
+public abstract class ProcessResult {
+
+  static ProcessResult create(int exitCode, List<String> out, List<String> err) {
+    return new AutoValue_ProcessResult(exitCode, out, err);
+  }
+
+  abstract int exitCode();
+
+  abstract List<String> out();
+
+  abstract List<String> err();
+
+  public String outString() {
+    return StringUtilities.joinLines(out());
+  }
+
+  public String errString() {
+    return StringUtilities.joinLines(err());
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunner.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunner.java
new file mode 100644
index 0000000000..2db77170c7
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunner.java
@@ -0,0 +1,194 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.integration.blackbox.framework;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.io.LineReader;
+import com.google.devtools.build.lib.util.StringUtilities;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
+/**
+ * Helper class for running Bazel process as external process from JUnit tests Can be used to run
+ * arbitrary external process and explore the results
+ */
+public final class ProcessRunner {
+  private static final Logger logger = Logger.getLogger(ProcessRunner.class.getName());
+  private final ProcessParameters parameters;
+  private final ExecutorService executorService;
+
+  /**
+   * Creates ProcessRunner
+   *
+   * @param parameters process parameters like executable name, arguments, timeout etc
+   * @param executorService to use for process output/error streams reading; intentionally passed as
+   *     a parameter so we can use the thread pool to speed up. Should be multi-threaded, as two
+   *     separate tasks are submitted, to read from output and error streams.
+   *     <p>SuppressWarnings: WeakerAccess - suppress the warning about constructor being public:
+   *     the class is intended to be used outside the package. (IDE currently marks the possibility
+   *     for the constructor to be package-private because the current usages are only inside the
+   *     package, but it is going to change)
+   */
+  @SuppressWarnings("WeakerAccess")
+  public ProcessRunner(ProcessParameters parameters, ExecutorService executorService) {
+    this.parameters = parameters;
+    this.executorService = executorService;
+  }
+
+  public ProcessResult runSynchronously() throws Exception {
+    ImmutableList<String> args = parameters.arguments();
+    final List<String> commandParts = new ArrayList<>(args.size() + 1);
+    commandParts.add(parameters.name());
+    commandParts.addAll(args);
+
+    logger.info("Running: " + commandParts.stream().collect(Collectors.joining(" ")));
+
+    ProcessBuilder processBuilder = new ProcessBuilder(commandParts);
+    processBuilder.directory(parameters.workingDirectory());
+    parameters.environment().ifPresent(map -> processBuilder.environment().putAll(map));
+
+    parameters.redirectOutput().ifPresent(path -> processBuilder.redirectOutput(path.toFile()));
+    parameters.redirectError().ifPresent(path -> processBuilder.redirectError(path.toFile()));
+
+    Process process = processBuilder.start();
+
+    try (ProcessStreamReader outReader =
+            parameters.redirectOutput().isPresent()
+                ? null
+                : createReader(process.getInputStream(), ">> ");
+        ProcessStreamReader errReader =
+            parameters.redirectError().isPresent()
+                ? null
+                : createReader(process.getErrorStream(), "ERROR: ")) {
+
+      long timeoutMillis = parameters.timeoutMillis();
+      if (!process.waitFor(timeoutMillis, TimeUnit.MILLISECONDS)) {
+        throw new TimeoutException(
+            String.format(
+                "%s timed out after %d seconds (%d millis)",
+                parameters.name(), timeoutMillis / 1000, timeoutMillis));
+      }
+
+      List<String> err =
+          errReader != null
+              ? errReader.get()
+              : Files.readAllLines(parameters.redirectError().get());
+      List<String> out =
+          outReader != null
+              ? outReader.get()
+              : Files.readAllLines(parameters.redirectOutput().get());
+
+      if (parameters.expectedExitCode() != process.exitValue()) {
+        throw new ProcessRunnerException(
+            String.format(
+                "Expected exit code %d, but found %d.\nError: %s\nOutput: %s",
+                parameters.expectedExitCode(),
+                process.exitValue(),
+                StringUtilities.joinLines(err),
+                StringUtilities.joinLines(out)));
+      }
+
+      if (parameters.expectedEmptyError()) {
+        if (!err.isEmpty()) {
+          throw new ProcessRunnerException(
+              "Expected empty error stream, but found: " + StringUtilities.joinLines(err));
+        }
+      }
+      return ProcessResult.create(parameters.expectedExitCode(), out, err);
+    } finally {
+      process.destroy();
+    }
+  }
+
+  private ProcessStreamReader createReader(InputStream stream, String prefix) {
+    return new ProcessStreamReader(executorService, stream, s -> logger.fine(prefix + s));
+  }
+
+  /** Specific runtime exception for external process errors */
+  public static class ProcessRunnerException extends RuntimeException {
+    ProcessRunnerException(String message) {
+      super(message);
+    }
+  }
+
+  private static class ProcessStreamReader implements AutoCloseable {
+
+    private final InputStream stream;
+    private final Future<List<String>> future;
+    private final AtomicReference<IOException> exception = new AtomicReference<>();
+
+    private ProcessStreamReader(
+        ExecutorService executorService,
+        InputStream stream,
+        @Nullable Consumer<String> logConsumer) {
+      this.stream = stream;
+      future =
+          executorService.submit(
+              () -> {
+                final List<String> lines = Lists.newArrayList();
+                try (BufferedReader reader =
+                    new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+                  LineReader lineReader = new LineReader(reader);
+                  String line;
+                  while ((line = lineReader.readLine()) != null) {
+                    if (logConsumer != null) {
+                      logConsumer.accept(line);
+                    }
+                    lines.add(line);
+                  }
+                } catch (IOException e) {
+                  exception.set(e);
+                }
+                return lines;
+              });
+    }
+
+    public List<String> get()
+        throws InterruptedException, ExecutionException, TimeoutException, IOException {
+      try {
+        List<String> lines = future.get(15, TimeUnit.SECONDS);
+        if (exception.get() != null) {
+          throw exception.get();
+        }
+        return lines;
+      } finally {
+        // if future is timed out
+        stream.close();
+      }
+    }
+
+    @Override
+    public void close() throws Exception {
+      stream.close();
+    }
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunnerTest.java b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunnerTest.java
new file mode 100644
index 0000000000..ac38e45a2a
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/integration/blackbox/framework/ProcessRunnerTest.java
@@ -0,0 +1,193 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.integration.blackbox.framework;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.devtools.build.lib.util.OS;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Test of {@link ProcessRunner} */
+@RunWith(JUnit4.class)
+public final class ProcessRunnerTest {
+  private static ExecutorService executorService;
+  private Path directory;
+  private Path path;
+
+  @BeforeClass
+  public static void setUpExecutor() {
+    // we need only two threads to schedule reading from output and error streams
+    executorService =
+        MoreExecutors.getExitingExecutorService(
+            (ThreadPoolExecutor) Executors.newFixedThreadPool(2), 1, TimeUnit.SECONDS);
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    directory = Files.createTempDirectory(getClass().getSimpleName());
+    path = Files.createTempFile(directory, "script", isWindows() ? ".bat" : "");
+    assertThat(Files.exists(path)).isTrue();
+    assertThat(path.toFile().setExecutable(true)).isTrue();
+    path.toFile().deleteOnExit();
+    directory.toFile().deleteOnExit();
+  }
+
+  @AfterClass
+  public static void tearDownExecutor() {
+    MoreExecutors.shutdownAndAwaitTermination(executorService, 5, TimeUnit.SECONDS);
+  }
+
+  @Test
+  public void testSuccess() throws Exception {
+    Files.write(path, createScriptText(/* exit code */ 0, /* output */ "Hello!", /* error */ null));
+
+    ProcessParameters parameters = createBuilder().build();
+    ProcessResult result = new ProcessRunner(parameters, executorService).runSynchronously();
+
+    assertThat(result.exitCode()).isEqualTo(0);
+    assertThat(result.outString()).isEqualTo("Hello!");
+    assertThat(result.errString()).isEmpty();
+  }
+
+  @Test
+  public void testFailure() throws Exception {
+    Files.write(
+        path, createScriptText(/* exit code */ 124, /* output */ null, /* error */ "Failure"));
+
+    ProcessParameters parameters =
+        createBuilder().setExpectedExitCode(124).setExpectedEmptyError(false).build();
+    ProcessResult result = new ProcessRunner(parameters, executorService).runSynchronously();
+
+    assertThat(result.exitCode()).isEqualTo(124);
+    assertThat(result.outString()).isEmpty();
+    assertThat(result.errString()).isEqualTo("Failure");
+  }
+
+  @Test
+  public void testTimeout() throws Exception {
+    // Windows script to sleep 5 seconds, so that we can test timeout.
+    // This script finds PowerShell using %systemroot% variable, which we assume is always
+    // defined. It passes some standard parameters like input and output formats,
+    // important part is the Command parameter, which actually calls Sleep from PowerShell.
+    String windowsScript =
+        "%systemroot%\\system32\\cmd.exe /C \"start /I /B powershell"
+            + " -Version 3.0 -NoLogo -Sta -NoProfile -InputFormat Text -OutputFormat Text"
+            + " -NonInteractive -Command \"\"&PowerShell Sleep 5\"";
+    Files.write(path, Collections.singleton(isWindows() ? windowsScript : "read smthg"));
+
+    ProcessParameters parameters =
+        createBuilder()
+            .setExpectedExitCode(-1)
+            .setExpectedEmptyError(false)
+            .setTimeoutMillis(100)
+            .build();
+    try {
+      new ProcessRunner(parameters, executorService).runSynchronously();
+      assertThat(false).isTrue();
+    } catch (TimeoutException e) {
+      // ignore
+    }
+  }
+
+  @Test
+  public void testRedirect() throws Exception {
+    Files.write(
+        path,
+        createScriptText(
+            /* exit code */ 12,
+            /* output */ Lists.newArrayList("Info", "Multi", "line"),
+            /* error */ Collections.singletonList("Failure")));
+
+    Path out = directory.resolve("out.txt");
+    Path err = directory.resolve("err.txt");
+
+    try {
+      ProcessParameters parameters =
+          createBuilder()
+              .setExpectedExitCode(12)
+              .setExpectedEmptyError(false)
+              .setRedirectOutput(out)
+              .setRedirectError(err)
+              .build();
+      ProcessResult result = new ProcessRunner(parameters, executorService).runSynchronously();
+
+      assertThat(result.exitCode()).isEqualTo(12);
+      assertThat(result.outString()).isEqualTo("Info\nMulti\nline");
+      assertThat(result.errString()).isEqualTo("Failure");
+    } finally {
+      Files.delete(out);
+      Files.delete(err);
+    }
+  }
+
+  private ProcessParameters.Builder createBuilder() {
+    return ProcessParameters.builder()
+        .setWorkingDirectory(directory.toFile())
+        .setName(path.toAbsolutePath().toString());
+  }
+
+  private static List<String> createScriptText(
+      final int exitCode, @Nullable final String output, @Nullable final String error) {
+    return createScriptText(
+        exitCode,
+        output != null ? Collections.singletonList(output) : null,
+        error != null ? Collections.singletonList(error) : null);
+  }
+
+  private static List<String> createScriptText(
+      final int exitCode, @Nullable final List<String> output, @Nullable final List<String> error) {
+    List<String> text = Lists.newArrayList();
+    if (isWindows()) {
+      text.add("@echo off");
+    }
+    text.addAll(echoStrings(output, ""));
+    text.addAll(echoStrings(error, isWindows() ? ">&2" : " 1>&2"));
+    text.add((isWindows() ? "exit /b " : "exit ") + exitCode);
+    return text;
+  }
+
+  private static List<String> echoStrings(@Nullable List<String> input, String redirect) {
+    if (input == null) {
+      return Collections.emptyList();
+    }
+    String quote = isWindows() ? "" : "\"";
+    return input
+        .stream()
+        .map(s -> String.format("echo %s%s%s%s", quote, s, quote, redirect))
+        .collect(Collectors.toList());
+  }
+
+  private static boolean isWindows() {
+    return OS.WINDOWS.equals(OS.getCurrent());
+  }
+}