Enable local action execution statistics collection for sandboxed actions that use either the LinuxSandboxedSpawnRunner or the ProcessWrapperSandboxedSpawnRunner.

In particular, record metrics for user and system CPU execution time, block I/O and involuntary context switches.

This feature is guarded behind a new option, --experimental_collect_local_sandbox_action_metrics.

Note: We still need to enable execution statistics for the DarwinSandboxedSpawnRunner in a later change.

RELNOTES: None.
PiperOrigin-RevId: 179976217

7 files changed, 303 insertions, 83 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
index 957449eb2b..c9329c2ce5 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
@@ -32,6 +32,7 @@ import com.google.devtools.build.lib.shell.AbnormalTerminationException;
 import com.google.devtools.build.lib.shell.Command;
 import com.google.devtools.build.lib.shell.CommandException;
 import com.google.devtools.build.lib.shell.CommandResult;
+import com.google.devtools.build.lib.shell.ExecutionStatistics;
 import com.google.devtools.build.lib.util.CommandFailureUtils;
 import com.google.devtools.build.lib.util.io.OutErr;
 import com.google.devtools.build.lib.vfs.FileSystem;
@@ -40,6 +41,7 @@ import com.google.devtools.build.lib.vfs.PathFragment;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Map;
+import java.util.Optional;
 
 /** Abstract common ancestor for sandbox spawn runners implementing the common parts. */
 abstract class AbstractSandboxSpawnRunner implements SpawnRunner {
@@ -88,14 +90,16 @@ abstract class AbstractSandboxSpawnRunner implements SpawnRunner {
       SpawnExecutionPolicy policy,
       Path execRoot,
       Path tmpDir,
-      Duration timeout)
+      Duration timeout,
+      Optional<String> statisticsPath)
       throws IOException, InterruptedException {
     try {
       sandbox.createFileSystem();
       OutErr outErr = policy.getFileOutErr();
       policy.prefetchInputs();
 
-      SpawnResult result = run(originalSpawn, sandbox, outErr, timeout, execRoot, tmpDir);
+      SpawnResult result =
+          run(originalSpawn, sandbox, outErr, timeout, execRoot, tmpDir, statisticsPath);
 
       policy.lockOutputFiles();
       try {
@@ -118,7 +122,8 @@ abstract class AbstractSandboxSpawnRunner implements SpawnRunner {
       OutErr outErr,
       Duration timeout,
       Path execRoot,
-      Path tmpDir)
+      Path tmpDir,
+      Optional<String> statisticsPath)
       throws IOException, InterruptedException {
     Command cmd = new Command(
         sandbox.getArguments().toArray(new String[0]),
@@ -177,14 +182,30 @@ abstract class AbstractSandboxSpawnRunner implements SpawnRunner {
         wasTimeout
             ? Status.TIMEOUT
             : (exitCode == 0) ? Status.SUCCESS : Status.NON_ZERO_EXIT;
-    return new SpawnResult.Builder()
-        .setStatus(status)
-        .setExitCode(exitCode)
-        .setWallTime(wallTime)
-        .setUserTime(commandResult.getUserExecutionTime())
-        .setSystemTime(commandResult.getSystemExecutionTime())
-        .setFailureMessage(status != Status.SUCCESS || exitCode != 0 ? failureMessage : "")
-        .build();
+
+    SpawnResult.Builder spawnResultBuilder =
+        new SpawnResult.Builder()
+            .setStatus(status)
+            .setExitCode(exitCode)
+            .setWallTime(wallTime)
+            .setFailureMessage(status != Status.SUCCESS || exitCode != 0 ? failureMessage : "");
+
+    if (statisticsPath.isPresent()) {
+      Optional<ExecutionStatistics.ResourceUsage> resourceUsage =
+          ExecutionStatistics.getResourceUsage(statisticsPath.get());
+      if (resourceUsage.isPresent()) {
+        spawnResultBuilder.setUserTime(resourceUsage.get().getUserExecutionTime());
+        spawnResultBuilder.setSystemTime(resourceUsage.get().getSystemExecutionTime());
+        spawnResultBuilder.setNumBlockOutputOperations(
+            resourceUsage.get().getBlockOutputOperations());
+        spawnResultBuilder.setNumBlockInputOperations(
+            resourceUsage.get().getBlockInputOperations());
+        spawnResultBuilder.setNumInvoluntaryContextSwitches(
+            resourceUsage.get().getInvoluntaryContextSwitches());
+      }
+    }
+
+    return spawnResultBuilder.build();
   }
 
   private boolean wasTimeout(Duration timeout, Duration wallTime) {
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java
index f33a8f08da..0e6db5dd92 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java
@@ -46,6 +46,7 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 /** Spawn runner that uses Darwin (macOS) sandboxing to execute a process. */
@@ -87,7 +88,7 @@ final class DarwinSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
   private final boolean allowNetwork;
   private final String productName;
   private final Path processWrapper;
-  private final int timeoutGraceSeconds;
+  private final Optional<Duration> timeoutKillDelay;
 
   /**
    * The set of directories that always should be writable, independent of the Spawn itself.
@@ -97,11 +98,50 @@ final class DarwinSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
   private final ImmutableSet<Path> alwaysWritableDirs;
   private final LocalEnvProvider localEnvProvider;
 
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code process-wrapper} tool and the MacOS
+   * {@code sandbox-exec} binary. If a spawn exceeds its timeout, then it will be killed instantly.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   */
+  DarwinSandboxedSpawnRunner(CommandEnvironment cmdEnv, Path sandboxBase, String productName)
+      throws IOException {
+    this(cmdEnv, sandboxBase, productName, Optional.empty());
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code process-wrapper} tool and the MacOS
+   * {@code sandbox-exec} binary. If a spawn exceeds its timeout, then it will be killed after the
+   * specified delay.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param timeoutKillDelay an additional grace period before killing timing out commands
+   */
+  DarwinSandboxedSpawnRunner(
+      CommandEnvironment cmdEnv, Path sandboxBase, String productName, Duration timeoutKillDelay)
+      throws IOException {
+    this(cmdEnv, sandboxBase, productName, Optional.of(timeoutKillDelay));
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code process-wrapper} tool and the MacOS
+   * {@code sandbox-exec} binary.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param timeoutKillDelay an optional, additional grace period before killing timing out
+   *     commands. If not present, then no grace period is used and commands are killed instantly.
+   */
   DarwinSandboxedSpawnRunner(
       CommandEnvironment cmdEnv,
       Path sandboxBase,
       String productName,
-      int timeoutGraceSeconds)
+      Optional<Duration> timeoutKillDelay)
       throws IOException {
     super(cmdEnv, sandboxBase);
     this.execRoot = cmdEnv.getExecRoot();
@@ -110,7 +150,7 @@ final class DarwinSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
     this.alwaysWritableDirs = getAlwaysWritableDirs(cmdEnv.getRuntime().getFileSystem());
     this.processWrapper = ProcessWrapperUtil.getProcessWrapper(cmdEnv);
     this.localEnvProvider = new XCodeLocalEnvProvider();
-    this.timeoutGraceSeconds = timeoutGraceSeconds;
+    this.timeoutKillDelay = timeoutKillDelay;
   }
 
   private static void addPathToSetIfExists(FileSystem fs, Set<Path> paths, String path)
@@ -193,8 +233,11 @@ final class DarwinSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
 
     final Path sandboxConfigPath = sandboxPath.getRelative("sandbox.sb");
     Duration timeout = policy.getTimeout();
-    List<String> arguments =
-        computeCommandLine(spawn, timeout, sandboxConfigPath, timeoutGraceSeconds);
+    List<String> arguments = computeCommandLine(spawn, timeout, sandboxConfigPath);
+
+    // TODO(b/62588075): Add execution statistics support for the DarwinSandboxedSpawnRunner.
+    Optional<String> statisticsPath = Optional.empty();
+
     Map<String, String> environment =
         localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, tmpDir, productName);
 
@@ -214,20 +257,21 @@ final class DarwinSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
             sandboxConfigPath, writableDirs, getInaccessiblePaths(), allowNetworkForThisSpawn);
       }
     };
-    return runSpawn(spawn, sandbox, policy, execRoot, tmpDir, timeout);
+    return runSpawn(spawn, sandbox, policy, execRoot, tmpDir, timeout, statisticsPath);
   }
 
-  private List<String> computeCommandLine(
-      Spawn spawn, Duration timeout, Path sandboxConfigPath, int timeoutGraceSeconds) {
+  private List<String> computeCommandLine(Spawn spawn, Duration timeout, Path sandboxConfigPath) {
     List<String> commandLineArgs = new ArrayList<>();
     commandLineArgs.add(SANDBOX_EXEC);
     commandLineArgs.add("-f");
     commandLineArgs.add(sandboxConfigPath.getPathString());
-    commandLineArgs.addAll(
+    ProcessWrapperUtil.CommandLineBuilder processWrapperCommandLineBuilder =
         ProcessWrapperUtil.commandLineBuilder(processWrapper.getPathString(), spawn.getArguments())
-            .setTimeout(timeout)
-            .setKillDelay(Duration.ofSeconds(timeoutGraceSeconds))
-            .build());
+            .setTimeout(timeout);
+    if (timeoutKillDelay.isPresent()) {
+      processWrapperCommandLineBuilder.setKillDelay(timeoutKillDelay.get());
+    }
+    commandLineArgs.addAll(processWrapperCommandLineBuilder.build());
     return commandLineArgs;
   }
 
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
index 674966dde3..4c727c0e69 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
@@ -41,6 +41,7 @@ import java.io.IOException;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.SortedMap;
 
@@ -82,16 +83,79 @@ final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
   private final Path inaccessibleHelperFile;
   private final Path inaccessibleHelperDir;
   private final LocalEnvProvider localEnvProvider;
-  private final int timeoutGraceSeconds;
+  private final Optional<Duration> timeoutKillDelay;
   private final String productName;
 
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool. If a spawn exceeds
+   * its timeout, then it will be killed instantly.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param inaccessibleHelperFile path to a file that is (already) inaccessible
+   * @param inaccessibleHelperDir path to a directory that is (already) inaccessible
+   */
+  LinuxSandboxedSpawnRunner(
+      CommandEnvironment cmdEnv,
+      Path sandboxBase,
+      String productName,
+      Path inaccessibleHelperFile,
+      Path inaccessibleHelperDir) {
+    this(
+        cmdEnv,
+        sandboxBase,
+        productName,
+        inaccessibleHelperFile,
+        inaccessibleHelperDir,
+        Optional.empty());
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool. If a spawn exceeds
+   * its timeout, then it will be killed after the specified delay.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param inaccessibleHelperFile path to a file that is (already) inaccessible
+   * @param inaccessibleHelperDir path to a directory that is (already) inaccessible
+   * @param timeoutKillDelay an additional grace period before killing timing out commands
+   */
   LinuxSandboxedSpawnRunner(
       CommandEnvironment cmdEnv,
       Path sandboxBase,
       String productName,
       Path inaccessibleHelperFile,
       Path inaccessibleHelperDir,
-      int timeoutGraceSeconds) {
+      Duration timeoutKillDelay) {
+    this(
+        cmdEnv,
+        sandboxBase,
+        productName,
+        inaccessibleHelperFile,
+        inaccessibleHelperDir,
+        Optional.of(timeoutKillDelay));
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param inaccessibleHelperFile path to a file that is (already) inaccessible
+   * @param inaccessibleHelperDir path to a directory that is (already) inaccessible
+   * @param timeoutKillDelay an optional, additional grace period before killing timing out
+   *     commands. If not present, then no grace period is used and commands are killed instantly.
+   */
+  LinuxSandboxedSpawnRunner(
+      CommandEnvironment cmdEnv,
+      Path sandboxBase,
+      String productName,
+      Path inaccessibleHelperFile,
+      Path inaccessibleHelperDir,
+      Optional<Duration> timeoutKillDelay) {
     super(cmdEnv, sandboxBase);
     this.fileSystem = cmdEnv.getRuntime().getFileSystem();
     this.blazeDirs = cmdEnv.getDirectories();
@@ -101,7 +165,7 @@ final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
     this.linuxSandbox = LinuxSandboxUtil.getLinuxSandbox(cmdEnv);
     this.inaccessibleHelperFile = inaccessibleHelperFile;
     this.inaccessibleHelperDir = inaccessibleHelperDir;
-    this.timeoutGraceSeconds = timeoutGraceSeconds;
+    this.timeoutKillDelay = timeoutKillDelay;
     this.localEnvProvider = LocalEnvProvider.ADD_TEMP_POSIX;
   }
 
@@ -119,60 +183,48 @@ final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
     Set<Path> writableDirs = getWritableDirs(sandboxExecRoot, spawn.getEnvironment(), tmpDir);
     ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn);
     Duration timeout = policy.getTimeout();
-    List<String> arguments =
-        computeCommandLine(
-            spawn,
-            timeout,
-            linuxSandbox,
-            writableDirs,
-            getTmpfsPaths(),
-            getReadOnlyBindMounts(blazeDirs, sandboxExecRoot),
-            allowNetwork || Spawns.requiresNetwork(spawn),
-            spawn.getExecutionInfo().containsKey(ExecutionRequirements.REQUIRES_FAKEROOT));
-    Map<String, String> environment =
-        localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, tmpDir, productName);
 
-    SandboxedSpawn sandbox =
-        new SymlinkedSandboxedSpawn(
-            sandboxPath,
-            sandboxExecRoot,
-            arguments,
-            environment,
-            SandboxHelpers.getInputFiles(spawn, policy, execRoot),
-            outputs,
-            writableDirs);
-    return runSpawn(spawn, sandbox, policy, execRoot, tmpDir, timeout);
-  }
-
-  private List<String> computeCommandLine(
-      Spawn spawn,
-      Duration timeout,
-      Path linuxSandbox,
-      Set<Path> writableDirs,
-      Set<Path> tmpfsPaths,
-      Map<Path, Path> bindMounts,
-      boolean allowNetwork,
-      boolean requiresFakeRoot) {
     LinuxSandboxUtil.CommandLineBuilder commandLineBuilder =
         LinuxSandboxUtil.commandLineBuilder(linuxSandbox.getPathString(), spawn.getArguments())
             .setWritableFilesAndDirectories(writableDirs)
-            .setTmpfsDirectories(tmpfsPaths)
-            .setBindMounts(bindMounts)
+            .setTmpfsDirectories(getTmpfsPaths())
+            .setBindMounts(getReadOnlyBindMounts(blazeDirs, sandboxExecRoot))
             .setUseFakeHostname(getSandboxOptions().sandboxFakeHostname)
-            .setCreateNetworkNamespace(!allowNetwork)
+            .setCreateNetworkNamespace(!(allowNetwork || Spawns.requiresNetwork(spawn)))
             .setUseDebugMode(getSandboxOptions().sandboxDebug);
+
     if (!timeout.isZero()) {
       commandLineBuilder.setTimeout(timeout);
     }
-    if (timeoutGraceSeconds != -1) {
-      commandLineBuilder.setKillDelay(Duration.ofSeconds(timeoutGraceSeconds));
+    if (timeoutKillDelay.isPresent()) {
+      commandLineBuilder.setKillDelay(timeoutKillDelay.get());
     }
-    if (requiresFakeRoot) {
+    if (spawn.getExecutionInfo().containsKey(ExecutionRequirements.REQUIRES_FAKEROOT)) {
       commandLineBuilder.setUseFakeRoot(true);
     } else if (getSandboxOptions().sandboxFakeUsername) {
       commandLineBuilder.setUseFakeUsername(true);
     }
-    return commandLineBuilder.build();
+
+    Optional<String> statisticsPath = Optional.empty();
+    if (getSandboxOptions().collectLocalSandboxExecutionStatistics) {
+      statisticsPath = Optional.of(sandboxPath.getRelative("stats.out").getPathString());
+      commandLineBuilder.setStatisticsPath(statisticsPath.get());
+    }
+
+    Map<String, String> environment =
+        localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, tmpDir, productName);
+
+    SandboxedSpawn sandbox =
+        new SymlinkedSandboxedSpawn(
+            sandboxPath,
+            sandboxExecRoot,
+            commandLineBuilder.build(),
+            environment,
+            SandboxHelpers.getInputFiles(spawn, policy, execRoot),
+            outputs,
+            writableDirs);
+
+    return runSpawn(spawn, sandbox, policy, execRoot, tmpDir, timeout, statisticsPath);
   }
 
   @Override
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
index 7a844317fb..526f155f31 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
@@ -22,6 +22,8 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import java.io.IOException;
+import java.time.Duration;
+import java.util.Optional;
 
 /** Strategy that uses sandboxing to execute a process. */
 // TODO(ulfjack): This class only exists for this annotation. Find a better way to handle this!
@@ -39,8 +41,48 @@ public final class LinuxSandboxedStrategy extends AbstractSpawnStrategy {
     return "sandboxed";
   }
 
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool. If a spawn exceeds
+   * its timeout, then it will be killed instantly.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   */
   static LinuxSandboxedSpawnRunner create(
-      CommandEnvironment cmdEnv, Path sandboxBase, String productName, int timeoutGraceSeconds)
+      CommandEnvironment cmdEnv, Path sandboxBase, String productName) throws IOException {
+    return create(cmdEnv, sandboxBase, productName, Optional.empty());
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool. If a spawn exceeds
+   * its timeout, then it will be killed after the specified delay.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param timeoutKillDelay an additional grace period before killing timing out commands
+   */
+  static LinuxSandboxedSpawnRunner create(
+      CommandEnvironment cmdEnv, Path sandboxBase, String productName, Duration timeoutKillDelay)
+      throws IOException {
+    return create(cmdEnv, sandboxBase, productName, Optional.of(timeoutKillDelay));
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param timeoutKillDelay an optional, additional grace period before killing timing out
+   *     commands. If not present, then no grace period is used and commands are killed instantly.
+   */
+  static LinuxSandboxedSpawnRunner create(
+      CommandEnvironment cmdEnv,
+      Path sandboxBase,
+      String productName,
+      Optional<Duration> timeoutKillDelay)
       throws IOException {
     Path inaccessibleHelperFile = sandboxBase.getRelative("inaccessibleHelperFile");
     FileSystemUtils.touchFile(inaccessibleHelperFile);
@@ -60,6 +102,6 @@ public final class LinuxSandboxedStrategy extends AbstractSpawnStrategy {
         productName,
         inaccessibleHelperFile,
         inaccessibleHelperDir,
-        timeoutGraceSeconds);
+        timeoutKillDelay);
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java
index 418d5162c5..832760332b 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedSpawnRunner.java
@@ -25,8 +25,8 @@ import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.vfs.Path;
 import java.io.IOException;
 import java.time.Duration;
-import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 /** Strategy that uses sandboxing to execute a process. */
 final class ProcessWrapperSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
@@ -39,17 +39,53 @@ final class ProcessWrapperSandboxedSpawnRunner extends AbstractSandboxSpawnRunne
   private final String productName;
   private final Path processWrapper;
   private final LocalEnvProvider localEnvProvider;
-  private final int timeoutGraceSeconds;
+  private final Optional<Duration> timeoutKillDelay;
 
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code process-wrapper} tool. If a spawn exceeds
+   * its timeout, then it will be killed instantly.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   */
+  ProcessWrapperSandboxedSpawnRunner(
+      CommandEnvironment cmdEnv, Path sandboxBase, String productName) {
+    this(cmdEnv, sandboxBase, productName, Optional.empty());
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code process-wrapper} tool. If a spawn exceeds
+   * its timeout, then it will be killed after the specified delay.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param timeoutKillDelay an additional grace period before killing timing out commands
+   */
+  ProcessWrapperSandboxedSpawnRunner(
+      CommandEnvironment cmdEnv, Path sandboxBase, String productName, Duration timeoutKillDelay) {
+    this(cmdEnv, sandboxBase, productName, Optional.of(timeoutKillDelay));
+  }
+
+  /**
+   * Creates a sandboxed spawn runner that uses the {@code process-wrapper} tool.
+   *
+   * @param cmdEnv the command environment to use
+   * @param sandboxBase path to the sandbox base directory
+   * @param productName the product name to use
+   * @param timeoutKillDelay an optional, additional grace period before killing timing out
+   *     commands. If not present, then no grace period is used and commands are killed instantly.
+   */
   ProcessWrapperSandboxedSpawnRunner(
       CommandEnvironment cmdEnv,
       Path sandboxBase,
       String productName,
-      int timeoutGraceSeconds) {
+      Optional<Duration> timeoutKillDelay) {
     super(cmdEnv, sandboxBase);
     this.execRoot = cmdEnv.getExecRoot();
     this.productName = productName;
-    this.timeoutGraceSeconds = timeoutGraceSeconds;
+    this.timeoutKillDelay = timeoutKillDelay;
     this.processWrapper = ProcessWrapperUtil.getProcessWrapper(cmdEnv);
     this.localEnvProvider =
         OS.getCurrent() == OS.DARWIN
@@ -69,11 +105,20 @@ final class ProcessWrapperSandboxedSpawnRunner extends AbstractSandboxSpawnRunne
     Path tmpDir = sandboxExecRoot.getRelative("tmp");
 
     Duration timeout = policy.getTimeout();
-    List<String> arguments =
+    ProcessWrapperUtil.CommandLineBuilder commandLineBuilder =
         ProcessWrapperUtil.commandLineBuilder(processWrapper.getPathString(), spawn.getArguments())
-            .setTimeout(timeout)
-            .setKillDelay(Duration.ofSeconds(timeoutGraceSeconds))
-            .build();
+            .setTimeout(timeout);
+
+    if (timeoutKillDelay.isPresent()) {
+      commandLineBuilder.setKillDelay(timeoutKillDelay.get());
+    }
+
+    Optional<String> statisticsPath = Optional.empty();
+    if (getSandboxOptions().collectLocalSandboxExecutionStatistics) {
+      statisticsPath = Optional.of(sandboxPath.getRelative("stats.out").getPathString());
+      commandLineBuilder.setStatisticsPath(statisticsPath.get());
+    }
+
     Map<String, String> environment =
         localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, tmpDir, productName);
 
@@ -81,13 +126,13 @@ final class ProcessWrapperSandboxedSpawnRunner extends AbstractSandboxSpawnRunne
         new SymlinkedSandboxedSpawn(
             sandboxPath,
             sandboxExecRoot,
-            arguments,
+            commandLineBuilder.build(),
             environment,
             SandboxHelpers.getInputFiles(spawn, policy, execRoot),
             SandboxHelpers.getOutputFiles(spawn),
             getWritableDirs(sandboxExecRoot, spawn.getEnvironment(), tmpDir));
 
-    return runSpawn(spawn, sandbox, policy, execRoot, tmpDir, timeout);
+    return runSpawn(spawn, sandbox, policy, execRoot, tmpDir, timeout, statisticsPath);
   }
 
   @Override
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java
index ee496e8e29..a520552b52 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java
@@ -32,6 +32,8 @@ import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.common.options.OptionsProvider;
 import java.io.IOException;
+import java.time.Duration;
+import java.util.Optional;
 
 /**
  * Provides the sandboxed spawn strategy.
@@ -48,8 +50,12 @@ final class SandboxActionContextProvider extends ActionContextProvider {
     ImmutableList.Builder<ActionContext> contexts = ImmutableList.builder();
 
     OptionsProvider options = cmdEnv.getOptions();
-    int timeoutGraceSeconds =
+    int timeoutKillDelaySeconds =
         options.getOptions(LocalExecutionOptions.class).localSigkillGraceSeconds;
+    Optional<Duration> timeoutKillDelay = Optional.empty();
+    if (timeoutKillDelaySeconds >= 0) {
+      timeoutKillDelay = Optional.of(Duration.ofSeconds(timeoutKillDelaySeconds));
+    }
     String productName = cmdEnv.getRuntime().getProductName();
 
     // This works on most platforms, but isn't the best choice, so we put it first and let later
@@ -59,7 +65,7 @@ final class SandboxActionContextProvider extends ActionContextProvider {
           withFallback(
               cmdEnv,
               new ProcessWrapperSandboxedSpawnRunner(
-                  cmdEnv, sandboxBase, productName, timeoutGraceSeconds));
+                  cmdEnv, sandboxBase, productName, timeoutKillDelay));
       contexts.add(new ProcessWrapperSandboxedStrategy(spawnRunner));
     }
 
@@ -68,7 +74,7 @@ final class SandboxActionContextProvider extends ActionContextProvider {
       SpawnRunner spawnRunner =
           withFallback(
               cmdEnv,
-              LinuxSandboxedStrategy.create(cmdEnv, sandboxBase, productName, timeoutGraceSeconds));
+              LinuxSandboxedStrategy.create(cmdEnv, sandboxBase, productName, timeoutKillDelay));
       contexts.add(new LinuxSandboxedStrategy(spawnRunner));
     }
 
@@ -77,8 +83,7 @@ final class SandboxActionContextProvider extends ActionContextProvider {
       SpawnRunner spawnRunner =
           withFallback(
               cmdEnv,
-              new DarwinSandboxedSpawnRunner(
-                  cmdEnv, sandboxBase, productName, timeoutGraceSeconds));
+              new DarwinSandboxedSpawnRunner(cmdEnv, sandboxBase, productName, timeoutKillDelay));
       contexts.add(new DarwinSandboxedStrategy(spawnRunner));
     }
 
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
index a833dafb8e..6b5db9b449 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java
@@ -193,4 +193,15 @@ public class SandboxOptions extends OptionsBase {
     }
     return ImmutableSet.copyOf(inaccessiblePaths);
   }
+
+  @Option(
+    name = "experimental_collect_local_sandbox_action_metrics",
+    defaultValue = "false",
+    documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+    effectTags = {OptionEffectTag.EXECUTION},
+    help =
+        "When enabled, execution statistics (such as user and system time) are recorded for "
+            + "locally executed actions which use sandboxing"
+  )
+  public boolean collectLocalSandboxExecutionStatistics;
 }