Extract the MacOS/XCode env rewrite logic into lib.exec.apple

Also add an interface to allow injecting that logic into LocalSpawnRunner; this
is in preparation for rewriting StandaloneSpawnStrategy to use
LocalSpawnRunner.

At the same time, this reduces the dependencies from exec / standalone to
rules.apple, which is a prerequisite for micro-Bazel.

There's a small semantic change hidden here - we now only set the new
XCodeLocalEnvProvider if we're actually running on Darwin, so we no longer
fail execution on non-Darwin platforms if XCODE_VERSION_OVERRIDE or
APPLE_SDK_VERSION_OVERRIDE is set. As a result, I moved the corresponding test
from StandaloneSpawnStrategyTest to the new XCodeLocalEnvProviderTest.

While I'm at it, also open source DottedVersionTest and CacheManagerTest.

PiperOrigin-RevId: 158829077

5 files changed, 393 insertions, 23 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/exec/apple/BUILD b/src/main/java/com/google/devtools/build/lib/exec/apple/BUILD
new file mode 100644
index 0000000000..c7c19dc84e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/apple/BUILD
@@ -0,0 +1,24 @@
+package(
+    default_visibility = ["//src:__subpackages__"],
+)
+
+java_library(
+    name = "apple",
+    srcs = glob(["*.java"]),
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib:build-base",
+        "//src/main/java/com/google/devtools/build/lib:packages-internal",
+        "//src/main/java/com/google/devtools/build/lib:shell",
+        "//src/main/java/com/google/devtools/build/lib:util",
+        "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/build/lib/exec/local",
+        "//src/main/java/com/google/devtools/build/lib/rules/apple",
+        "//third_party:guava",
+    ],
+)
+
+filegroup(
+    name = "srcs",
+    testonly = 0,  # All srcs should be not test only, overwrite package default.
+    srcs = glob(["**"]),
+)
diff --git a/src/main/java/com/google/devtools/build/lib/exec/apple/CacheManager.java b/src/main/java/com/google/devtools/build/lib/exec/apple/CacheManager.java
new file mode 100644
index 0000000000..0c7e52a328
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/apple/CacheManager.java
@@ -0,0 +1,108 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.exec.apple;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * General cache file manager for mapping one or more keys to host-related path information.
+ *
+ * <p> Cache management has some notable restrictions:
+ * <ul>
+ * <li>Each cache entry must have the same number of (string) keys, and one value.</li>
+ * <li>An entry, once written to the cache, must be stable between builds. Clearing the cache
+ *     requires a full clean of the bazel output directory.</li>
+ * </ul>
+ *
+ * <p> Note that a single cache manager instance is not thread-safe, though multiple threads may
+ * hold cache manager instances for the same cache file. As a result, it is possible multiple
+ * threads may write the same entry to cache. This is fine, as retrieval from the cache will simply
+ * return the first found entry.
+ */
+final class CacheManager {
+
+  private final Path cacheFilePath;
+  private boolean cacheFileTouched;
+
+  /**
+   * @param outputRoot path to the bazel's output root
+   * @param cacheFilename name of the cache file
+   */
+  CacheManager(Path outputRoot, String cacheFilename) {
+    cacheFilePath = outputRoot.getRelative(cacheFilename);
+  }
+  
+  private void touchCacheFile() throws IOException {
+    if (!cacheFileTouched) {
+      FileSystemUtils.touchFile(cacheFilePath);
+      cacheFileTouched = true;
+    }
+  }
+  
+  /**
+   * Returns the value associated with the given list of string keys from the cache,
+   * or null if the entry is not present in the cache. If there is more than one value for the
+   * given key, the first value is returned.
+   */
+  @Nullable
+  public String getValue(String... keys) throws IOException {
+    Preconditions.checkArgument(keys.length > 0);
+    touchCacheFile();
+
+    List<String> keyList = ImmutableList.copyOf(keys);
+    Iterable<String> cacheContents =
+        FileSystemUtils.readLines(cacheFilePath, StandardCharsets.UTF_8);
+    for (String cacheLine : cacheContents) {
+      if (cacheLine.isEmpty()) {
+        continue;
+      }
+      List<String> cacheEntry = Splitter.on(':').splitToList(cacheLine);
+      List<String> cacheKeys = cacheEntry.subList(0, cacheEntry.size() - 1);
+      String cacheValue = cacheEntry.get(cacheEntry.size() - 1);
+      if (keyList.size() != cacheKeys.size()) {
+        throw new IllegalStateException(
+            String.format("cache file %s is malformed. Expected %s keys. line: '%s'",
+                cacheFilePath, keyList.size(), cacheLine));
+      }
+      if (keyList.equals(cacheKeys)) {
+        return cacheValue;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Write an entry to the cache. An entry consists of one or more keys and a single value.
+   * No validation is made regarding whether there are redundant or conflicting entries in the
+   * cache; it is thus the responsibility of the caller to ensure that any redundant entries
+   * (entries which have the same keys) also have the same value.
+   */
+  public void writeEntry(List<String> keys, String value) throws IOException {
+    Preconditions.checkArgument(!keys.isEmpty());
+
+    touchCacheFile();
+    FileSystemUtils.appendLinesAs(cacheFilePath, StandardCharsets.UTF_8,
+        Joiner.on(":").join(ImmutableList.builder().addAll(keys).add(value).build()));
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProvider.java
new file mode 100644
index 0000000000..8744ef0186
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProvider.java
@@ -0,0 +1,215 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec.apple;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.exec.local.LocalEnvProvider;
+import com.google.devtools.build.lib.rules.apple.AppleConfiguration;
+import com.google.devtools.build.lib.rules.apple.DottedVersion;
+import com.google.devtools.build.lib.shell.AbnormalTerminationException;
+import com.google.devtools.build.lib.shell.Command;
+import com.google.devtools.build.lib.shell.CommandException;
+import com.google.devtools.build.lib.shell.CommandResult;
+import com.google.devtools.build.lib.shell.TerminationStatus;
+import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+/**
+ * Adds to the given environment all variables that are dependent on system state of the host
+ * machine.
+ *
+ * <p>Admittedly, hermeticity is "best effort" in such cases; these environment values should be
+ * as tied to configuration parameters as possible.
+ *
+ * <p>For example, underlying iOS toolchains require that SDKROOT resolve to an absolute system
+ * path, but, when selecting which SDK to resolve, the version number comes from build
+ * configuration.
+ */
+public final class XCodeLocalEnvProvider implements LocalEnvProvider {
+  private static final String XCRUN_CACHE_FILENAME = "__xcruncache";
+  private static final String XCODE_LOCATOR_CACHE_FILENAME = "__xcodelocatorcache";
+
+  @Override
+  public Map<String, String> rewriteLocalEnv(
+      Map<String, String> env, Path execRoot, String productName) throws IOException {
+    boolean containsXcodeVersion = env.containsKey(AppleConfiguration.XCODE_VERSION_ENV_NAME);
+    boolean containsAppleSdkVersion =
+        env.containsKey(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME);
+    if (!containsXcodeVersion && !containsAppleSdkVersion) {
+      return env;
+    }
+
+    ImmutableMap.Builder<String, String> newEnvBuilder = ImmutableMap.builder();
+    newEnvBuilder.putAll(env);
+    // Empty developer dir indicates to use the system default.
+    // TODO(bazel-team): Bazel's view of the xcode version and developer dir should be explicitly
+    // set for build hermeticity.
+    String developerDir = "";
+    if (containsXcodeVersion) {
+      String version = env.get(AppleConfiguration.XCODE_VERSION_ENV_NAME);
+      developerDir = getDeveloperDir(execRoot, DottedVersion.fromString(version), productName);
+      newEnvBuilder.put("DEVELOPER_DIR", developerDir);
+    }
+    if (containsAppleSdkVersion) {
+      // The Apple platform is needed to select the appropriate SDK.
+      if (!env.containsKey(AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME)) {
+        throw new IOException("Could not resolve apple platform for determining SDK");
+      }
+      String iosSdkVersion = env.get(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME);
+      String appleSdkPlatform = env.get(AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME);
+      newEnvBuilder.put(
+          "SDKROOT",
+          getSdkRoot(execRoot, developerDir, iosSdkVersion, appleSdkPlatform, productName));
+    }
+    return newEnvBuilder.build();
+  }
+
+  /**
+   * Returns the absolute root path of the target Apple SDK on the host system for a given
+   * version of xcode (as defined by the given {@code developerDir}). This may spawn a
+   * process and use the {@code /usr/bin/xcrun} binary to locate the target SDK. This uses a local
+   * cache file under {@code bazel-out}, and will only spawn a new {@code xcrun} process in the case
+   * of a cache miss.
+   *
+   * @param execRoot the execution root path, used to locate the cache file
+   * @param developerDir the value of {@code DEVELOPER_DIR} for the target version of xcode
+   * @param sdkVersion the sdk version, for example, "9.1"
+   * @param appleSdkPlatform the sdk platform, for example, "iPhoneOS"
+   * @param productName the product name
+   * @throws IOException if there is an issue with obtaining the root from the spawned
+   *     process, either because the SDK platform/version pair doesn't exist, or there was an
+   *     unexpected issue finding or running the tool
+   */
+  private static String getSdkRoot(Path execRoot, String developerDir,
+      String sdkVersion, String appleSdkPlatform, String productName) throws IOException {
+    if (OS.getCurrent() != OS.DARWIN) {
+      throw new IOException("Cannot locate iOS SDK on non-darwin operating system");
+    }
+    try {
+      CacheManager cacheManager =
+          new CacheManager(execRoot.getRelative(
+              BlazeDirectories.getRelativeOutputPath(productName)),
+              XCRUN_CACHE_FILENAME);
+
+      String sdkString = appleSdkPlatform.toLowerCase() + sdkVersion;
+      String cacheResult = cacheManager.getValue(developerDir, sdkString);
+      if (cacheResult != null) {
+        return cacheResult;
+      } else {
+        Map<String, String> env = Strings.isNullOrEmpty(developerDir)
+            ? ImmutableMap.<String, String>of() : ImmutableMap.of("DEVELOPER_DIR", developerDir);
+        CommandResult xcrunResult = new Command(
+            new String[] {"/usr/bin/xcrun", "--sdk", sdkString, "--show-sdk-path"},
+            env, null).execute();
+
+        // calling xcrun via Command returns a value with a newline on the end.
+        String sdkRoot = new String(xcrunResult.getStdout(), StandardCharsets.UTF_8).trim();
+
+        cacheManager.writeEntry(ImmutableList.of(developerDir, sdkString), sdkRoot);
+        return sdkRoot;
+      }
+    } catch (AbnormalTerminationException e) {
+      TerminationStatus terminationStatus = e.getResult().getTerminationStatus();
+
+      if (terminationStatus.exited()) {
+        throw new IOException(
+            String.format("xcrun failed with code %s.\n"
+                + "This most likely indicates that SDK version [%s] for platform [%s] is "
+                + "unsupported for the target version of xcode.\n"
+                + "%s\n"
+                + "Stderr: %s",
+                terminationStatus.getExitCode(),
+                sdkVersion, appleSdkPlatform,
+                terminationStatus.toString(),
+                new String(e.getResult().getStderr(), StandardCharsets.UTF_8)));
+      }
+      String message = String.format("xcrun failed.\n%s\n%s",
+          e.getResult().getTerminationStatus(),
+          new String(e.getResult().getStderr(), StandardCharsets.UTF_8));
+      throw new IOException(message, e);
+    } catch (CommandException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Returns the absolute root path of the xcode developer directory on the host system for
+   * the given xcode version. This may spawn a process and use the {@code xcode-locator} binary.
+   * This uses a local cache file under {@code bazel-out}, and will only spawn a new process in the
+   * case of a cache miss.
+   *
+   * @param execRoot the execution root path, used to locate the cache file
+   * @param version the xcode version number to look up
+   * @param productName the product name
+   * @throws IOException if there is an issue with obtaining the path from the spawned
+   *     process, either because there is no installed xcode with the given version, or
+   *     there was an unexpected issue finding or running the tool
+   */
+  private static String getDeveloperDir(Path execRoot, DottedVersion version, String productName)
+      throws IOException {
+    if (OS.getCurrent() != OS.DARWIN) {
+      throw new IOException(
+          "Cannot locate xcode developer directory on non-darwin operating system");
+    }
+    try {
+      CacheManager cacheManager =
+          new CacheManager(
+              execRoot.getRelative(BlazeDirectories.getRelativeOutputPath(productName)),
+              XCODE_LOCATOR_CACHE_FILENAME);
+
+      String cacheResult = cacheManager.getValue(version.toString());
+      if (cacheResult != null) {
+        return cacheResult;
+      } else {
+        CommandResult xcodeLocatorResult = new Command(new String[] {
+            execRoot.getRelative("_bin/xcode-locator").getPathString(), version.toString()})
+            .execute();
+
+        String developerDir =
+            new String(xcodeLocatorResult.getStdout(), StandardCharsets.UTF_8).trim();
+
+        cacheManager.writeEntry(ImmutableList.of(version.toString()), developerDir);
+        return developerDir;
+      }
+    } catch (AbnormalTerminationException e) {
+      TerminationStatus terminationStatus = e.getResult().getTerminationStatus();
+
+      String message;
+      if (e.getResult().getTerminationStatus().exited()) {
+        message = String.format("xcode-locator failed with code %s.\n"
+            + "This most likely indicates that xcode version %s is not available on the host "
+            + "machine.\n"
+            + "%s\n"
+            + "stderr: %s",
+            terminationStatus.getExitCode(),
+            version,
+            terminationStatus.toString(),
+            new String(e.getResult().getStderr(), StandardCharsets.UTF_8));
+      } else {
+        message = String.format("xcode-locator failed. %s\nstderr: %s",
+            e.getResult().getTerminationStatus(),
+            new String(e.getResult().getStderr(), StandardCharsets.UTF_8));
+      }
+      throw new IOException(message, e);
+    } catch (CommandException e) {
+      throw new IOException(e);
+    }
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java
new file mode 100644
index 0000000000..52610761c5
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java
@@ -0,0 +1,36 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec.local;
+
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Allows just-in-time rewriting of the environment used for local actions. Do not use! This class
+ * probably should not exist, but is currently necessary for our local MacOS support.
+ */
+public interface LocalEnvProvider {
+  public static final LocalEnvProvider UNMODIFIED = new LocalEnvProvider() {
+    @Override
+    public Map<String, String> rewriteLocalEnv(
+        Map<String, String> env, Path execRoot, String productName) throws IOException {
+      return env;
+    }
+  };
+
+  /** Rewrites the environment if necessary. */
+  Map<String, String> rewriteLocalEnv(Map<String, String> env, Path execRoot, String productName)
+      throws IOException;
+}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java
index 215ae2ef14..418646b1cd 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java
@@ -34,7 +34,6 @@ import com.google.devtools.build.lib.shell.Command;
 import com.google.devtools.build.lib.shell.CommandException;
 import com.google.devtools.build.lib.shell.CommandResult;
 import com.google.devtools.build.lib.util.NetUtil;
-import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.util.Preconditions;
 import com.google.devtools.build.lib.util.io.FileOutErr;
 import com.google.devtools.build.lib.vfs.Path;
@@ -75,6 +74,9 @@ public final class LocalSpawnRunner implements SpawnRunner {
   private final boolean useProcessWrapper;
   private final String processWrapper;
 
+  private final String productName;
+  private final LocalEnvProvider localEnvProvider;
+
   public LocalSpawnRunner(
       Logger logger,
       AtomicInteger execCount,
@@ -82,7 +84,9 @@ public final class LocalSpawnRunner implements SpawnRunner {
       ActionInputPrefetcher actionInputPrefetcher,
       LocalExecutionOptions localExecutionOptions,
       ResourceManager resourceManager,
-      boolean useProcessWrapper) {
+      boolean useProcessWrapper,
+      String productName,
+      LocalEnvProvider localEnvProvider) {
     this.logger = logger;
     this.execRoot = execRoot;
     this.actionInputPrefetcher = Preconditions.checkNotNull(actionInputPrefetcher);
@@ -92,25 +96,8 @@ public final class LocalSpawnRunner implements SpawnRunner {
     this.execCount = execCount;
     this.resourceManager = resourceManager;
     this.useProcessWrapper = useProcessWrapper;
-  }
-
-  public LocalSpawnRunner(
-      Path execRoot,
-      ActionInputPrefetcher actionInputPrefetcher,
-      LocalExecutionOptions localExecutionOptions,
-      ResourceManager resourceManager) {
-    this(
-        null,
-        new AtomicInteger(),
-        execRoot,
-        actionInputPrefetcher,
-        localExecutionOptions,
-        resourceManager,
-        // TODO(bazel-team): process-wrapper seems to work on Windows, but requires additional setup
-        // as it is an msys2 binary, so it needs msys2 DLLs on %PATH%. Disable it for now to make
-        // the setup easier and to avoid further PATH hacks. Ideally we should have a native
-        // implementation of process-wrapper for Windows.
-        OS.getCurrent() != OS.WINDOWS);
+    this.productName = productName;
+    this.localEnvProvider = localEnvProvider;
   }
 
   @Override
@@ -247,14 +234,14 @@ public final class LocalSpawnRunner implements SpawnRunner {
         cmdLine.addAll(spawn.getArguments());
         cmd = new Command(
             cmdLine.toArray(new String[0]),
-            spawn.getEnvironment(),
+            localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName),
             execRoot.getPathFile());
       } else {
         stdOut = outErr.getOutputStream();
         stdErr = outErr.getErrorStream();
         cmd = new Command(
             spawn.getArguments().toArray(new String[0]),
-            spawn.getEnvironment(),
+            localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName),
             execRoot.getPathFile(),
             policy.getTimeoutMillis());
       }