diff options
author | 2017-06-13 12:56:07 +0200 | |
---|---|---|
committer | 2017-06-13 17:13:17 +0200 | |
commit | 1d62c67a2d6d6eccf415ad5647d860d08f4c5966 (patch) | |
tree | 23eeec1aa45038fc5693b09c8d2556df5f92aa59 /src | |
parent | 3e87c626ed76536420aa06e4c258209b32bb76e0 (diff) |
Extract the MacOS/XCode env rewrite logic into lib.exec.apple
Also add an interface to allow injecting that logic into LocalSpawnRunner; this
is in preparation for rewriting StandaloneSpawnStrategy to use
LocalSpawnRunner.
At the same time, this reduces the dependencies from exec / standalone to
rules.apple, which is a prerequisite for micro-Bazel.
There's a small semantic change hidden here - we now only set the new
XCodeLocalEnvProvider if we're actually running on Darwin, so we no longer
fail execution on non-Darwin platforms if XCODE_VERSION_OVERRIDE or
APPLE_SDK_VERSION_OVERRIDE is set. As a result, I moved the corresponding test
from StandaloneSpawnStrategyTest to the new XCodeLocalEnvProviderTest.
While I'm at it, also open source DottedVersionTest and CacheManagerTest.
PiperOrigin-RevId: 158829077
Diffstat (limited to 'src')
15 files changed, 372 insertions, 172 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index be4f82d109..f53a85f321 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -24,6 +24,7 @@ filegroup( "//src/main/java/com/google/devtools/build/lib/buildeventservice/client:srcs", "//src/main/java/com/google/devtools/build/lib/causes:srcs", "//src/main/java/com/google/devtools/build/lib/cmdline:srcs", + "//src/main/java/com/google/devtools/build/lib/exec/apple:srcs", "//src/main/java/com/google/devtools/build/lib/exec/local:srcs", "//src/main/java/com/google/devtools/build/lib/query2:srcs", "//src/main/java/com/google/devtools/build/lib/remote:srcs", diff --git a/src/main/java/com/google/devtools/build/lib/exec/apple/BUILD b/src/main/java/com/google/devtools/build/lib/exec/apple/BUILD new file mode 100644 index 0000000000..c7c19dc84e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/apple/BUILD @@ -0,0 +1,24 @@ +package( + default_visibility = ["//src:__subpackages__"], +) + +java_library( + name = "apple", + srcs = glob(["*.java"]), + deps = [ + "//src/main/java/com/google/devtools/build/lib:build-base", + "//src/main/java/com/google/devtools/build/lib:packages-internal", + "//src/main/java/com/google/devtools/build/lib:shell", + "//src/main/java/com/google/devtools/build/lib:util", + "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/build/lib/exec/local", + "//src/main/java/com/google/devtools/build/lib/rules/apple", + "//third_party:guava", + ], +) + +filegroup( + name = "srcs", + testonly = 0, # All srcs should be not test only, overwrite package default. + srcs = glob(["**"]), +) diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/CacheManager.java b/src/main/java/com/google/devtools/build/lib/exec/apple/CacheManager.java index 390ef50572..0c7e52a328 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/CacheManager.java +++ b/src/main/java/com/google/devtools/build/lib/exec/apple/CacheManager.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.rules.apple; +package com.google.devtools.build.lib.exec.apple; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -20,11 +20,9 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; - import javax.annotation.Nullable; /** @@ -42,7 +40,7 @@ import javax.annotation.Nullable; * threads may write the same entry to cache. This is fine, as retrieval from the cache will simply * return the first found entry. */ -class CacheManager { +final class CacheManager { private final Path cacheFilePath; private boolean cacheFileTouched; diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleHostInfo.java b/src/main/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProvider.java index dd0023a23c..8744ef0186 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleHostInfo.java +++ b/src/main/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProvider.java @@ -1,4 +1,4 @@ -// Copyright 2015 The Bazel Authors. All rights reserved. +// Copyright 2017 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,34 +11,76 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -package com.google.devtools.build.lib.rules.apple; +package com.google.devtools.build.lib.exec.apple; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; +import com.google.devtools.build.lib.rules.apple.AppleConfiguration; +import com.google.devtools.build.lib.rules.apple.DottedVersion; import com.google.devtools.build.lib.shell.AbnormalTerminationException; import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.shell.CommandResult; import com.google.devtools.build.lib.shell.TerminationStatus; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.Path; - import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Map; /** - * Obtains information pertaining to Apple host machines required for using Apple toolkits in - * local action execution. + * Adds to the given environment all variables that are dependent on system state of the host + * machine. + * + * <p>Admittedly, hermeticity is "best effort" in such cases; these environment values should be + * as tied to configuration parameters as possible. + * + * <p>For example, underlying iOS toolchains require that SDKROOT resolve to an absolute system + * path, but, when selecting which SDK to resolve, the version number comes from build + * configuration. */ -public class AppleHostInfo { - +public final class XCodeLocalEnvProvider implements LocalEnvProvider { private static final String XCRUN_CACHE_FILENAME = "__xcruncache"; private static final String XCODE_LOCATOR_CACHE_FILENAME = "__xcodelocatorcache"; + @Override + public Map<String, String> rewriteLocalEnv( + Map<String, String> env, Path execRoot, String productName) throws IOException { + boolean containsXcodeVersion = env.containsKey(AppleConfiguration.XCODE_VERSION_ENV_NAME); + boolean containsAppleSdkVersion = + env.containsKey(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME); + if (!containsXcodeVersion && !containsAppleSdkVersion) { + return env; + } + + ImmutableMap.Builder<String, String> newEnvBuilder = ImmutableMap.builder(); + newEnvBuilder.putAll(env); + // Empty developer dir indicates to use the system default. + // TODO(bazel-team): Bazel's view of the xcode version and developer dir should be explicitly + // set for build hermeticity. + String developerDir = ""; + if (containsXcodeVersion) { + String version = env.get(AppleConfiguration.XCODE_VERSION_ENV_NAME); + developerDir = getDeveloperDir(execRoot, DottedVersion.fromString(version), productName); + newEnvBuilder.put("DEVELOPER_DIR", developerDir); + } + if (containsAppleSdkVersion) { + // The Apple platform is needed to select the appropriate SDK. + if (!env.containsKey(AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME)) { + throw new IOException("Could not resolve apple platform for determining SDK"); + } + String iosSdkVersion = env.get(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME); + String appleSdkPlatform = env.get(AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME); + newEnvBuilder.put( + "SDKROOT", + getSdkRoot(execRoot, developerDir, iosSdkVersion, appleSdkPlatform, productName)); + } + return newEnvBuilder.build(); + } + /** * Returns the absolute root path of the target Apple SDK on the host system for a given * version of xcode (as defined by the given {@code developerDir}). This may spawn a @@ -51,12 +93,15 @@ public class AppleHostInfo { * @param sdkVersion the sdk version, for example, "9.1" * @param appleSdkPlatform the sdk platform, for example, "iPhoneOS" * @param productName the product name - * @throws UserExecException if there is an issue with obtaining the root from the spawned + * @throws IOException if there is an issue with obtaining the root from the spawned * process, either because the SDK platform/version pair doesn't exist, or there was an * unexpected issue finding or running the tool */ - public static String getSdkRoot(Path execRoot, String developerDir, - String sdkVersion, String appleSdkPlatform, String productName) throws UserExecException { + private static String getSdkRoot(Path execRoot, String developerDir, + String sdkVersion, String appleSdkPlatform, String productName) throws IOException { + if (OS.getCurrent() != OS.DARWIN) { + throw new IOException("Cannot locate iOS SDK on non-darwin operating system"); + } try { CacheManager cacheManager = new CacheManager(execRoot.getRelative( @@ -84,7 +129,7 @@ public class AppleHostInfo { TerminationStatus terminationStatus = e.getResult().getTerminationStatus(); if (terminationStatus.exited()) { - throw new UserExecException( + throw new IOException( String.format("xcrun failed with code %s.\n" + "This most likely indicates that SDK version [%s] for platform [%s] is " + "unsupported for the target version of xcode.\n" @@ -98,9 +143,9 @@ public class AppleHostInfo { String message = String.format("xcrun failed.\n%s\n%s", e.getResult().getTerminationStatus(), new String(e.getResult().getStderr(), StandardCharsets.UTF_8)); - throw new UserExecException(message, e); - } catch (CommandException | IOException e) { - throw new UserExecException(e); + throw new IOException(message, e); + } catch (CommandException e) { + throw new IOException(e); } } @@ -113,15 +158,20 @@ public class AppleHostInfo { * @param execRoot the execution root path, used to locate the cache file * @param version the xcode version number to look up * @param productName the product name - * @throws UserExecException if there is an issue with obtaining the path from the spawned + * @throws IOException if there is an issue with obtaining the path from the spawned * process, either because there is no installed xcode with the given version, or * there was an unexpected issue finding or running the tool */ - public static String getDeveloperDir(Path execRoot, DottedVersion version, String productName) - throws UserExecException { + private static String getDeveloperDir(Path execRoot, DottedVersion version, String productName) + throws IOException { + if (OS.getCurrent() != OS.DARWIN) { + throw new IOException( + "Cannot locate xcode developer directory on non-darwin operating system"); + } try { CacheManager cacheManager = - new CacheManager(execRoot.getRelative(BlazeDirectories.getRelativeOutputPath(productName)), + new CacheManager( + execRoot.getRelative(BlazeDirectories.getRelativeOutputPath(productName)), XCODE_LOCATOR_CACHE_FILENAME); String cacheResult = cacheManager.getValue(version.toString()); @@ -157,9 +207,9 @@ public class AppleHostInfo { e.getResult().getTerminationStatus(), new String(e.getResult().getStderr(), StandardCharsets.UTF_8)); } - throw new UserExecException(message, e); - } catch (CommandException | IOException e) { - throw new UserExecException(e); + throw new IOException(message, e); + } catch (CommandException e) { + throw new IOException(e); } } } diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java new file mode 100644 index 0000000000..52610761c5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalEnvProvider.java @@ -0,0 +1,36 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec.local; + +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import java.util.Map; + +/** + * Allows just-in-time rewriting of the environment used for local actions. Do not use! This class + * probably should not exist, but is currently necessary for our local MacOS support. + */ +public interface LocalEnvProvider { + public static final LocalEnvProvider UNMODIFIED = new LocalEnvProvider() { + @Override + public Map<String, String> rewriteLocalEnv( + Map<String, String> env, Path execRoot, String productName) throws IOException { + return env; + } + }; + + /** Rewrites the environment if necessary. */ + Map<String, String> rewriteLocalEnv(Map<String, String> env, Path execRoot, String productName) + throws IOException; +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java index 215ae2ef14..418646b1cd 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java @@ -34,7 +34,6 @@ import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.shell.CommandResult; import com.google.devtools.build.lib.util.NetUtil; -import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; @@ -75,6 +74,9 @@ public final class LocalSpawnRunner implements SpawnRunner { private final boolean useProcessWrapper; private final String processWrapper; + private final String productName; + private final LocalEnvProvider localEnvProvider; + public LocalSpawnRunner( Logger logger, AtomicInteger execCount, @@ -82,7 +84,9 @@ public final class LocalSpawnRunner implements SpawnRunner { ActionInputPrefetcher actionInputPrefetcher, LocalExecutionOptions localExecutionOptions, ResourceManager resourceManager, - boolean useProcessWrapper) { + boolean useProcessWrapper, + String productName, + LocalEnvProvider localEnvProvider) { this.logger = logger; this.execRoot = execRoot; this.actionInputPrefetcher = Preconditions.checkNotNull(actionInputPrefetcher); @@ -92,25 +96,8 @@ public final class LocalSpawnRunner implements SpawnRunner { this.execCount = execCount; this.resourceManager = resourceManager; this.useProcessWrapper = useProcessWrapper; - } - - public LocalSpawnRunner( - Path execRoot, - ActionInputPrefetcher actionInputPrefetcher, - LocalExecutionOptions localExecutionOptions, - ResourceManager resourceManager) { - this( - null, - new AtomicInteger(), - execRoot, - actionInputPrefetcher, - localExecutionOptions, - resourceManager, - // TODO(bazel-team): process-wrapper seems to work on Windows, but requires additional setup - // as it is an msys2 binary, so it needs msys2 DLLs on %PATH%. Disable it for now to make - // the setup easier and to avoid further PATH hacks. Ideally we should have a native - // implementation of process-wrapper for Windows. - OS.getCurrent() != OS.WINDOWS); + this.productName = productName; + this.localEnvProvider = localEnvProvider; } @Override @@ -247,14 +234,14 @@ public final class LocalSpawnRunner implements SpawnRunner { cmdLine.addAll(spawn.getArguments()); cmd = new Command( cmdLine.toArray(new String[0]), - spawn.getEnvironment(), + localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName), execRoot.getPathFile()); } else { stdOut = outErr.getOutputStream(); stdErr = outErr.getErrorStream(); cmd = new Command( spawn.getArguments().toArray(new String[0]), - spawn.getEnvironment(), + localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName), execRoot.getPathFile(), policy.getTimeoutMillis()); } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/BUILD b/src/main/java/com/google/devtools/build/lib/sandbox/BUILD index 82010f3128..d93532c094 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/BUILD +++ b/src/main/java/com/google/devtools/build/lib/sandbox/BUILD @@ -10,18 +10,16 @@ java_library( ], deps = [ "//src/main/java/com/google/devtools/build/lib:build-base", - "//src/main/java/com/google/devtools/build/lib:concurrent", "//src/main/java/com/google/devtools/build/lib:events", "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:packages-internal", - "//src/main/java/com/google/devtools/build/lib:process_util", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:shell", - "//src/main/java/com/google/devtools/build/lib:unix", "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/actions", - "//src/main/java/com/google/devtools/build/lib/rules/cpp", + "//src/main/java/com/google/devtools/build/lib/exec/apple", + "//src/main/java/com/google/devtools/build/lib/exec/local", "//src/main/java/com/google/devtools/build/lib/standalone", "//src/main/java/com/google/devtools/common/options", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java index 90170a4d0e..5798303c1e 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java @@ -16,7 +16,6 @@ package com.google.devtools.build.lib.sandbox; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionStatusMessage; @@ -27,17 +26,19 @@ import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.exec.SpawnInputExpander; +import com.google.devtools.build.lib.exec.apple.XCodeLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.shell.CommandResult; -import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -60,6 +61,7 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { * <p>We cache this, because creating it involves executing {@code getconf}, which is expensive. */ private final ImmutableSet<Path> alwaysWritableDirs; + private final LocalEnvProvider localEnvProvider; private DarwinSandboxedStrategy( CommandEnvironment cmdEnv, @@ -80,6 +82,7 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { this.productName = productName; this.alwaysWritableDirs = alwaysWritableDirs; this.spawnInputExpander = new SpawnInputExpander(false); + this.localEnvProvider = new XCodeLocalEnvProvider(); } public static DarwinSandboxedStrategy create( @@ -169,8 +172,8 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { Path sandboxPath = getSandboxRoot(); Path sandboxExecRoot = sandboxPath.getRelative("execroot").getRelative(execRoot.getBaseName()); - ImmutableMap<String, String> spawnEnvironment = - StandaloneSpawnStrategy.locallyDeterminedEnv(execRoot, productName, spawn.getEnvironment()); + Map<String, String> spawnEnvironment = + localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName); HashSet<Path> writableDirs = new HashSet<>(alwaysWritableDirs); ImmutableSet<Path> extraWritableDirs = getWritableDirs(sandboxExecRoot, spawnEnvironment); diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java index 64fecbcc4d..a8f0e8063c 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.sandbox; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionStatusMessage; @@ -25,12 +24,15 @@ import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.exec.SpawnInputExpander; +import com.google.devtools.build.lib.exec.apple.XCodeLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; import com.google.devtools.build.lib.runtime.CommandEnvironment; -import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -50,6 +52,7 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy { private final boolean verboseFailures; private final String productName; private final SpawnInputExpander spawnInputExpander; + private final LocalEnvProvider localEnvProvider; ProcessWrapperSandboxedStrategy( CommandEnvironment cmdEnv, @@ -68,6 +71,9 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy { this.verboseFailures = verboseFailures; this.productName = productName; this.spawnInputExpander = new SpawnInputExpander(false); + this.localEnvProvider = OS.getCurrent() == OS.DARWIN + ? new XCodeLocalEnvProvider() + : LocalEnvProvider.UNMODIFIED; } @Override @@ -88,8 +94,8 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy { Path sandboxPath = getSandboxRoot(); Path sandboxExecRoot = sandboxPath.getRelative("execroot").getRelative(execRoot.getBaseName()); - ImmutableMap<String, String> spawnEnvironment = - StandaloneSpawnStrategy.locallyDeterminedEnv(execRoot, productName, spawn.getEnvironment()); + Map<String, String> spawnEnvironment = + localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName); Set<Path> writableDirs = getWritableDirs(sandboxExecRoot, spawn.getEnvironment()); SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot); diff --git a/src/main/java/com/google/devtools/build/lib/standalone/BUILD b/src/main/java/com/google/devtools/build/lib/standalone/BUILD index 0b2171e25e..64e70e52c7 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/BUILD +++ b/src/main/java/com/google/devtools/build/lib/standalone/BUILD @@ -12,12 +12,13 @@ java_library( "//src/main/java/com/google/devtools/build/lib:build-base", "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:packages-internal", - "//src/main/java/com/google/devtools/build/lib:process_util", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:shell", "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/exec/apple", + "//src/main/java/com/google/devtools/build/lib/exec/local", "//src/main/java/com/google/devtools/build/lib/rules/apple", "//src/main/java/com/google/devtools/build/lib/rules/cpp", "//third_party:guava", diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java index fe9774d760..724542ff1b 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java @@ -13,7 +13,6 @@ // limitations under the License. package com.google.devtools.build.lib.standalone; -import com.google.common.collect.ImmutableMap; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionExecutionMetadata; @@ -27,9 +26,8 @@ import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.UserExecException; -import com.google.devtools.build.lib.rules.apple.AppleConfiguration; -import com.google.devtools.build.lib.rules.apple.AppleHostInfo; -import com.google.devtools.build.lib.rules.apple.DottedVersion; +import com.google.devtools.build.lib.exec.apple.XCodeLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; import com.google.devtools.build.lib.shell.AbnormalTerminationException; import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; @@ -40,6 +38,7 @@ import com.google.devtools.build.lib.util.OsUtils; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -53,6 +52,7 @@ public class StandaloneSpawnStrategy implements SpawnActionContext { private final Path execRoot; private final String productName; private final ResourceManager resourceManager; + private final LocalEnvProvider localEnvProvider; public StandaloneSpawnStrategy(Path execRoot, boolean verboseFailures, String productName) { this(execRoot, verboseFailures, productName, ResourceManager.instance()); @@ -66,6 +66,9 @@ public class StandaloneSpawnStrategy implements SpawnActionContext { "_bin/process-wrapper" + OsUtils.executableExtension()); this.productName = productName; this.resourceManager = resourceManager; + this.localEnvProvider = OS.getCurrent() == OS.DARWIN + ? new XCodeLocalEnvProvider() + : LocalEnvProvider.UNMODIFIED; } /** @@ -81,7 +84,11 @@ public class StandaloneSpawnStrategy implements SpawnActionContext { try (ResourceHandle handle = resourceManager.acquireResources(owner, spawn.getLocalResources())) { eventBus.post(ActionStatusMessage.runningStrategy(owner, "standalone")); - actuallyExec(spawn, actionExecutionContext); + try { + actuallyExec(spawn, actionExecutionContext); + } catch (IOException e) { + throw new UserExecException("I/O exception during local execution", e); + } } } @@ -90,7 +97,7 @@ public class StandaloneSpawnStrategy implements SpawnActionContext { */ private void actuallyExec(Spawn spawn, ActionExecutionContext actionExecutionContext) - throws ExecException { + throws ExecException, IOException { Executor executor = actionExecutionContext.getExecutor(); if (executor.reportsSubcommands()) { @@ -123,7 +130,7 @@ public class StandaloneSpawnStrategy implements SpawnActionContext { Command cmd = new Command( args.toArray(new String[] {}), - locallyDeterminedEnv(execRoot, productName, spawn.getEnvironment()), + localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName), new File(cwd), OS.getCurrent() == OS.WINDOWS && timeoutSeconds >= 0 ? timeoutSeconds * 1000 : -1); @@ -155,74 +162,6 @@ public class StandaloneSpawnStrategy implements SpawnActionContext { return "standalone"; } - /** - * Adds to the given environment all variables that are dependent on system state of the host - * machine. - * - * <p>Admittedly, hermeticity is "best effort" in such cases; these environment values should be - * as tied to configuration parameters as possible. - * - * <p>For example, underlying iOS toolchains require that SDKROOT resolve to an absolute system - * path, but, when selecting which SDK to resolve, the version number comes from build - * configuration. - * - * @return the new environment, comprised of the old environment plus any new variables - * @throws UserExecException if any variables dependent on system state could not be resolved - */ - public static ImmutableMap<String, String> locallyDeterminedEnv( - Path execRoot, String productName, ImmutableMap<String, String> env) - throws UserExecException { - // TODO(bazel-team): Remove apple-specific logic from this class. - ImmutableMap.Builder<String, String> newEnvBuilder = ImmutableMap.builder(); - newEnvBuilder.putAll(env); - // Empty developer dir indicates to use the system default. - // TODO(bazel-team): Bazel's view of the xcode version and developer dir - // should be explicitly set for build hermiticity. - String developerDir = ""; - if (env.containsKey(AppleConfiguration.XCODE_VERSION_ENV_NAME)) { - developerDir = - getDeveloperDir( - execRoot, productName, env.get(AppleConfiguration.XCODE_VERSION_ENV_NAME)); - newEnvBuilder.put("DEVELOPER_DIR", developerDir); - } - if (env.containsKey(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME)) { - // The Apple platform is needed to select the appropriate SDK. - if (!env.containsKey(AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME)) { - throw new UserExecException("Could not resolve apple platform for determining SDK"); - } - String iosSdkVersion = env.get(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME); - String appleSdkPlatform = env.get(AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME); - newEnvBuilder.put( - "SDKROOT", - getSdkRootEnv(execRoot, productName, developerDir, iosSdkVersion, appleSdkPlatform)); - } - return newEnvBuilder.build(); - } - - private static String getDeveloperDir(Path execRoot, String productName, String xcodeVersion) - throws UserExecException { - if (OS.getCurrent() != OS.DARWIN) { - throw new UserExecException( - "Cannot locate xcode developer directory on non-darwin operating system"); - } - return AppleHostInfo.getDeveloperDir(execRoot, DottedVersion.fromString(xcodeVersion), - productName); - } - - private static String getSdkRootEnv( - Path execRoot, - String productName, - String developerDir, - String iosSdkVersion, - String appleSdkPlatform) - throws UserExecException { - if (OS.getCurrent() != OS.DARWIN) { - throw new UserExecException("Cannot locate iOS SDK on non-darwin operating system"); - } - return AppleHostInfo.getSdkRoot(execRoot, developerDir, iosSdkVersion, appleSdkPlatform, - productName); - } - @Override public boolean shouldPropagateExecException() { return false; diff --git a/src/test/java/com/google/devtools/build/lib/exec/apple/CacheManagerTest.java b/src/test/java/com/google/devtools/build/lib/exec/apple/CacheManagerTest.java new file mode 100644 index 0000000000..386e6e355d --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/exec/apple/CacheManagerTest.java @@ -0,0 +1,110 @@ +// Copyright 2017 The Bazel Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec.apple; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link CacheManager}. + */ +@RunWith(JUnit4.class) +public class CacheManagerTest { + private static final String CACHE_FILENAME = "cachefilename"; + private Path cachePath; + private CacheManager cacheManager; + private final InMemoryFileSystem fs = new InMemoryFileSystem(); + + @Before + public void init() throws Exception { + Path outputBase = fs.getPath("/somewhere"); + assertThat(outputBase.createDirectory()).isTrue(); + cachePath = outputBase.getRelative(CACHE_FILENAME); + cacheManager = new CacheManager(outputBase, CACHE_FILENAME); + } + + @Test + public void testEmptyCache() throws Exception { + assertThat(cacheManager.getValue("foo")).isNull(); + } + + @Test + public void testSingleKeyCacheHits() throws Exception { + cacheManager.writeEntry(ImmutableList.of("foo"), "sdkroot1"); + cacheManager.writeEntry(ImmutableList.of("bar"), "sdkroot2"); + cacheManager.writeEntry(ImmutableList.of("baz"), "sdkroot3"); + + assertThat(cacheManager.getValue("bar")).isEqualTo("sdkroot2"); + assertThat(cacheManager.getValue("baz")).isEqualTo("sdkroot3"); + assertThat(cacheManager.getValue("foo")).isEqualTo("sdkroot1"); + } + + @Test + public void testSingleKeyCacheMiss() throws Exception { + cacheManager.writeEntry(ImmutableList.of("foo"), "sdkroot1"); + + assertThat(cacheManager.getValue("bar")).isNull(); + } + + @Test + public void testMultiKeyCacheHits() throws Exception { + cacheManager.writeEntry(ImmutableList.of("foo", "cat"), "sdkroot1"); + cacheManager.writeEntry(ImmutableList.of("bar", "cat"), "sdkroot2"); + cacheManager.writeEntry(ImmutableList.of("baz", "dog"), "sdkroot3"); + + assertThat(cacheManager.getValue("bar", "cat")).isEqualTo("sdkroot2"); + assertThat(cacheManager.getValue("baz", "dog")).isEqualTo("sdkroot3"); + assertThat(cacheManager.getValue("foo", "cat")).isEqualTo("sdkroot1"); + } + + @Test + public void testMultiKeyCacheMiss() throws Exception { + cacheManager.writeEntry(ImmutableList.of("foo", "cat"), "sdkroot1"); + + assertThat(cacheManager.getValue("bar", "cat")).isNull(); + } + + @Test + public void testKeyCountMismatch() throws Exception { + cacheManager.writeEntry(ImmutableList.of("foo", "cat"), "sdkroot1"); + + try { + cacheManager.getValue("foo"); + fail("Key count mismatch, should have thrown exception"); + } catch (IllegalStateException expected) { + assertThat(expected).hasMessageThat().contains("malformed"); + } + } + + @Test + public void testBadCache() throws Exception { + FileSystemUtils.writeContentAsLatin1(cachePath, "blah blah blah"); + + try { + cacheManager.getValue("foo"); + fail("Cache file was corrupt, should have thrown exception"); + } catch (IllegalStateException expected) { + assertThat(expected).hasMessageThat().contains("malformed"); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProviderTest.java b/src/test/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProviderTest.java new file mode 100644 index 0000000000..1b8e3bb243 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/exec/apple/XCodeLocalEnvProviderTest.java @@ -0,0 +1,52 @@ +// Copyright 2017 The Bazel Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec.apple; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.rules.apple.AppleConfiguration; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import java.io.IOException; +import org.junit.Test; + +/** + * Tests for {@link XCodeLocalEnvProvider}. + */ +public class XCodeLocalEnvProviderTest { + private final FileSystem fs = new JavaIoFileSystem(); + + @Test + public void testIOSEnvironmentOnNonDarwin() throws Exception { + if (OS.getCurrent() == OS.DARWIN) { + return; + } + try { + new XCodeLocalEnvProvider().rewriteLocalEnv( + ImmutableMap.<String, String>of( + AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME, "8.4", + AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME, "iPhoneSimulator"), + fs.getPath("/tmp"), + "bazel"); + fail("action should fail due to being unable to resolve SDKROOT"); + } catch (IOException e) { + assertThat(e) + .hasMessageThat() + .contains("Cannot locate iOS SDK on non-darwin operating system"); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java index fb6fb9264c..29e46e9ba4 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunnerTest.java @@ -17,6 +17,7 @@ package com.google.devtools.build.lib.exec.local; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -216,7 +217,7 @@ public class LocalSpawnRunnerTest { options.localSigkillGraceSeconds = 456; LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, - resourceManager, USE_WRAPPER); + resourceManager, USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); timeoutMillis = 123 * 1000L; outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); @@ -251,7 +252,7 @@ public class LocalSpawnRunnerTest { options.localSigkillGraceSeconds = 456; LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, - resourceManager, NO_WRAPPER); + resourceManager, NO_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); timeoutMillis = 123 * 1000L; outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); @@ -281,7 +282,7 @@ public class LocalSpawnRunnerTest { LocalExecutionOptions options = Options.getDefaults(LocalExecutionOptions.class); LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, - resourceManager, USE_WRAPPER); + resourceManager, USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); SpawnResult result = runner.exec(SIMPLE_SPAWN, policy); @@ -311,7 +312,7 @@ public class LocalSpawnRunnerTest { LocalExecutionOptions options = Options.getDefaults(LocalExecutionOptions.class); LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, - resourceManager, USE_WRAPPER); + resourceManager, USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); assertThat(fs.getPath("/out").createDirectory()).isTrue(); outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); @@ -335,7 +336,7 @@ public class LocalSpawnRunnerTest { options.allowedLocalAction = Pattern.compile("none"); LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, - resourceManager, USE_WRAPPER); + resourceManager, USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); outErr = new FileOutErr(); SpawnResult reply = runner.exec(SIMPLE_SPAWN, policy); @@ -374,7 +375,7 @@ public class LocalSpawnRunnerTest { LocalExecutionOptions options = Options.getDefaults(LocalExecutionOptions.class); LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, - resourceManager, USE_WRAPPER); + resourceManager, USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); try { @@ -397,7 +398,7 @@ public class LocalSpawnRunnerTest { LocalExecutionOptions options = Options.getDefaults(LocalExecutionOptions.class); LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), mockPrefetcher, options, resourceManager, - USE_WRAPPER); + USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); timeoutMillis = 123 * 1000L; outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); @@ -416,7 +417,7 @@ public class LocalSpawnRunnerTest { LocalExecutionOptions options = Options.getDefaults(LocalExecutionOptions.class); LocalSpawnRunner runner = new LocalSpawnRunner( logger, execCount, fs.getPath("/execroot"), mockPrefetcher, options, resourceManager, - USE_WRAPPER); + USE_WRAPPER, "product-name", LocalEnvProvider.UNMODIFIED); timeoutMillis = 123 * 1000L; outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); @@ -426,4 +427,24 @@ public class LocalSpawnRunnerTest { // This would throw if the runner called prefetchFiles(). runner.exec(spawn, policy); } + + @Test + public void checkLocalEnvProviderCalled() throws Exception { + Subprocess.Factory factory = mock(Subprocess.Factory.class); + when(factory.create(any())).thenReturn(new FinishedSubprocess(0)); + SubprocessBuilder.setSubprocessFactory(factory); + LocalEnvProvider localEnvProvider = mock(LocalEnvProvider.class); + + LocalExecutionOptions options = Options.getDefaults(LocalExecutionOptions.class); + LocalSpawnRunner runner = new LocalSpawnRunner( + logger, execCount, fs.getPath("/execroot"), ActionInputPrefetcher.NONE, options, + resourceManager, USE_WRAPPER, "product-name", localEnvProvider); + + timeoutMillis = 123 * 1000L; + outErr = new FileOutErr(fs.getPath("/out/stdout"), fs.getPath("/out/stderr")); + + runner.exec(SIMPLE_SPAWN, policy); + verify(localEnvProvider) + .rewriteLocalEnv(any(), eq(fs.getPath("/execroot")), eq("product-name")); + } } diff --git a/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java b/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java index d8f54b637c..5f437637f3 100644 --- a/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java +++ b/src/test/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategyTest.java @@ -38,7 +38,6 @@ import com.google.devtools.build.lib.exec.BlazeExecutor; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.SingleBuildFileCache; import com.google.devtools.build.lib.integration.util.IntegrationMock; -import com.google.devtools.build.lib.rules.apple.AppleConfiguration; import com.google.devtools.build.lib.testutil.TestConstants; import com.google.devtools.build.lib.testutil.TestFileOutErr; import com.google.devtools.build.lib.testutil.TestUtils; @@ -212,29 +211,4 @@ public class StandaloneSpawnStrategyTest { assertThat(err()).isEqualTo("Oops!\n"); assertThat(out()).isEmpty(); } - - // Test an action with environment variables set indicating an action running on a darwin host - // system. Such actions should fail given the fact that these tests run on a non darwin - // architecture. - @Test - public void testIOSEnvironmentOnNonDarwin() throws Exception { - if (OS.getCurrent() == OS.DARWIN) { - return; - } - Spawn spawn = new BaseSpawn.Local( - Arrays.asList("/bin/sh", "-c", "echo $SDKROOT"), - ImmutableMap.<String, String>of(AppleConfiguration.APPLE_SDK_VERSION_ENV_NAME, "8.4", - AppleConfiguration.APPLE_SDK_PLATFORM_ENV_NAME, "iPhoneSimulator"), - new ActionsTestUtil.NullAction(), - ResourceSet.ZERO); - - try { - run(spawn); - fail("action should fail due to being unable to resolve SDKROOT"); - } catch (ExecException e) { - assertThat(e) - .hasMessageThat() - .contains("Cannot locate iOS SDK on non-darwin operating system"); - } - } } |