diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/exec')
10 files changed, 1043 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java new file mode 100644 index 0000000000..0e484f749a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/AlwaysOutOfDateAction.java @@ -0,0 +1,21 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +/** + * Marker interface for actions that must be run unconditionally. + */ +public interface AlwaysOutOfDateAction { + +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java new file mode 100644 index 0000000000..84f3aef20d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/CheckUpToDateFilter.java @@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.rules.test.TestRunnerAction; + +/** + * Class implements --check_???_up_to_date execution filter predicate + * that prevents certain actions from being executed (thus aborting + * the build if action is not up-to-date). + */ +public final class CheckUpToDateFilter implements Predicate<Action> { + + /** + * Determines an execution filter based on the --check_up_to_date and + * --check_tests_up_to_date options. Returns a singleton if possible. + */ + public static Predicate<Action> fromOptions(ExecutionOptions options) { + if (!options.testCheckUpToDate && !options.checkUpToDate) { + return Predicates.alwaysTrue(); + } + return new CheckUpToDateFilter(options); + } + + private final boolean allowBuildActionExecution; + private final boolean allowTestActionExecution; + + /** + * Creates new execution filter based on --check_up_to_date and + * --check_tests_up_to_date options. + */ + private CheckUpToDateFilter(ExecutionOptions options) { + // If we want to check whether test is up-to-date, we should disallow + // test execution. + this.allowTestActionExecution = !options.testCheckUpToDate; + + // Build action execution should be prohibited in two cases - if we are + // checking whether build is up-to-date or if we are checking that tests + // are up-to-date (and test execution is not allowed). + this.allowBuildActionExecution = allowTestActionExecution && !options.checkUpToDate; + } + + /** + * @return true if actions' execution is allowed, false - otherwise + */ + @Override + public boolean apply(Action action) { + if (action instanceof AlwaysOutOfDateAction) { + // Always allow fileset manifest action to execute because it identifies files included + // in the fileset during execution time. + return true; + } else if (action instanceof TestRunnerAction) { + return allowTestActionExecution; + } else { + return allowBuildActionExecution; + } + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/Digest.java b/src/main/java/com/google/devtools/build/lib/exec/Digest.java new file mode 100644 index 0000000000..4262711c39 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/Digest.java @@ -0,0 +1,182 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.cache.VirtualActionInput; +import com.google.devtools.build.lib.util.Pair; +import com.google.protobuf.ByteString; +import com.google.protobuf.MessageLite; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * A utility class for obtaining MD5 digests. + * Digests are represented as 32 characters in lowercase ASCII. + */ +public class Digest { + + public static final ByteString EMPTY_DIGEST = fromContent(new byte[]{}); + + private Digest() { + } + + /** + * Get the digest from the given byte array. + * @param bytes the byte array. + * @return a digest. + */ + public static ByteString fromContent(byte[] bytes) { + MessageDigest md = newBuilder(); + md.update(bytes, 0, bytes.length); + return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest())); + } + + /** + * Get the digest from the given ByteBuffer. + * @param buffer the ByteBuffer. + * @return a digest. + */ + public static ByteString fromBuffer(ByteBuffer buffer) { + MessageDigest md = newBuilder(); + md.update(buffer); + return toByteString(BaseEncoding.base16().lowerCase().encode(md.digest())); + } + + /** + * Gets the digest of the given proto. + * + * @param proto a protocol buffer. + * @return the digest. + */ + public static ByteString fromProto(MessageLite proto) { + MD5OutputStream md5Stream = new MD5OutputStream(); + try { + proto.writeTo(md5Stream); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException: ", e); + } + return toByteString(md5Stream.getDigest()); + } + + /** + * Gets the digest and size of a given VirtualActionInput. + * + * @param input the VirtualActionInput. + * @return the digest and size. + */ + public static Pair<ByteString, Long> fromVirtualActionInput(VirtualActionInput input) + throws IOException { + CountingMD5OutputStream md5Stream = new CountingMD5OutputStream(); + input.writeTo(md5Stream); + ByteString digest = toByteString(md5Stream.getDigest()); + return Pair.of(digest, md5Stream.getSize()); + } + + /** + * A Sink that does an online MD5 calculation, which avoids forcing us to keep the entire + * proto message in memory. + */ + private static class MD5OutputStream extends OutputStream { + private final MessageDigest md = newBuilder(); + + @Override + public void write(int b) { + md.update((byte) b); + } + + @Override + public void write(byte[] b, int off, int len) { + md.update(b, off, len); + } + + public String getDigest() { + return BaseEncoding.base16().lowerCase().encode(md.digest()); + } + } + + private static final class CountingMD5OutputStream extends MD5OutputStream { + private long size; + + @Override + public void write(int b) { + super.write(b); + size++; + } + + @Override + public void write(byte[] b, int off, int len) { + super.write(b, off, len); + size += len; + } + + public long getSize() { + return size; + } + } + + /** + * @param digest the digest to check. + * @return true iff digest is a syntactically legal digest. It must be 32 + * characters of hex with lowercase letters. + */ + public static boolean isDigest(ByteString digest) { + if (digest == null || digest.size() != 32) { + return false; + } + + for (byte b : digest) { + char c = (char) b; + if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + continue; + } + return false; + } + return true; + } + + /** + * @param digest the digest. + * @return true iff the digest is that of an empty file. + */ + public static boolean isEmpty(ByteString digest) { + return digest.equals(EMPTY_DIGEST); + } + + /** + * @return a new MD5 digest builder. + */ + public static MessageDigest newBuilder() { + try { + return MessageDigest.getInstance("md5"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("MD5 not available"); + } + } + + /** + * Convert a String digest into a ByteString using ascii. + * @param digest the digest in ascii. + * @return the digest as a ByteString. + */ + public static ByteString toByteString(String digest) { + return ByteString.copyFrom(digest.getBytes(US_ASCII)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java new file mode 100644 index 0000000000..58e360bfe0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java @@ -0,0 +1,195 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.rules.test.TestStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.Options; +import com.google.devtools.common.options.OptionsBase; + +import java.util.Map; + +/** + * Options affecting the execution phase of a build. + * + * These options are interpreted by the BuildTool to choose an Executor to + * be used for the build. + * + * Note: from the user's point of view, the characteristic function of this + * set of options is indistinguishable from that of the BuildRequestOptions: + * they are all per-request. The difference is only apparent in the + * implementation: these options are used only by the lib.exec machinery, which + * affects how C++ and Java compilation occur. (The BuildRequestOptions + * contain a mixture of "semantic" options affecting the choice of targets to + * build, and "non-semantic" options affecting the lib.actions machinery.) + * Ideally, the user would be unaware of the difference. For now, the usage + * strings are identical modulo "part 1", "part 2". + */ +public class ExecutionOptions extends OptionsBase { + + public static final ExecutionOptions DEFAULTS = Options.getDefaults(ExecutionOptions.class); + + @Option(name = "verbose_failures", + defaultValue = "false", + category = "verbosity", + help = "If a command fails, print out the full command line.") + public boolean verboseFailures; + + @Option(name = "subcommands", + abbrev = 's', + defaultValue = "false", + category = "verbosity", + help = "Display the subcommands executed during a build.") + public boolean showSubcommands; + + @Option(name = "check_up_to_date", + defaultValue = "false", + category = "what", + help = "Don't perform the build, just check if it is up-to-date. If all targets are " + + "up-to-date, the build completes successfully. If any step needs to be executed " + + "an error is reported and the build fails.") + public boolean checkUpToDate; + + @Option(name = "check_tests_up_to_date", + defaultValue = "false", + category = "testing", + implicitRequirements = { "--check_up_to_date" }, + help = "Don't run tests, just check if they are up-to-date. If all tests results are " + + "up-to-date, the testing completes successfully. If any test needs to be built or " + + "executed, an error is reported and the testing fails. This option implies " + + "--check_up_to_date behavior." + ) + public boolean testCheckUpToDate; + + @Option(name = "test_strategy", + defaultValue = "", + category = "testing", + help = "Specifies which strategy to use when running tests.") + public String testStrategy; + + @Option(name = "test_keep_going", + defaultValue = "true", + category = "testing", + help = "When disabled, any non-passing test will cause the entire build to stop. By default " + + "all tests are run, even if some do not pass.") + public boolean testKeepGoing; + + @Option(name = "runs_per_test_detects_flakes", + defaultValue = "false", + category = "testing", + help = "If true, any shard in which at least one run/attempt passes and at least one " + + "run/attempt fails gets a FLAKY status.") + public boolean runsPerTestDetectsFlakes; + + @Option(name = "flaky_test_attempts", + defaultValue = "default", + category = "testing", + converter = TestStrategy.TestAttemptsConverter.class, + help = "Each test will be retried up to the specified number of times in case of any test " + + "failure. Tests that required more than one attempt to pass would be marked as " + + "'FLAKY' in the test summary. If this option is set, it should specify an int N or the " + + "string 'default'. If it's an int, then all tests will be run up to N times. If it is " + + "not specified or its value is 'default', then only a single test attempt will be made " + + "for regular tests and three for tests marked explicitly as flaky by their rule " + + "(flaky=1 attribute).") + public int testAttempts; + + @Option(name = "test_tmpdir", + defaultValue = "null", + category = "testing", + converter = OptionsUtils.PathFragmentConverter.class, + help = "Specifies the base temporary directory for 'blaze test' to use.") + public PathFragment testTmpDir; + + @Option(name = "test_output", + defaultValue = "summary", + category = "testing", + converter = TestStrategy.TestOutputFormat.Converter.class, + help = "Specifies desired output mode. Valid values are 'summary' to " + + "output only test status summary, 'errors' to also print test logs " + + "for failed tests, 'all' to print logs for all tests and 'streamed' " + + "to output logs for all tests in real time (this will force tests " + + "to be executed locally one at a time regardless of --test_strategy " + + "value).") + public TestOutputFormat testOutput; + + @Option(name = "test_summary", + defaultValue = "short", + category = "testing", + converter = TestStrategy.TestSummaryFormat.Converter.class, + help = "Specifies the desired format ot the test summary. Valid values " + + "are 'short' to print information only about tests executed, " + + "'terse', to print information only about unsuccessful tests," + + "'detailed' to print detailed information about failed test cases, " + + "and 'none' to omit the summary.") + public TestSummaryFormat testSummary; + + @Option(name = "test_timeout", + defaultValue = "-1", + category = "testing", + converter = TestTimeout.TestTimeoutConverter.class, + help = "Override the default test timeout values for test timeouts (in secs). If a single " + + "positive integer value is specified it will override all categories. If 4 comma-" + + "separated integers are specified, they will override the timeouts for short, " + + "moderate, long and eternal (in that order). In either form, a value of -1 tells blaze " + + "to use its default timeouts for that category.") + public Map<TestTimeout, Integer> testTimeout; + + + @Option(name = "resource_autosense", + defaultValue = "false", + category = "strategy", + help = "Periodically (every 3 seconds) poll system CPU load and available memory " + + "and allow execution of build commands if system has sufficient idle CPU and " + + "free RAM resources. By default this option is disabled, and Blaze will rely on " + + "approximation algorithms based on the total amount of available memory and number " + + "of CPU cores.") + public boolean useResourceAutoSense; + + @Option(name = "ram_utilization_factor", + defaultValue = "67", + category = "strategy", + help = "Specify what percentage of the system's RAM Blaze should try to use for its " + + "subprocesses. " + + "This option affects how many processes Blaze will try to run in parallel. " + + "If you run several Blaze builds in parallel, using a lower value for " + + "this option may avoid thrashing and thus improve overall throughput. " + + "Using a value higher than the default is NOT recommended. " + + "Note that Blaze's estimates are very coarse, so the actual RAM usage may be much " + + "higher or much lower than specified. " + + "Note also that this option does not affect the amount of memory that the Blaze " + + "server itself will use. " + + "Also, this option has no effect if --resource_autosense is enabled." + ) + public int ramUtilizationPercentage; + + @Option(name = "local_resources", + defaultValue = "null", + category = "strategy", + help = "Explicitly set amount of local resources available to Blaze. " + + "By default, Blaze will query system configuration to estimate amount of RAM (in MB) " + + "and number of CPU cores available for the locally executed build actions. It would also " + + "assume default I/O capabilities of the local workstation (1.0). This options allows to " + + "explicitly set all 3 values. Note, that if this option is used, Blaze will ignore " + + "both --ram_utilization_factor and --resource_autosense values.", + converter = ResourceSet.ResourceSetConverter.class + ) + public ResourceSet availableResources; +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java new file mode 100644 index 0000000000..5dc9914415 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/FileWriteStrategy.java @@ -0,0 +1,73 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.exec; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** + * A strategy for executing an {@link AbstractFileWriteAction}. + */ +@ExecutionStrategy(name = { "local" }, contextType = FileWriteActionContext.class) +public final class FileWriteStrategy implements FileWriteActionContext { + + public static final Class<FileWriteStrategy> TYPE = FileWriteStrategy.class; + + public FileWriteStrategy() { + } + + @Override + public void exec(Executor executor, AbstractFileWriteAction action, FileOutErr outErr, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + EventHandler reporter = executor == null ? null : executor.getEventHandler(); + try { + Path outputPath = Iterables.getOnlyElement(action.getOutputs()).getPath(); + try (OutputStream out = new BufferedOutputStream(outputPath.getOutputStream())) { + action.newDeterministicWriter(reporter, executor).writeOutputFile(out); + } + if (action.makeExecutable()) { + outputPath.setExecutable(true); + } + } catch (IOException e) { + throw new EnvironmentalExecException("failed to create file '" + + Iterables.getOnlyElement(action.getOutputs()).prettyPrint() + + "' due to I/O error: " + e.getMessage(), e); + } + } + + @Override + public ResourceSet estimateResourceConsumption(AbstractFileWriteAction action) { + return action.estimateResourceConsumptionLocal(); + } + + @Override + public String strategyLocality(AbstractFileWriteAction action) { + return "local"; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/OutputService.java b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java new file mode 100644 index 0000000000..88d9b942b7 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/OutputService.java @@ -0,0 +1,122 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.BatchStat; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * An OutputService retains control over the Blaze output tree, and provides a higher level of + * abstraction compared to the VFS layer. + * + * <p>Higher-level facilities include batch statting, cleaning the output tree, creating symlink + * trees, and out-of-band insertion of metadata into the tree. + */ +public interface OutputService { + + /** + * @return the name of filesystem, akin to what you might see in /proc/mounts + */ + String getFilesSystemName(); + + /** + * @return true if the output service uses FUSE + */ + boolean usesFuse(); + + /** + * @return a human-readable, one word name for the service + */ + String getName(); + + /** + * Start the build. + * + * @throws BuildFailedException if build preparation failed + * @throws InterruptedException + */ + void startBuild() throws BuildFailedException, AbruptExitException, InterruptedException; + + /** + * Finish the build. + * + * @param buildSuccessful iff build was successful + * @throws BuildFailedException on failure + */ + void finalizeBuild(boolean buildSuccessful) throws BuildFailedException, AbruptExitException; + + /** + * Stages the given tool from the package path, possibly copying it to local disk. + * + * @param tool target representing the tool to stage + * @return a Path pointing to the staged target + */ + Path stageTool(Target tool) throws IOException; + + /** + * @return the name of the workspace this output service controls. + */ + String getWorkspace(); + + /** + * @return the BatchStat instance or null. + */ + BatchStat getBatchStatter(); + + /** + * @return true iff createSymlinkTree() is available. + */ + boolean canCreateSymlinkTree(); + + /** + * Creates the symlink tree + * + * @param inputPath the input manifest + * @param outputPath the output manifest + * @param filesetTree is true iff we're constructing a Fileset + * @param symlinkTreeRoot the symlink tree root, relative to the execRoot + * @throws ExecException on failure + * @throws InterruptedException + */ + void createSymlinkTree(Path inputPath, Path outputPath, boolean filesetTree, + PathFragment symlinkTreeRoot) throws ExecException, InterruptedException; + + /** + * Cleans the entire output tree. + * + * @throws ExecException on failure + * @throws InterruptedException + */ + void clean() throws ExecException, InterruptedException; + + /** + * @param file the File + * @return true iff the file actually lives on a remote server + */ + boolean isRemoteFile(Path file); + + /** + * @param path a fully-resolved path + * @return true iff path is under this output service's control + */ + boolean resolvedPathUnderTree(Path path); +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java new file mode 100644 index 0000000000..8ec1e51583 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java @@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Maps; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.DigestOfDirectoryException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.protobuf.ByteString; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * An in-memory cache to ensure we do I/O for source files only once during a single build. + * + * <p>Simply maintains a two-way cached mapping from digest <--> filename that may be populated + * only once. + */ +@ThreadSafe +public class SingleBuildFileCache implements ActionInputFileCache { + + private final String cwd; + private final FileSystem fs; + + public SingleBuildFileCache(String cwd, FileSystem fs) { + this.fs = Preconditions.checkNotNull(fs); + this.cwd = Preconditions.checkNotNull(cwd); + } + + // If we can't get the digest, we store the exception. This avoids extra file IO for files + // that are allowed to be missing, as we first check a likely non-existent content file + // first. Further we won't need to unwrap the exception in getDigest(). + private final LoadingCache<ActionInput, Pair<ByteString, IOException>> pathToDigest = + CacheBuilder.newBuilder() + // We default to 10 disk read threads, but we don't expect them all to edit the map + // simultaneously. + .concurrencyLevel(8) + // Even small-ish builds, as of 11/21/2011 typically have over 10k artifacts, so it's + // unlikely that this default will adversely affect memory in most cases. + .initialCapacity(10000) + .build(new CacheLoader<ActionInput, Pair<ByteString, IOException>>() { + @Override + public Pair<ByteString, IOException> load(ActionInput input) { + Path path = null; + try { + path = fs.getPath(fullPath(input)); + BaseEncoding hex = BaseEncoding.base16().lowerCase(); + ByteString digest = ByteString.copyFrom( + hex.encode(path.getMD5Digest()) + .getBytes(US_ASCII)); + pathToBytes.put(input, path.getFileSize()); + // Inject reverse mapping. Doing this unconditionally in getDigest() showed up + // as a hotspot in CPU profiling. + digestToPath.put(digest, input); + return Pair.of(digest, null); + } catch (IOException e) { + if (path != null && path.isDirectory()) { + pathToBytes.put(input, 0L); + return Pair.<ByteString, IOException>of(null, new DigestOfDirectoryException( + "Input is a directory: " + input.getExecPathString())); + } + + // Put value into size map to avoid trying to read file again later. + pathToBytes.put(input, 0L); + return Pair.of(null, e); + } + } + }); + + private final Map<ByteString, ActionInput> digestToPath = Maps.newConcurrentMap(); + + private final Map<ActionInput, Long> pathToBytes = Maps.newConcurrentMap(); + + @Nullable + @Override + public File getFileFromDigest(ByteString digest) { + ActionInput relPath = digestToPath.get(digest); + return relPath == null ? null : new File(fullPath(relPath)); + } + + @Override + public long getSizeInBytes(ActionInput input) throws IOException { + // TODO(bazel-team): this only works if pathToDigest has already been called. + Long sz = pathToBytes.get(input); + if (sz != null) { + return sz; + } + Path path = fs.getPath(fullPath(input)); + sz = path.getFileSize(); + pathToBytes.put(input, sz); + return sz; + } + + @Override + public ByteString getDigest(ActionInput input) throws IOException { + Pair<ByteString, IOException> result = pathToDigest.getUnchecked(input); + if (result.second != null) { + throw result.second; + } + return result.first; + } + + @Override + public boolean contentsAvailableLocally(ByteString digest) { + return digestToPath.containsKey(digest); + } + + /** + * Creates a File object that refers to fileName, if fileName is an absolute path. Otherwise, + * returns a File object that refers to the fileName appended to the (absolute) current working + * directory. + */ + private String fullPath(ActionInput input) { + String relPath = input.getExecPathString(); + return relPath.startsWith("/") ? relPath : new File(cwd, relPath).getPath(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java new file mode 100644 index 0000000000..40fed778c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SourceManifestActionContextImpl.java @@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.analysis.SourceManifestAction; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * A context for {@link SourceManifestAction} that uses the runtime to determine + * the workspace suffix. + */ +@ExecutionStrategy(contextType = SourceManifestAction.Context.class) +public class SourceManifestActionContextImpl implements SourceManifestAction.Context { + private final PathFragment runfilesPrefix; + + public SourceManifestActionContextImpl(PathFragment runfilesPrefix) { + this.runfilesPrefix = runfilesPrefix; + } + + @Override + public PathFragment getRunfilesPrefix() { + return runfilesPrefix; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java new file mode 100644 index 0000000000..6127ceec80 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeHelper.java @@ -0,0 +1,137 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * Helper class responsible for the symlink tree creation. + * Used to generate runfiles and fileset symlink farms. + */ +public final class SymlinkTreeHelper { + + private static final String BUILD_RUNFILES = "build-runfiles" + OsUtils.executableExtension(); + + /** + * These actions run faster overall when serialized, because most of their + * cost is in the ext2 block allocator, and there's less seeking required if + * their directory creations get non-interleaved allocations. So we give them + * a huge resource cost. + */ + public static final ResourceSet RESOURCE_SET = new ResourceSet(1000, 0.5, 0.75); + + private final PathFragment inputManifest; + private final PathFragment symlinkTreeRoot; + private final boolean filesetTree; + + /** + * Creates SymlinkTreeHelper instance. Can be used independently of + * SymlinkTreeAction. + * + * @param inputManifest exec path to the input runfiles manifest + * @param symlinkTreeRoot exec path to the symlink tree location + * @param filesetTree true if this is fileset symlink tree, + * false if this is a runfiles symlink tree. + */ + public SymlinkTreeHelper(PathFragment inputManifest, PathFragment symlinkTreeRoot, + boolean filesetTree) { + this.inputManifest = inputManifest; + this.symlinkTreeRoot = symlinkTreeRoot; + this.filesetTree = filesetTree; + } + + public PathFragment getSymlinkTreeRoot() { return symlinkTreeRoot; } + + /** + * Creates a symlink tree using a CommandBuilder. This means that the symlink + * tree will always be present on the developer's workstation. Useful when + * running commands locally. + * + * Warning: this method REALLY executes the command on the box Blaze was + * run on, without any kind of synchronization, locking, or anything else. + * + * @param config the configuration that is used for creating the symlink tree. + * @throws CommandException + */ + public void createSymlinksUsingCommand(Path execRoot, + BuildConfiguration config, BinTools binTools) throws CommandException { + List<String> argv = getSpawnArgumentList(execRoot, binTools); + + CommandBuilder builder = new CommandBuilder(); + builder.addArgs(argv); + builder.setWorkingDir(execRoot); + builder.build().execute(); + } + + /** + * Creates symlink tree using appropriate method. At this time tree + * always created using build-runfiles helper application. + * + * Note: method may try to acquire resources - meaning that it would + * block for undetermined period of time. If it is interrupted during + * that wait, ExecException will be thrown but interrupted bit will be + * preserved. + * + * @param action action instance that requested symlink tree creation + * @param actionExecutionContext Services that are in the scope of the action. + */ + public void createSymlinks(AbstractAction action, ActionExecutionContext actionExecutionContext, + BinTools binTools) throws ExecException, InterruptedException { + List<String> args = getSpawnArgumentList( + actionExecutionContext.getExecutor().getExecRoot(), binTools); + try { + ResourceManager.instance().acquireResources(action, RESOURCE_SET); + actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()).exec( + new BaseSpawn.Local(args, ImmutableMap.<String, String>of(), action), + actionExecutionContext); + } finally { + ResourceManager.instance().releaseResources(action, RESOURCE_SET); + } + } + + /** + * Returns the complete argument list build-runfiles has to be called with. + */ + private List<String> getSpawnArgumentList(Path execRoot, BinTools binTools) { + List<String> args = Lists.newArrayList( + execRoot.getRelative(binTools.getExecPath(BUILD_RUNFILES)) + .getPathString()); + + if (filesetTree) { + args.add("--allow_relative"); + args.add("--use_metadata"); + } + + args.add(inputManifest.getPathString()); + args.add(symlinkTreeRoot.getPathString()); + + return args; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java new file mode 100644 index 0000000000..d9470e4157 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SymlinkTreeStrategy.java @@ -0,0 +1,60 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.SymlinkTreeAction; +import com.google.devtools.build.lib.analysis.SymlinkTreeActionContext; +import com.google.devtools.build.lib.analysis.config.BinTools; + +/** + * Implements SymlinkTreeAction by using the output service or by running an embedded script to + * create the symlink tree. + */ +@ExecutionStrategy(contextType = SymlinkTreeActionContext.class) +public final class SymlinkTreeStrategy implements SymlinkTreeActionContext { + private final OutputService outputService; + private final BinTools binTools; + + public SymlinkTreeStrategy(OutputService outputService, BinTools binTools) { + this.outputService = outputService; + this.binTools = binTools; + } + + @Override + public void createSymlinks(SymlinkTreeAction action, + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + SymlinkTreeHelper helper = new SymlinkTreeHelper( + action.getInputManifest().getExecPath(), + action.getOutputManifest().getExecPath().getParentDirectory(), action.isFilesetTree()); + if (outputService != null && outputService.canCreateSymlinkTree()) { + outputService.createSymlinkTree(action.getInputManifest().getPath(), + action.getOutputManifest().getPath(), + action.isFilesetTree(), helper.getSymlinkTreeRoot()); + } else { + helper.createSymlinks(action, actionExecutionContext, binTools); + } + } catch (ExecException e) { + throw e.toActionExecutionException( + action.getProgressMessage(), executor.getVerboseFailures(), action); + } + } +} |