diff options
author | 2015-02-25 16:45:20 +0100 | |
---|---|---|
committer | 2015-02-25 16:45:20 +0100 | |
commit | d08b27fa9701fecfdb69e1b0d1ac2459efc2129b (patch) | |
tree | 5d50963026239ca5aebfb47ea5b8db7e814e57c8 /src/main/java/com/google/devtools/build/lib/buildtool |
Update from Google.
--
MOE_MIGRATED_REVID=85702957
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/buildtool')
16 files changed, 3113 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java new file mode 100644 index 0000000000..95fbde5362 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java @@ -0,0 +1,532 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Converters.RangeConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +/** + * A BuildRequest represents a single invocation of the build tool by a user. + * A request specifies a list of targets to be built for a single + * configuration, a pair of output/error streams, and additional options such + * as --keep_going, --jobs, etc. + */ +public class BuildRequest implements OptionsClassProvider { + private static final String DEFAULT_SYMLINK_PREFIX_MARKER = "...---:::@@@DEFAULT@@@:::--..."; + + /** + * A converter for symlink prefixes that defaults to {@code Constants.PRODUCT_NAME} and a + * minus sign if the option is not given. + * + * <p>Required because you cannot specify a non-constant value in annotation attributes. + */ + public static class SymlinkPrefixConverter implements Converter<String> { + @Override + public String convert(String input) throws OptionsParsingException { + return input.equals(DEFAULT_SYMLINK_PREFIX_MARKER) + ? Constants.PRODUCT_NAME + "-" + : input; + } + + @Override + public String getTypeDescription() { + return "a string"; + } + } + + /** + * Options interface--can be used to parse command-line arguments. + * + * See also ExecutionOptions; from the user's point of view, there's no + * qualitative difference between these two sets of options. + */ + public static class BuildRequestOptions extends OptionsBase { + + /* "Execution": options related to the execution of a build: */ + + @Option(name = "jobs", + abbrev = 'j', + defaultValue = "200", + category = "strategy", + help = "The number of concurrent jobs to run. " + + "0 means build sequentially. Values above " + MAX_JOBS + + " are not allowed.") + public int jobs; + + @Option(name = "progress_report_interval", + defaultValue = "0", + category = "verbosity", + converter = ProgressReportIntervalConverter.class, + help = "The number of seconds to wait between two reports on" + + " still running jobs. The default value 0 means to use" + + " the default 10:30:60 incremental algorithm.") + public int progressReportInterval; + + @Option(name = "show_builder_stats", + defaultValue = "false", + category = "verbosity", + help = "If set, parallel builder will report worker-related statistics.") + public boolean useBuilderStatistics; + + @Option(name = "explain", + defaultValue = "null", + category = "verbosity", + converter = OptionsUtils.PathFragmentConverter.class, + help = "Causes Blaze to explain each executed step of the build. " + + "The explanation is written to the specified log file.") + public PathFragment explanationPath; + + @Option(name = "verbose_explanations", + defaultValue = "false", + category = "verbosity", + help = "Increases the verbosity of the explanations issued if --explain is enabled. " + + "Has no effect if --explain is not enabled.") + public boolean verboseExplanations; + + @Deprecated + @Option(name = "dump_makefile", + defaultValue = "false", + category = "undocumented", + help = "this flag has no effect.") + public boolean dumpMakefile; + + @Deprecated + @Option(name = "dump_action_graph", + defaultValue = "false", + category = "undocumented", + help = "this flag has no effect.") + + public boolean dumpActionGraph; + + @Deprecated + @Option(name = "dump_action_graph_for_package", + allowMultiple = true, + defaultValue = "", + category = "undocumented", + help = "this flag has no effect.") + public List<String> dumpActionGraphForPackage = new ArrayList<>(); + + @Deprecated + @Option(name = "dump_action_graph_with_middlemen", + defaultValue = "true", + category = "undocumented", + help = "this flag has no effect.") + public boolean dumpActionGraphWithMiddlemen; + + @Deprecated + @Option(name = "dump_providers", + defaultValue = "false", + category = "undocumented", + help = "This is a no-op.") + public boolean dumpProviders; + + @Option(name = "incremental_builder", + deprecationWarning = "incremental_builder is now a no-op and will be removed in an" + + " upcoming Blaze release", + defaultValue = "true", + category = "strategy", + help = "Enables an incremental builder aimed at faster " + + "incremental builds. Currently it has the greatest effect on null" + + "builds.") + public boolean useIncrementalDependencyChecker; + + @Deprecated + @Option(name = "dump_targets", + defaultValue = "null", + category = "undocumented", + help = "this flag has no effect.") + public String dumpTargets; + + @Deprecated + @Option(name = "dump_host_deps", + defaultValue = "true", + category = "undocumented", + help = "Deprecated") + public boolean dumpHostDeps; + + @Deprecated + @Option(name = "dump_to_stdout", + defaultValue = "false", + category = "undocumented", + help = "Deprecated") + public boolean dumpToStdout; + + @Option(name = "analyze", + defaultValue = "true", + category = "undocumented", + help = "Execute the analysis phase; this is the usual behaviour. " + + "Specifying --noanalyze causes the build to stop before starting the " + + "analysis phase, returning zero iff the package loading completed " + + "successfully; this mode is useful for testing.") + public boolean performAnalysisPhase; + + @Option(name = "build", + defaultValue = "true", + category = "what", + help = "Execute the build; this is the usual behaviour. " + + "Specifying --nobuild causes the build to stop before executing the " + + "build actions, returning zero iff the package loading and analysis " + + "phases completed successfully; this mode is useful for testing " + + "those phases.") + public boolean performExecutionPhase; + + @Option(name = "compile_only", + defaultValue = "false", + category = "what", + help = "If specified, Blaze will only build files that are generated by lightweight " + + "compilation actions, skipping more expensive build steps (such as linking).") + public boolean compileOnly; + + @Option(name = "compilation_prerequisites_only", + defaultValue = "false", + category = "what", + help = "If specified, Blaze will only build files that are prerequisites to compilation " + + "of the given target (for example, generated source files and headers) without " + + "building the target itself. This flag is ignored if --compile_only is enabled.") + public boolean compilationPrerequisitesOnly; + + @Option(name = "output_groups", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + defaultValue = "", + category = "undocumented", + help = "Specifies, which output groups of the top-level target to build.") + public List<String> outputGroups; + + @Option(name = "show_result", + defaultValue = "1", + category = "verbosity", + help = "Show the results of the build. For each " + + "target, state whether or not it was brought up-to-date, and if " + + "so, a list of output files that were built. The printed files " + + "are convenient strings for copy+pasting to the shell, to " + + "execute them.\n" + + "This option requires an integer argument, which " + + "is the threshold number of targets above which result " + + "information is not printed. " + + "Thus zero causes suppression of the message and MAX_INT " + + "causes printing of the result to occur always. The default is one.") + public int maxResultTargets; + + @Option(name = "announce", + defaultValue = "false", + category = "verbosity", + help = "Deprecated. No-op.", + deprecationWarning = "This option is now deprecated and is a no-op") + public boolean announce; + + @Option(name = "symlink_prefix", + defaultValue = DEFAULT_SYMLINK_PREFIX_MARKER, + converter = SymlinkPrefixConverter.class, + category = "misc", + help = "The prefix that is prepended to any of the convenience symlinks that are created " + + "after a build. If '/' is passed, then no symlinks are created and no warning is " + + "emitted." + ) + public String symlinkPrefix; + + @Option(name = "experimental_multi_cpu", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + defaultValue = "", + category = "semantics", + help = "This flag allows specifying multiple target CPUs. If this is specified, " + + "the --cpu option is ignored.") + public List<String> multiCpus; + + @Option(name = "experimental_check_output_files", + defaultValue = "true", + category = "undocumented", + help = "Check for modifications made to the output files of a build. Consider setting " + + "this flag to false to see the effect on incremental build times.") + public boolean checkOutputFiles; + } + + /** + * Converter for progress_report_interval: [0, 3600]. + */ + public static class ProgressReportIntervalConverter extends RangeConverter { + public ProgressReportIntervalConverter() { + super(0, 3600); + } + } + + private static final int MAX_JOBS = 2000; + private static final int JOBS_TOO_HIGH_WARNING = 1000; + + private final UUID id; + private final LoadingCache<Class<? extends OptionsBase>, Optional<OptionsBase>> optionsCache; + + /** A human-readable description of all the non-default option settings. */ + private final String optionsDescription; + + /** + * The name of the Blaze command that the user invoked. + * Used for --announce. + */ + private final String commandName; + + private final OutErr outErr; + private final List<String> targets; + + private long startTimeMillis = 0; // milliseconds since UNIX epoch. + + private boolean runningInEmacs = false; + private boolean runTests = false; + + private static final List<Class<? extends OptionsBase>> MANDATORY_OPTIONS = ImmutableList.of( + BuildRequestOptions.class, + PackageCacheOptions.class, + LoadingPhaseRunner.Options.class, + BuildView.Options.class, + ExecutionOptions.class); + + private BuildRequest(String commandName, + final OptionsProvider options, + final OptionsProvider startupOptions, + List<String> targets, + OutErr outErr, + UUID id, + long startTimeMillis) { + this.commandName = commandName; + this.optionsDescription = OptionsUtils.asShellEscapedString(options); + this.outErr = outErr; + this.targets = targets; + this.id = id; + this.startTimeMillis = startTimeMillis; + this.optionsCache = CacheBuilder.newBuilder() + .build(new CacheLoader<Class<? extends OptionsBase>, Optional<OptionsBase>>() { + @Override + public Optional<OptionsBase> load(Class<? extends OptionsBase> key) throws Exception { + OptionsBase result = options.getOptions(key); + if (result == null && startupOptions != null) { + result = startupOptions.getOptions(key); + } + + return Optional.fromNullable(result); + } + }); + + for (Class<? extends OptionsBase> optionsClass : MANDATORY_OPTIONS) { + Preconditions.checkNotNull(getOptions(optionsClass)); + } + } + + /** + * Returns a unique identifier that universally identifies this build. + */ + public UUID getId() { + return id; + } + + /** + * Returns the name of the Blaze command that the user invoked. + */ + public String getCommandName() { + return commandName; + } + + /** + * Set to true if this build request was initiated by Emacs. + * (Certain output formatting may be necessary.) + */ + public void setRunningInEmacs() { + runningInEmacs = true; + } + + boolean isRunningInEmacs() { + return runningInEmacs; + } + + /** + * Enables test execution for this build request. + */ + public void setRunTests() { + runTests = true; + } + + /** + * Returns true if tests should be run by the build tool. + */ + public boolean shouldRunTests() { + return runTests; + } + + /** + * Returns the (immutable) list of targets to build in commandline + * form. + */ + public List<String> getTargets() { + return targets; + } + + /** + * Returns the output/error streams to which errors and progress messages + * should be sent during the fulfillment of this request. + */ + public OutErr getOutErr() { + return outErr; + } + + @Override + @SuppressWarnings("unchecked") + public <T extends OptionsBase> T getOptions(Class<T> clazz) { + try { + return (T) optionsCache.get(clazz).orNull(); + } catch (ExecutionException e) { + throw new IllegalStateException(e); + } + } + + /** + * Returns the set of command-line options specified for this request. + */ + public BuildRequestOptions getBuildOptions() { + return getOptions(BuildRequestOptions.class); + } + + /** + * Returns the set of options related to the loading phase. + */ + public PackageCacheOptions getPackageCacheOptions() { + return getOptions(PackageCacheOptions.class); + } + + /** + * Returns the set of options related to the loading phase. + */ + public LoadingPhaseRunner.Options getLoadingOptions() { + return getOptions(LoadingPhaseRunner.Options.class); + } + + /** + * Returns the set of command-line options related to the view specified for + * this request. + */ + public BuildView.Options getViewOptions() { + return getOptions(BuildView.Options.class); + } + + /** + * Returns the human-readable description of the non-default options + * for this build request. + */ + public String getOptionsDescription() { + return optionsDescription; + } + + /** + * Return the time (according to System.currentTimeMillis()) at which the + * service of this request was started. + */ + public long getStartTime() { + return startTimeMillis; + } + + /** + * Validates the options for this BuildRequest. + * + * <p>Issues warnings or throws {@code InvalidConfigurationException} for option settings that + * conflict. + * + * @return list of warnings + */ + public List<String> validateOptions() throws InvalidConfigurationException { + List<String> warnings = new ArrayList<>(); + // Validate "jobs". + int jobs = getBuildOptions().jobs; + if (jobs < 0 || jobs > MAX_JOBS) { + throw new InvalidConfigurationException(String.format( + "Invalid parameter for --jobs: %d. Only values 0 <= jobs <= %d are allowed.", jobs, + MAX_JOBS)); + } + if (jobs > JOBS_TOO_HIGH_WARNING) { + warnings.add( + String.format("High value for --jobs: %d. You may run into memory issues", jobs)); + } + + // Validate other BuildRequest options. + if (getBuildOptions().verboseExplanations && getBuildOptions().explanationPath == null) { + warnings.add("--verbose_explanations has no effect when --explain=<file> is not enabled"); + } + if (getBuildOptions().compileOnly && getBuildOptions().compilationPrerequisitesOnly) { + throw new InvalidConfigurationException( + "--compile_only and --compilation_prerequisites_only are not compatible"); + } + + return warnings; + } + + /** Creates a new TopLevelArtifactContext from this build request. */ + public TopLevelArtifactContext getTopLevelArtifactContext() { + return new TopLevelArtifactContext(getCommandName(), + getBuildOptions().compileOnly, getBuildOptions().compilationPrerequisitesOnly, + getOptions(ExecutionOptions.class).testStrategy.equals("exclusive"), + ImmutableSet.<String>copyOf(getBuildOptions().outputGroups), shouldRunTests()); + } + + public String getSymlinkPrefix() { + return getBuildOptions().symlinkPrefix; + } + + public ImmutableSortedSet<String> getMultiCpus() { + return ImmutableSortedSet.copyOf(getBuildOptions().multiCpus); + } + + public static BuildRequest create(String commandName, OptionsProvider options, + OptionsProvider startupOptions, + List<String> targets, OutErr outErr, UUID commandId, long commandStartTime) { + + BuildRequest request = new BuildRequest(commandName, options, startupOptions, targets, outErr, + commandId, commandStartTime); + + // All this, just to pass a global boolean from the client to the server. :( + if (options.getOptions(BlazeCommandEventHandler.Options.class).runningInEmacs) { + request.setRunningInEmacs(); + } + + return request; + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java new file mode 100644 index 0000000000..22c36f831d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildResult.java @@ -0,0 +1,196 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.util.ExitCode; + +import java.util.Collection; +import java.util.Collections; + +import javax.annotation.Nullable; + +/** + * Contains information about the result of a build. While BuildRequest is immutable, this class is + * mutable. + */ +public final class BuildResult { + private long startTimeMillis = 0; // milliseconds since UNIX epoch. + private long stopTimeMillis = 0; + + private Throwable crash = null; + private boolean catastrophe = false; + private ExitCode exitCondition = ExitCode.BLAZE_INTERNAL_ERROR; + private Collection<ConfiguredTarget> actualTargets; + private Collection<ConfiguredTarget> testTargets; + private Collection<ConfiguredTarget> successfulTargets; + + public BuildResult(long startTimeMillis) { + this.startTimeMillis = startTimeMillis; + } + + /** + * Record the time (according to System.currentTimeMillis()) at which the + * service of this request was completed. + */ + public void setStopTime(long stopTimeMillis) { + this.stopTimeMillis = stopTimeMillis; + } + + /** + * Return the time (according to System.currentTimeMillis()) at which the + * service of this request was completed. + */ + public long getStopTime() { + return stopTimeMillis; + } + + /** + * Returns the elapsed time in seconds for the service of this request. Not + * defined for requests that have not been serviced. + */ + public double getElapsedSeconds() { + if (startTimeMillis == 0 || stopTimeMillis == 0) { + throw new IllegalStateException("BuildRequest has not been serviced"); + } + return (stopTimeMillis - startTimeMillis) / 1000.0; + } + + public void setExitCondition(ExitCode exitCondition) { + this.exitCondition = exitCondition; + } + + /** + * True iff the build request has been successfully completed. + */ + public boolean getSuccess() { + return exitCondition == ExitCode.SUCCESS; + } + + /** + * Gets the Blaze exit condition. + */ + public ExitCode getExitCondition() { + return exitCondition; + } + + /** + * Sets the RuntimeException / Error that induced a Blaze crash. + */ + public void setUnhandledThrowable(Throwable crash) { + Preconditions.checkState(crash == null || + ((crash instanceof RuntimeException) || (crash instanceof Error))); + this.crash = crash; + } + + /** + * Sets a "catastrophe": A build failure severe enough to halt a keep_going build. + */ + public void setCatastrophe() { + this.catastrophe = true; + } + + /** + * Was the build a "catastrophe": A build failure severe enough to halt a keep_going build. + */ + public boolean wasCatastrophe() { + return catastrophe; + } + + /** + * Gets the Blaze crash Throwable. Null if Blaze did not crash. + */ + public Throwable getUnhandledThrowable() { + return crash; + } + + /** + * @see #getActualTargets + */ + public void setActualTargets(Collection<ConfiguredTarget> actualTargets) { + this.actualTargets = actualTargets; + } + + /** + * Returns the actual set of targets which we attempted to build. This value + * is set during the build, after the target patterns have been parsed and + * resolved. If --keep_going is specified, this set may exclude targets that + * could not be found or successfully analyzed. It may be examined after the + * build. May be null even after the build, if there were errors in the + * loading or analysis phases. + */ + public Collection<ConfiguredTarget> getActualTargets() { + return actualTargets; + } + + /** + * @see #getTestTargets + */ + public void setTestTargets(@Nullable Collection<ConfiguredTarget> testTargets) { + this.testTargets = testTargets == null ? null : Collections.unmodifiableCollection(testTargets); + } + + /** + * Returns the actual unmodifiable collection of targets which we attempted to + * test. This value is set at the end of the build analysis phase, after the + * test target patterns have been parsed and resolved. If --keep_going is + * specified, this collection may exclude targets that could not be found or + * successfully analyzed. It may be examined after the build. May be null even + * after the build, if there were errors in the loading or analysis phases or + * if testing was not requested. + */ + public Collection<ConfiguredTarget> getTestTargets() { + return testTargets; + } + + /** + * @see #getSuccessfulTargets + */ + void setSuccessfulTargets(Collection<ConfiguredTarget> successfulTargets) { + this.successfulTargets = successfulTargets; + } + + /** + * Returns the set of targets which successfully built. This value + * is set at the end of the build, after the target patterns have been parsed + * and resolved and after attempting to build the targets. If --keep_going + * is specified, this set may exclude targets that could not be found or + * successfully analyzed, or could not be built. It may be examined after + * the build. May be null if the execution phase was not attempted, as + * may happen if there are errors in the loading phase, for example. + */ + public Collection<ConfiguredTarget> getSuccessfulTargets() { + return successfulTargets; + } + + /** For debugging. */ + @Override + @SuppressWarnings("deprecation") + public String toString() { + // We need to be compatible with Guava, so we use this, even though it is deprecated. + return Objects.toStringHelper(this) + .add("startTimeMillis", startTimeMillis) + .add("stopTimeMillis", stopTimeMillis) + .add("crash", crash) + .add("catastrophe", catastrophe) + .add("exitCondition", exitCondition) + .add("actualTargets", actualTargets) + .add("testTargets", testTargets) + .add("successfulTargets", successfulTargets) + .toString(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java new file mode 100644 index 0000000000..a27cc50433 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java @@ -0,0 +1,540 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.AnalysisPhaseCompleteEvent; +import com.google.devtools.build.lib.analysis.BuildInfoEvent; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult; +import com.google.devtools.build.lib.analysis.ConfigurationsCreatedEvent; +import com.google.devtools.build.lib.analysis.ConfiguredAttributeMapper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LicensesProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; +import com.google.devtools.build.lib.analysis.MakeEnvironmentEvent; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; +import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent; +import com.google.devtools.build.lib.cmdline.TargetParsingException; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.packages.InputFile; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.packages.License.DistributionType; +import com.google.devtools.build.lib.packages.PackageIdentifier; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.Callback; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Provides the bulk of the implementation of the 'blaze build' command. + * + * <p>The various concrete build command classes handle the command options and request + * setup, then delegate the handling of the request (the building of targets) to this class. + * + * <p>The main entry point is {@link #buildTargets}. + * + * <p>This class is always instantiated and managed as a singleton, being constructed and held by + * {@link BlazeRuntime}. This is so multiple kinds of build commands can share this single + * instance. + * + * <p>Most of analysis is handled in {@link BuildView}, and execution in {@link ExecutionTool}. + */ +public class BuildTool { + + private static final Logger LOG = Logger.getLogger(BuildTool.class.getName()); + + protected final BlazeRuntime runtime; + + /** + * Constructs a BuildTool. + * + * @param runtime a reference to the blaze runtime. + */ + public BuildTool(BlazeRuntime runtime) { + this.runtime = runtime; + } + + /** + * The crux of the build system. Builds the targets specified in the request using the specified + * Executor. + * + * <p>Performs loading, analysis and execution for the specified set of targets, honoring the + * configuration options in the BuildRequest. Returns normally iff successful, throws an exception + * otherwise. + * + * <p>Callers must ensure that {@link #stopRequest} is called after this method, even if it + * throws. + * + * <p>The caller is responsible for setting up and syncing the package cache. + * + * <p>During this function's execution, the actualTargets and successfulTargets + * fields of the request object are set. + * + * @param request the build request that this build tool is servicing, which specifies various + * options; during this method's execution, the actualTargets and successfulTargets fields + * of the request object are populated + * @param result the build result that is the mutable result of this build + * @param validator target validator + */ + public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator) + throws BuildFailedException, LocalEnvironmentException, + InterruptedException, ViewCreationFailedException, + TargetParsingException, LoadingFailedException, ExecutorInitException, + AbruptExitException, InvalidConfigurationException, TestExecException { + validateOptions(request); + BuildOptions buildOptions = runtime.createBuildOptions(request); + // Sync the package manager before sending the BuildStartingEvent in runLoadingPhase() + runtime.setupPackageCache(request.getPackageCacheOptions(), + DefaultsPackage.getDefaultsPackageContent(buildOptions)); + + ExecutionTool executionTool = null; + LoadingResult loadingResult = null; + BuildConfigurationCollection configurations = null; + try { + getEventBus().post(new BuildStartingEvent(runtime.getOutputFileSystem(), request)); + LOG.info("Build identifier: " + request.getId()); + executionTool = new ExecutionTool(runtime, request); + if (needsExecutionPhase(request.getBuildOptions())) { + // Initialize the execution tool early if we need it. This hides the latency of setting up + // the execution backends. + executionTool.init(); + } + + // Loading phase. + loadingResult = runLoadingPhase(request, validator); + + // Create the build configurations. + if (!request.getMultiCpus().isEmpty()) { + getReporter().handle(Event.warn( + "The --experimental_multi_cpu option is _very_ experimental and only intended for " + + "internal testing at this time. If you do not work on the build tool, then you " + + "should stop now!")); + if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) { + throw new InvalidConfigurationException( + "The experimental setting to select multiple CPUs is only supported for 'build' and " + + "'test' right now!"); + } + } + configurations = getConfigurations( + runtime.getBuildConfigurationKey(buildOptions, request.getMultiCpus()), + request.getViewOptions().keepGoing); + + getEventBus().post(new ConfigurationsCreatedEvent(configurations)); + runtime.throwPendingException(); + if (configurations.getTargetConfigurations().size() == 1) { + // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case + // where there's only a single configuration, but we don't send an event in the multi-config + // case. Can we do better? [multi-config] + getEventBus().post(new MakeEnvironmentEvent( + configurations.getTargetConfigurations().get(0).getMakeEnvironment())); + } + LOG.info("Configurations created"); + + // Analysis phase. + AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations); + result.setActualTargets(analysisResult.getTargetsToBuild()); + result.setTestTargets(analysisResult.getTargetsToTest()); + + reportTargets(analysisResult); + + // Execution phase. + if (needsExecutionPhase(request.getBuildOptions())) { + executionTool.executeBuild(analysisResult, result, runtime.getSkyframeExecutor(), + configurations, mergePackageRoots(loadingResult.getPackageRoots(), + runtime.getSkyframeExecutor().getPackageRoots())); + } + + String delayedErrorMsg = analysisResult.getError(); + if (delayedErrorMsg != null) { + throw new BuildFailedException(delayedErrorMsg); + } + } catch (RuntimeException e) { + // Print an error message for unchecked runtime exceptions. This does not concern Error + // subclasses such as OutOfMemoryError. + request.getOutErr().printErrLn("Unhandled exception thrown during build; message: " + + e.getMessage()); + throw e; + } finally { + // Delete dirty nodes to ensure that they do not accumulate indefinitely. + long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc; + if (versionWindow != -1) { + runtime.getSkyframeExecutor().deleteOldNodes(versionWindow); + } + + if (executionTool != null) { + executionTool.shutdown(); + } + // The workspace status actions will not run with certain flags, or if an error + // occurs early in the build. Tell a lie so that the event is not missing. + // If multiple build_info events are sent, only the first is kept, so this does not harm + // successful runs (which use the workspace status action). + getEventBus().post(new BuildInfoEvent( + runtime.getworkspaceStatusActionFactory().createDummyWorkspaceStatus())); + } + + if (loadingResult != null && loadingResult.hasTargetPatternError()) { + throw new BuildFailedException("execution phase successful, but there were errors " + + "parsing the target pattern"); + } + } + + private ImmutableMap<PathFragment, Path> mergePackageRoots( + ImmutableMap<PackageIdentifier, Path> first, + ImmutableMap<PackageIdentifier, Path> second) { + Map<PathFragment, Path> builder = Maps.newHashMap(); + for (Map.Entry<PackageIdentifier, Path> entry : first.entrySet()) { + builder.put(entry.getKey().getPackageFragment(), entry.getValue()); + } + for (Map.Entry<PackageIdentifier, Path> entry : second.entrySet()) { + if (first.containsKey(entry.getKey())) { + Preconditions.checkState(first.get(entry.getKey()).equals(entry.getValue())); + } else { + // This could overwrite entries from first in other repositories. + builder.put(entry.getKey().getPackageFragment(), entry.getValue()); + } + } + return ImmutableMap.copyOf(builder); + } + + private void reportExceptionError(Exception e) { + if (e.getMessage() != null) { + getReporter().handle(Event.error(e.getMessage())); + } + } + /** + * The crux of the build system. Builds the targets specified in the request using the specified + * Executor. + * + * <p>Performs loading, analysis and execution for the specified set of targets, honoring the + * configuration options in the BuildRequest. Returns normally iff successful, throws an exception + * otherwise. + * + * <p>The caller is responsible for setting up and syncing the package cache. + * + * <p>During this function's execution, the actualTargets and successfulTargets + * fields of the request object are set. + * + * @param request the build request that this build tool is servicing, which specifies various + * options; during this method's execution, the actualTargets and successfulTargets fields + * of the request object are populated + * @param validator target validator + * @return the result as a {@link BuildResult} object + */ + public BuildResult processRequest(BuildRequest request, TargetValidator validator) { + BuildResult result = new BuildResult(request.getStartTime()); + runtime.getEventBus().register(result); + Throwable catastrophe = null; + ExitCode exitCode = ExitCode.BLAZE_INTERNAL_ERROR; + try { + buildTargets(request, result, validator); + exitCode = ExitCode.SUCCESS; + } catch (BuildFailedException e) { + if (e.isErrorAlreadyShown()) { + // The actual error has already been reported by the Builder. + } else { + reportExceptionError(e); + } + if (e.isCatastrophic()) { + result.setCatastrophe(); + } + exitCode = ExitCode.BUILD_FAILURE; + } catch (InterruptedException e) { + exitCode = ExitCode.INTERRUPTED; + getReporter().handle(Event.error("build interrupted")); + getEventBus().post(new BuildInterruptedEvent()); + } catch (TargetParsingException | LoadingFailedException | ViewCreationFailedException e) { + exitCode = ExitCode.PARSING_FAILURE; + reportExceptionError(e); + } catch (TestExecException e) { + // ExitCode.SUCCESS means that build was successful. Real return code of program + // is going to be calculated in TestCommand.doTest(). + exitCode = ExitCode.SUCCESS; + reportExceptionError(e); + } catch (InvalidConfigurationException e) { + exitCode = ExitCode.COMMAND_LINE_ERROR; + reportExceptionError(e); + } catch (AbruptExitException e) { + exitCode = e.getExitCode(); + reportExceptionError(e); + result.setCatastrophe(); + } catch (Throwable throwable) { + catastrophe = throwable; + Throwables.propagate(throwable); + } finally { + stopRequest(request, result, catastrophe, exitCode); + } + + return result; + } + + protected final BuildConfigurationCollection getConfigurations(BuildConfigurationKey key, + boolean keepGoing) + throws InvalidConfigurationException, InterruptedException { + SkyframeExecutor executor = runtime.getSkyframeExecutor(); + // TODO(bazel-team): consider a possibility of moving ConfigurationFactory construction into + // skyframe. + return executor.createConfigurations(keepGoing, runtime.getConfigurationFactory(), key); + } + + @VisibleForTesting + protected final LoadingResult runLoadingPhase(final BuildRequest request, + final TargetValidator validator) + throws LoadingFailedException, TargetParsingException, InterruptedException, + AbruptExitException { + Profiler.instance().markPhase(ProfilePhase.LOAD); + runtime.throwPendingException(); + + final boolean keepGoing = request.getViewOptions().keepGoing; + + Callback callback = new Callback() { + @Override + public void notifyTargets(Collection<Target> targets) throws LoadingFailedException { + if (validator != null) { + validator.validateTargets(targets, keepGoing); + } + } + + @Override + public void notifyVisitedPackages(Set<PackageIdentifier> visitedPackages) { + runtime.getSkyframeExecutor().updateLoadedPackageSet(visitedPackages); + } + }; + + LoadingResult result = runtime.getLoadingPhaseRunner().execute(getReporter(), + getEventBus(), request.getTargets(), request.getLoadingOptions(), + runtime.createBuildOptions(request).getAllLabels(), keepGoing, + request.shouldRunTests(), callback); + runtime.throwPendingException(); + return result; + } + + /** + * Performs the initial phases 0-2 of the build: Setup, Loading and Analysis. + * <p> + * Postcondition: On success, populates the BuildRequest's set of targets to + * build. + * + * @return null if loading / analysis phases were successful; a useful error + * message if loading or analysis phase errors were encountered and + * request.keepGoing. + * @throws InterruptedException if the current thread was interrupted. + * @throws ViewCreationFailedException if analysis failed for any reason. + */ + private AnalysisResult runAnalysisPhase(BuildRequest request, LoadingResult loadingResult, + BuildConfigurationCollection configurations) + throws InterruptedException, ViewCreationFailedException { + Stopwatch timer = Stopwatch.createStarted(); + if (!request.getBuildOptions().performAnalysisPhase) { + getReporter().handle(Event.progress("Loading complete.")); + LOG.info("No analysis requested, so finished"); + return AnalysisResult.EMPTY; + } + + getReporter().handle(Event.progress("Loading complete. Analyzing...")); + Profiler.instance().markPhase(ProfilePhase.ANALYZE); + + AnalysisResult analysisResult = getView().update(loadingResult, configurations, + request.getViewOptions(), request.getTopLevelArtifactContext(), getReporter(), + getEventBus()); + + // TODO(bazel-team): Merge these into one event. + getEventBus().post(new AnalysisPhaseCompleteEvent(analysisResult.getTargetsToBuild(), + getView().getTargetsVisited(), timer.stop().elapsed(TimeUnit.MILLISECONDS))); + getEventBus().post(new TestFilteringCompleteEvent(analysisResult.getTargetsToBuild(), + analysisResult.getTargetsToTest())); + + // Check licenses. + // We check licenses if the first target configuration has license checking enabled. Right now, + // it is not possible to have multiple target configurations with different settings for this + // flag, which allows us to take this short cut. + boolean checkLicenses = configurations.getTargetConfigurations().get(0).checkLicenses(); + if (checkLicenses) { + Profiler.instance().markPhase(ProfilePhase.LICENSE); + validateLicensingForTargets(analysisResult.getTargetsToBuild(), + request.getViewOptions().keepGoing); + } + + return analysisResult; + } + + private static boolean needsExecutionPhase(BuildRequestOptions options) { + return options.performAnalysisPhase && options.performExecutionPhase; + } + + /** + * Stops processing the specified request. + * + * <p>This logs the build result, cleans up and stops the clock. + * + * @param request the build request that this build tool is servicing + * @param crash Any unexpected RuntimeException or Error. May be null + * @param exitCondition A suggested exit condition from either the build logic or + * a thrown exception somewhere along the way. + */ + public void stopRequest(BuildRequest request, BuildResult result, Throwable crash, + ExitCode exitCondition) { + Preconditions.checkState((crash == null) || (exitCondition != ExitCode.SUCCESS)); + result.setUnhandledThrowable(crash); + result.setExitCondition(exitCondition); + // The stop time has to be captured before we send the BuildCompleteEvent. + result.setStopTime(runtime.getClock().currentTimeMillis()); + getEventBus().post(new BuildCompleteEvent(request, result)); + } + + private void reportTargets(AnalysisResult analysisResult) { + Collection<ConfiguredTarget> targetsToBuild = analysisResult.getTargetsToBuild(); + Collection<ConfiguredTarget> targetsToTest = analysisResult.getTargetsToTest(); + if (targetsToTest != null) { + int testCount = targetsToTest.size(); + int targetCount = targetsToBuild.size() - testCount; + if (targetCount == 0) { + getReporter().handle(Event.info("Found " + + testCount + (testCount == 1 ? " test target..." : " test targets..."))); + } else { + getReporter().handle(Event.info("Found " + + targetCount + (targetCount == 1 ? " target and " : " targets and ") + + testCount + (testCount == 1 ? " test target..." : " test targets..."))); + } + } else { + int targetCount = targetsToBuild.size(); + getReporter().handle(Event.info("Found " + + targetCount + (targetCount == 1 ? " target..." : " targets..."))); + } + } + + /** + * Validates the options for this BuildRequest. + * + * <p>Issues warnings for the use of deprecated options, and warnings or errors for any option + * settings that conflict. + */ + @VisibleForTesting + public void validateOptions(BuildRequest request) throws InvalidConfigurationException { + for (String issue : request.validateOptions()) { + getReporter().handle(Event.warn(issue)); + } + } + + /** + * Takes a set of configured targets, and checks if the distribution methods + * declared for the targets are compatible with the constraints imposed by + * their prerequisites' licenses. + * + * @param configuredTargets the targets to check + * @param keepGoing if false, and a licensing error is encountered, both + * generates an error message on the reporter, <em>and</em> throws an + * exception. If true, then just generates a message on the reporter. + * @throws ViewCreationFailedException if the license checking failed (and not + * --keep_going) + */ + private void validateLicensingForTargets(Iterable<ConfiguredTarget> configuredTargets, + boolean keepGoing) throws ViewCreationFailedException { + for (ConfiguredTarget configuredTarget : configuredTargets) { + final Target target = configuredTarget.getTarget(); + + if (TargetUtils.isTestRule(target)) { + continue; // Tests are exempt from license checking + } + + final Set<DistributionType> distribs = target.getDistributions(); + BuildConfiguration config = configuredTarget.getConfiguration(); + boolean staticallyLinked = (config != null) && config.performsStaticLink(); + staticallyLinked |= (config != null) && (target instanceof Rule) + && ((Rule) target).getRuleClassObject().hasAttr("linkopts", Type.STRING_LIST) + && ConfiguredAttributeMapper.of((RuleConfiguredTarget) configuredTarget) + .get("linkopts", Type.STRING_LIST).contains("-static"); + + LicensesProvider provider = configuredTarget.getProvider(LicensesProvider.class); + if (provider != null) { + NestedSet<TargetLicense> licenses = provider.getTransitiveLicenses(); + for (TargetLicense targetLicense : licenses) { + if (!targetLicense.getLicense().checkCompatibility( + distribs, target, targetLicense.getLabel(), getReporter(), staticallyLinked)) { + if (!keepGoing) { + throw new ViewCreationFailedException("Build aborted due to licensing error"); + } + } + } + } else if (configuredTarget.getTarget() instanceof InputFile) { + // Input file targets do not provide licenses because they do not + // depend on the rule where their license is taken from. This is usually + // not a problem, because the transitive collection of licenses always + // hits the rule they come from, except when the input file is a + // top-level target. Thus, we need to handle that case specially here. + // + // See FileTarget#getLicense for more information about the handling of + // license issues with File targets. + License license = configuredTarget.getTarget().getLicense(); + if (!license.checkCompatibility(distribs, target, configuredTarget.getLabel(), + getReporter(), staticallyLinked)) { + if (!keepGoing) { + throw new ViewCreationFailedException("Build aborted due to licensing error"); + } + } + } + } + } + + public BuildView getView() { + return runtime.getView(); + } + + private Reporter getReporter() { + return runtime.getReporter(); + } + + private EventBus getEventBus() { + return runtime.getEventBus(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java new file mode 100644 index 0000000000..5b3229c817 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/CachesSavedEvent.java @@ -0,0 +1,39 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +/** + * Event that is raised when the action and artifact metadata caches are saved at the end of the + * build. Contains statistics. + */ +public class CachesSavedEvent { + /** Cache serialization statistics. */ + private final long actionCacheSaveTimeInMillis; + private final long actionCacheSizeInBytes; + + public CachesSavedEvent( + long actionCacheSaveTimeInMillis, + long actionCacheSizeInBytes) { + this.actionCacheSaveTimeInMillis = actionCacheSaveTimeInMillis; + this.actionCacheSizeInBytes = actionCacheSizeInBytes; + } + + public long getActionCacheSaveTimeInMillis() { + return actionCacheSaveTimeInMillis; + } + + public long getActionCacheSizeInBytes() { + return actionCacheSizeInBytes; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java new file mode 100644 index 0000000000..74143cc6df --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.collect.ImmutableMap; + +import java.util.HashMap; +import java.util.Map; + +/** + * Event signaling the end of the execution phase. Contains statistics about the action cache, + * the metadata cache and about last file save times. + */ +public class ExecutionFinishedEvent { + /** The mtime of the most recently saved source file when the build starts. */ + private long lastFileSaveTimeInMillis; + + /** + * The (filename, mtime) pairs of all files saved between the last build's + * start time and the current build's start time. Only applies to builds + * running with existing Blaze servers. Currently disabled. + */ + private Map<String, Long> changedFileSaveTimes = new HashMap<>(); + + public ExecutionFinishedEvent(Map<String, Long> changedFileSaveTimes, + long lastFileSaveTimeInMillis) { + this.changedFileSaveTimes = ImmutableMap.copyOf(changedFileSaveTimes); + this.lastFileSaveTimeInMillis = lastFileSaveTimeInMillis; + } + + public long getLastFileSaveTimeInMillis() { + return lastFileSaveTimeInMillis; + } + + public Map<String, Long> getChangedFileSaveTimes() { + return changedFileSaveTimes; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java new file mode 100644 index 0000000000..771cfe643b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -0,0 +1,875 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Stopwatch; +import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.common.collect.Table; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BlazeExecutor; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.actions.LocalHostCapacity; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.InputFileConfiguredTarget; +import com.google.devtools.build.lib.analysis.OutputFileConfiguredTarget; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.buildtool.buildevent.ExecutionPhaseCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.CheckUpToDateFilter; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.OutputService; +import com.google.devtools.build.lib.exec.SingleBuildFileCache; +import com.google.devtools.build.lib.exec.SourceManifestActionContextImpl; +import com.google.devtools.build.lib.exec.SymlinkTreeStrategy; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; +import com.google.devtools.build.lib.rules.fileset.FilesetActionContextImpl; +import com.google.devtools.build.lib.rules.test.TestActionContext; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.skyframe.Builder; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * This class manages the execution phase. The entry point is {@link #executeBuild}. + * + * <p>This is only intended for use by {@link BuildTool}. + * + * <p>This class contains an ActionCache, and refers to the BlazeRuntime's BuildView and + * PackageCache. + * + * @see BuildTool + * @see BuildView + */ +public class ExecutionTool { + private static class StrategyConverter { + private Table<Class<? extends ActionContext>, String, ActionContext> classMap = + HashBasedTable.create(); + private Map<Class<? extends ActionContext>, ActionContext> defaultClassMap = + new HashMap<>(); + + /** + * Aggregates all {@link ActionContext}s that are in {@code contextProviders}. + */ + @SuppressWarnings("unchecked") + private StrategyConverter(Iterable<ActionContextProvider> contextProviders) { + for (ActionContextProvider provider : contextProviders) { + for (ActionContext strategy : provider.getActionContexts()) { + ExecutionStrategy annotation = + strategy.getClass().getAnnotation(ExecutionStrategy.class); + if (annotation != null) { + defaultClassMap.put(annotation.contextType(), strategy); + + for (String name : annotation.name()) { + classMap.put(annotation.contextType(), name, strategy); + } + } + } + } + } + + @SuppressWarnings("unchecked") + private <T extends ActionContext> T getStrategy(Class<T> clazz, String name) { + return (T) (name.isEmpty() ? defaultClassMap.get(clazz) : classMap.get(clazz, name)); + } + + private String getValidValues(Class<? extends ActionContext> context) { + return Joiner.on(", ").join(Ordering.natural().sortedCopy(classMap.row(context).keySet())); + } + + private String getUserFriendlyName(Class<? extends ActionContext> context) { + ActionContextMarker marker = context.getAnnotation(ActionContextMarker.class); + return marker != null + ? marker.name() + : context.getSimpleName(); + } + } + + static final Logger LOG = Logger.getLogger(ExecutionTool.class.getName()); + + private final BlazeRuntime runtime; + private final BuildRequest request; + private BlazeExecutor executor; + private ActionInputFileCache fileCache; + private List<ActionContextProvider> actionContextProviders; + + private Map<String, ActionContext> spawnStrategyMap = new HashMap<>(); + private List<ActionContext> strategies = new ArrayList<>(); + + ExecutionTool(BlazeRuntime runtime, BuildRequest request) throws ExecutorInitException { + this.runtime = runtime; + this.request = request; + + List<ActionContextConsumer> actionContextConsumers = new ArrayList<>(); + actionContextProviders = new ArrayList<>(); + for (BlazeModule module : runtime.getBlazeModules()) { + ActionContextProvider provider = module.getActionContextProvider(); + if (provider != null) { + actionContextProviders.add(provider); + } + + ActionContextConsumer consumer = module.getActionContextConsumer(); + if (consumer != null) { + actionContextConsumers.add(consumer); + } + } + + actionContextProviders.add(new FilesetActionContextImpl.Provider( + runtime.getReporter(), runtime.getWorkspaceName())); + + strategies.add(new SourceManifestActionContextImpl(runtime.getRunfilesPrefix())); + strategies.add(new SymlinkTreeStrategy(runtime.getOutputService(), runtime.getBinTools())); + + StrategyConverter strategyConverter = new StrategyConverter(actionContextProviders); + strategies.add(strategyConverter.getStrategy(FilesetActionContext.class, "")); + strategies.add(strategyConverter.getStrategy(WorkspaceStatusAction.Context.class, "")); + + for (ActionContextConsumer consumer : actionContextConsumers) { + // There are many different SpawnActions, and we want to control the action context they use + // independently from each other, for example, to run genrules locally and Java compile action + // in prod. Thus, for SpawnActions, we decide the action context to use not only based on the + // context class, but also the mnemonic of the action. + for (Map.Entry<String, String> entry : consumer.getSpawnActionContexts().entrySet()) { + SpawnActionContext context = + strategyConverter.getStrategy(SpawnActionContext.class, entry.getValue()); + if (context == null) { + throw makeExceptionForInvalidStrategyValue(entry.getValue(), "spawn", + strategyConverter.getValidValues(SpawnActionContext.class)); + } + + spawnStrategyMap.put(entry.getKey(), context); + } + + for (Map.Entry<Class<? extends ActionContext>, String> entry : + consumer.getActionContexts().entrySet()) { + ActionContext context = strategyConverter.getStrategy(entry.getKey(), entry.getValue()); + if (context != null) { + strategies.add(context); + } else if (!entry.getValue().isEmpty()) { + // If the action context consumer requested the default value (by passing in the empty + // string), we do not throw the exception, because we assume that whoever put together + // the modules in this Blaze binary knew what they were doing. + throw makeExceptionForInvalidStrategyValue(entry.getValue(), + strategyConverter.getUserFriendlyName(entry.getKey()), + strategyConverter.getValidValues(entry.getKey())); + } + } + } + + // If tests are to be run during build, too, we have to explicitly load the test action context. + if (request.shouldRunTests()) { + String testStrategyValue = request.getOptions(ExecutionOptions.class).testStrategy; + ActionContext context = strategyConverter.getStrategy(TestActionContext.class, + testStrategyValue); + if (context == null) { + throw makeExceptionForInvalidStrategyValue(testStrategyValue, "test", + strategyConverter.getValidValues(TestActionContext.class)); + } + strategies.add(context); + } + } + + private static ExecutorInitException makeExceptionForInvalidStrategyValue(String value, + String strategy, String validValues) { + return new ExecutorInitException(String.format( + "'%s' is an invalid value for %s strategy. Valid values are: %s", value, strategy, + validValues), ExitCode.COMMAND_LINE_ERROR); + } + + Executor getExecutor() throws ExecutorInitException { + if (executor == null) { + executor = createExecutor(); + } + return executor; + } + + /** + * Creates an executor for the current set of blaze runtime, execution options, and request. + */ + private BlazeExecutor createExecutor() + throws ExecutorInitException { + return new BlazeExecutor( + runtime.getDirectories().getExecRoot(), + runtime.getDirectories().getOutputPath(), + getReporter(), + getEventBus(), + runtime.getClock(), + request, + request.getOptions(ExecutionOptions.class).verboseFailures, + request.getOptions(ExecutionOptions.class).showSubcommands, + strategies, + spawnStrategyMap, + actionContextProviders); + } + + void init() throws ExecutorInitException { + createToolsSymlinks(); + getExecutor(); + } + + void shutdown() { + for (ActionContextProvider actionContextProvider : actionContextProviders) { + actionContextProvider.executionPhaseEnding(); + } + } + + /** + * Performs the execution phase (phase 3) of the build, in which the Builder + * is applied to the action graph to bring the targets up to date. (This + * function will return prior to execution-proper if --nobuild was specified.) + * + * @param analysisResult the analysis phase output + * @param buildResult the mutable build result + * @param skyframeExecutor the skyframe executor (if any) + * @param packageRoots package roots collected from loading phase and BuildConfigutaionCollection + * creation + */ + void executeBuild(AnalysisResult analysisResult, + BuildResult buildResult, @Nullable SkyframeExecutor skyframeExecutor, + BuildConfigurationCollection configurations, + ImmutableMap<PathFragment, Path> packageRoots) + throws BuildFailedException, InterruptedException, AbruptExitException, TestExecException, + ViewCreationFailedException { + Stopwatch timer = Stopwatch.createStarted(); + prepare(packageRoots, configurations); + + ActionGraph actionGraph = analysisResult.getActionGraph(); + + // Get top-level artifacts. + ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild(); + + // If --nobuild is specified, this request completes successfully without + // execution. + if (!request.getBuildOptions().performExecutionPhase) { + return; + } + + // Create symlinks only after we've verified that we're actually + // supposed to build something. + if (getWorkspace().getFileSystem().supportsSymbolicLinks()) { + List<BuildConfiguration> targetConfigurations = + getView().getConfigurationCollection().getTargetConfigurations(); + // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case where + // there's only a single configuration, but we don't create any symlinks in the multi-config + // case. Can we do better? [multi-config] + if (targetConfigurations.size() == 1) { + OutputDirectoryLinksUtils.createOutputDirectoryLinks( + runtime.getWorkspaceName(), getWorkspace(), getExecRoot(), + runtime.getOutputPath(), getReporter(), targetConfigurations.get(0), + request.getSymlinkPrefix()); + } + } + + OutputService outputService = runtime.getOutputService(); + if (outputService != null) { + outputService.startBuild(); + } else { + startLocalOutputBuild(); // TODO(bazel-team): this could be just another OutputService + } + + ActionCache actionCache = getActionCache(); + Builder builder = createBuilder(request, executor, actionCache, skyframeExecutor); + + // + // Execution proper. All statements below are logically nested in + // begin/end pairs. No early returns or exceptions please! + // + + Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets(); + getEventBus().post(new ExecutionStartingEvent(configuredTargets)); + + getReporter().handle(Event.progress("Building...")); + + // Conditionally record dependency-checker log: + ExplanationHandler explanationHandler = + installExplanationHandler(request.getBuildOptions().explanationPath, + request.getOptionsDescription()); + + Set<ConfiguredTarget> builtTargets = new HashSet<>(); + boolean interrupted = false; + try { + Iterable<Artifact> allArtifactsForProviders = Iterables.concat(additionalArtifacts, + TopLevelArtifactHelper.getAllArtifactsToBuild( + analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext()), + TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest())); + if (request.isRunningInEmacs()) { + // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs + request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'"); + } + for (ActionContextProvider actionContextProvider : actionContextProviders) { + actionContextProvider.executionPhaseStarting( + fileCache, + actionGraph, + allArtifactsForProviders); + } + executor.executionPhaseStarting(); + skyframeExecutor.drainChangedFiles(); + + if (request.getViewOptions().discardAnalysisCache) { + // Free memory by removing cache entries that aren't going to be needed. Note that in + // skyframe full, this destroys the action graph as well, so we can only do it after the + // action graph is no longer needed. + getView().clearAnalysisCache(analysisResult.getTargetsToBuild()); + actionGraph = null; + } + + configureResourceManager(request); + + Profiler.instance().markPhase(ProfilePhase.EXECUTE); + + builder.buildArtifacts(additionalArtifacts, + analysisResult.getParallelTests(), + analysisResult.getExclusiveTests(), + analysisResult.getTargetsToBuild(), + executor, builtTargets, + request.getBuildOptions().explanationPath != null); + + } catch (InterruptedException e) { + interrupted = true; + throw e; + } finally { + if (request.isRunningInEmacs()) { + request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'"); + } + if (!interrupted) { + getReporter().handle(Event.progress("Building complete.")); + } + + // Transfer over source file "last save time" stats so the remote logger can find them. + runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0)); + + // Disable system load polling (noop if it was not enabled). + ResourceManager.instance().setAutoSensing(false); + executor.executionPhaseEnding(); + for (ActionContextProvider actionContextProvider : actionContextProviders) { + actionContextProvider.executionPhaseEnding(); + } + + Profiler.instance().markPhase(ProfilePhase.FINISH); + + if (!interrupted) { + saveCaches(actionCache); + } + + long startTime = Profiler.nanoTimeMaybe(); + determineSuccessfulTargets(buildResult, configuredTargets, builtTargets, timer); + showBuildResult(request, buildResult, configuredTargets); + Preconditions.checkNotNull(buildResult.getSuccessfulTargets()); + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Show results"); + if (explanationHandler != null) { + uninstallExplanationHandler(explanationHandler); + } + // Finalize output service last, so that if we do throw an exception, we know all the other + // code has already run. + if (runtime.getOutputService() != null) { + boolean isBuildSuccessful = + buildResult.getSuccessfulTargets().size() == configuredTargets.size(); + runtime.getOutputService().finalizeBuild(isBuildSuccessful); + } + } + } + + private void prepare(ImmutableMap<PathFragment, Path> packageRoots, + BuildConfigurationCollection configurations) + throws ViewCreationFailedException { + // Prepare for build. + Profiler.instance().markPhase(ProfilePhase.PREPARE); + + // Create some tools symlinks / cleanup per-build state + createActionLogDirectory(); + + // Plant the symlink forest. + plantSymlinkForest(packageRoots, configurations); + } + + private void createToolsSymlinks() throws ExecutorInitException { + try { + runtime.getBinTools().setupBuildTools(); + } catch (ExecException e) { + throw new ExecutorInitException("Tools symlink creation failed: " + + e.getMessage() + "; build aborted", e); + } + } + + private void plantSymlinkForest(ImmutableMap<PathFragment, Path> packageRoots, + BuildConfigurationCollection configurations) throws ViewCreationFailedException { + try { + FileSystemUtils.deleteTreesBelowNotPrefixed(getExecRoot(), + new String[] { ".", "_", Constants.PRODUCT_NAME + "-"}); + // Delete the build configuration's temporary directories + for (BuildConfiguration configuration : configurations.getTargetConfigurations()) { + configuration.prepareForExecutionPhase(); + } + FileSystemUtils.plantLinkForest(packageRoots, getExecRoot()); + } catch (IOException e) { + throw new ViewCreationFailedException("Source forest creation failed: " + e.getMessage() + + "; build aborted", e); + } + } + + private void createActionLogDirectory() throws ViewCreationFailedException { + Path directory = runtime.getDirectories().getActionConsoleOutputDirectory(); + try { + if (directory.exists()) { + FileSystemUtils.deleteTree(directory); + } + directory.createDirectory(); + } catch (IOException ex) { + throw new ViewCreationFailedException("couldn't delete action output directory: " + + ex.getMessage()); + } + } + + /** + * Prepare for a local output build. + */ + private void startLocalOutputBuild() throws BuildFailedException { + long startTime = Profiler.nanoTimeMaybe(); + + try { + Path outputPath = runtime.getOutputPath(); + Path localOutputPath = runtime.getDirectories().getLocalOutputPath(); + + if (outputPath.isSymbolicLink()) { + // Remove the existing symlink first. + outputPath.delete(); + if (localOutputPath.exists()) { + // Pre-existing local output directory. Move to outputPath. + localOutputPath.renameTo(outputPath); + } + } + } catch (IOException e) { + throw new BuildFailedException(e.getMessage()); + } finally { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, + "Starting local output build"); + } + } + + /** + * If a path is supplied, creates and installs an ExplanationHandler. Returns + * an instance on success. Reports an error and returns null otherwise. + */ + private ExplanationHandler installExplanationHandler(PathFragment explanationPath, + String allOptions) { + if (explanationPath == null) { + return null; + } + ExplanationHandler handler; + try { + handler = new ExplanationHandler( + getWorkspace().getRelative(explanationPath).getOutputStream(), + allOptions); + } catch (IOException e) { + getReporter().handle(Event.warn(String.format( + "Cannot write explanation of rebuilds to file '%s': %s", + explanationPath, e.getMessage()))); + return null; + } + getReporter().handle( + Event.info("Writing explanation of rebuilds to '" + explanationPath + "'")); + getReporter().addHandler(handler); + return handler; + } + + /** + * Uninstalls the specified ExplanationHandler (if any) and closes the log + * file. + */ + private void uninstallExplanationHandler(ExplanationHandler handler) { + if (handler != null) { + getReporter().removeHandler(handler); + handler.log.close(); + } + } + + /** + * An ErrorEventListener implementation that records DEPCHECKER events into a log + * file, iff the --explain flag is specified during a build. + */ + private static class ExplanationHandler implements EventHandler { + + private final PrintWriter log; + + private ExplanationHandler(OutputStream log, String optionsDescription) { + this.log = new PrintWriter(log); + this.log.println("Build options: " + optionsDescription); + } + + + @Override + public void handle(Event event) { + if (event.getKind() == EventKind.DEPCHECKER) { + log.println(event.getMessage()); + } + } + } + + /** + * Computes the result of the build. Sets the list of successful (up-to-date) + * targets in the request object. + * + * @param configuredTargets The configured targets whose artifacts are to be + * built. + * @param timer A timer that was started when the execution phase started. + */ + private void determineSuccessfulTargets(BuildResult result, + Collection<ConfiguredTarget> configuredTargets, Set<ConfiguredTarget> builtTargets, + Stopwatch timer) { + // Maintain the ordering by copying builtTargets into a LinkedHashSet in the same iteration + // order as configuredTargets. + Collection<ConfiguredTarget> successfulTargets = new LinkedHashSet<>(); + for (ConfiguredTarget target : configuredTargets) { + if (builtTargets.contains(target)) { + successfulTargets.add(target); + } + } + getEventBus().post( + new ExecutionPhaseCompleteEvent(timer.stop().elapsed(TimeUnit.MILLISECONDS))); + result.setSuccessfulTargets(successfulTargets); + } + + /** + * Shows the result of the build. Information includes the list of up-to-date + * and failed targets and list of output artifacts for successful targets + * + * @param request The build request, which specifies various options. + * @param configuredTargets The configured targets whose artifacts are to be + * built. + * + * TODO(bazel-team): (2010) refactor into using Reporter and info/progress events + */ + private void showBuildResult(BuildRequest request, BuildResult result, + Collection<ConfiguredTarget> configuredTargets) { + // NOTE: be careful what you print! We don't want to create a consistency + // problem where the summary message and the exit code disagree. The logic + // here is already complex. + + // Filter the targets we care about into two buckets: + Collection<ConfiguredTarget> succeeded = new ArrayList<>(); + Collection<ConfiguredTarget> failed = new ArrayList<>(); + for (ConfiguredTarget target : configuredTargets) { + // TODO(bazel-team): this is quite ugly. Add a marker provider for this check. + if (target instanceof InputFileConfiguredTarget) { + // Suppress display of source files (because we do no work to build them). + continue; + } + if (target.getTarget() instanceof Rule) { + Rule rule = (Rule) target.getTarget(); + if (rule.getRuleClass().contains("$")) { + // Suppress display of hidden rules + continue; + } + } + if (target instanceof OutputFileConfiguredTarget) { + // Suppress display of generated files (because they appear underneath + // their generating rule), EXCEPT those ones which are not part of the + // filesToBuild of their generating rule (e.g. .par, _deploy.jar + // files), OR when a user explicitly requests an output file but not + // its rule. + TransitiveInfoCollection generatingRule = + getView().getGeneratingRule((OutputFileConfiguredTarget) target); + if (CollectionUtils.containsAll( + generatingRule.getProvider(FileProvider.class).getFilesToBuild(), + target.getProvider(FileProvider.class).getFilesToBuild()) && + configuredTargets.contains(generatingRule)) { + continue; + } + } + + Collection<ConfiguredTarget> successfulTargets = result.getSuccessfulTargets(); + (successfulTargets.contains(target) ? succeeded : failed).add(target); + } + + // Suppress summary if --show_result value is exceeded: + if (succeeded.size() + failed.size() > request.getBuildOptions().maxResultTargets) { + return; + } + + OutErr outErr = request.getOutErr(); + + for (ConfiguredTarget target : succeeded) { + Label label = target.getLabel(); + // For up-to-date targets report generated artifacts, but only + // if they have associated action and not middleman artifacts. + boolean headerFlag = true; + for (Artifact artifact : getFilesToBuild(target, request)) { + if (!artifact.isSourceArtifact()) { + if (headerFlag) { + outErr.printErr("Target " + label + " up-to-date:\n"); + headerFlag = false; + } + outErr.printErrLn(" " + + OutputDirectoryLinksUtils.getPrettyPath(artifact.getPath(), + runtime.getWorkspaceName(), getWorkspace(), request.getSymlinkPrefix())); + } + } + if (headerFlag) { + outErr.printErr( + "Target " + label + " up-to-date (nothing to build)\n"); + } + } + + for (ConfiguredTarget target : failed) { + outErr.printErr("Target " + target.getLabel() + " failed to build\n"); + + // For failed compilation, it is still useful to examine temp artifacts, + // (ie, preprocessed and assembler files). + TempsProvider tempsProvider = target.getProvider(TempsProvider.class); + if (tempsProvider != null) { + for (Artifact temp : tempsProvider.getTemps()) { + if (temp.getPath().exists()) { + outErr.printErrLn(" See temp at " + + OutputDirectoryLinksUtils.getPrettyPath(temp.getPath(), + runtime.getWorkspaceName(), getWorkspace(), request.getSymlinkPrefix())); + } + } + } + } + if (!failed.isEmpty() && !request.getOptions(ExecutionOptions.class).verboseFailures) { + outErr.printErr("Use --verbose_failures to see the command lines of failed build steps.\n"); + } + } + + /** + * Gets all the files to build for a given target and build request. + * There may be artifacts that should be built which are not represented in the + * configured target graph. Currently, this only occurs when "--save_temps" is on. + * + * @param target configured target + * @param request the build request + * @return artifacts to build + */ + private static Collection<Artifact> getFilesToBuild(ConfiguredTarget target, + BuildRequest request) { + ImmutableSet.Builder<Artifact> result = ImmutableSet.builder(); + if (request.getBuildOptions().compileOnly) { + FilesToCompileProvider provider = target.getProvider(FilesToCompileProvider.class); + if (provider != null) { + result.addAll(provider.getFilesToCompile()); + } + } else if (request.getBuildOptions().compilationPrerequisitesOnly) { + CompilationPrerequisitesProvider provider = + target.getProvider(CompilationPrerequisitesProvider.class); + if (provider != null) { + result.addAll(provider.getCompilationPrerequisites()); + } + } else { + FileProvider provider = target.getProvider(FileProvider.class); + if (provider != null) { + result.addAll(provider.getFilesToBuild()); + } + } + TempsProvider tempsProvider = target.getProvider(TempsProvider.class); + if (tempsProvider != null) { + result.addAll(tempsProvider.getTemps()); + } + + return result.build(); + } + + private ActionCache getActionCache() throws LocalEnvironmentException { + try { + return runtime.getPersistentActionCache(); + } catch (IOException e) { + // TODO(bazel-team): (2010) Ideally we should just remove all cache data and reinitialize + // caches. + LoggingUtil.logToRemote(Level.WARNING, "Failed to initialize action cache: " + + e.getMessage(), e); + throw new LocalEnvironmentException("couldn't create action cache: " + e.getMessage() + + ". If error persists, use 'blaze clean'"); + } + } + + private Builder createBuilder(BuildRequest request, + Executor executor, + ActionCache actionCache, + SkyframeExecutor skyframeExecutor) { + BuildRequest.BuildRequestOptions options = request.getBuildOptions(); + boolean verboseExplanations = options.verboseExplanations; + boolean keepGoing = request.getViewOptions().keepGoing; + + Path actionOutputRoot = runtime.getDirectories().getActionConsoleOutputDirectory(); + Predicate<Action> executionFilter = CheckUpToDateFilter.fromOptions( + request.getOptions(ExecutionOptions.class)); + + // jobs should have been verified in BuildRequest#validateOptions(). + Preconditions.checkState(options.jobs >= -1); + int actualJobs = options.jobs == 0 ? 1 : options.jobs; // Treat 0 jobs as a single task. + + // Unfortunately, the exec root cache is not shared with caches in the remote execution + // client. + fileCache = createBuildSingleFileCache(executor.getExecRoot()); + skyframeExecutor.setActionOutputRoot(actionOutputRoot); + return new SkyframeBuilder(skyframeExecutor, + new ActionCacheChecker(actionCache, getView().getArtifactFactory(), executionFilter, + verboseExplanations), + keepGoing, actualJobs, options.checkOutputFiles, fileCache, + request.getBuildOptions().progressReportInterval); + } + + private void configureResourceManager(BuildRequest request) { + ResourceManager resourceMgr = ResourceManager.instance(); + ExecutionOptions options = request.getOptions(ExecutionOptions.class); + if (options.availableResources != null) { + resourceMgr.setAvailableResources(options.availableResources); + resourceMgr.setRamUtilizationPercentage(100); + } else { + resourceMgr.setAvailableResources(LocalHostCapacity.getLocalHostCapacity()); + resourceMgr.setRamUtilizationPercentage(options.ramUtilizationPercentage); + if (options.useResourceAutoSense) { + getReporter().handle( + Event.warn("Not using resource autosense due to known responsiveness issues")); + } + ResourceManager.instance().setAutoSensing(/*autosense=*/false); + } + } + + /** + * Writes the cache files to disk, reporting any errors that occurred during + * writing. + */ + private void saveCaches(ActionCache actionCache) { + long actionCacheSizeInBytes = 0; + long actionCacheSaveTime; + + long startTime = BlazeClock.nanoTime(); + try { + LOG.info("saving action cache..."); + actionCacheSizeInBytes = actionCache.save(); + LOG.info("action cache saved"); + } catch (IOException e) { + getReporter().handle(Event.error("I/O error while writing action log: " + e.getMessage())); + } finally { + long stopTime = BlazeClock.nanoTime(); + actionCacheSaveTime = + TimeUnit.MILLISECONDS.convert(stopTime - startTime, TimeUnit.NANOSECONDS); + Profiler.instance().logSimpleTask(startTime, stopTime, + ProfilerTask.INFO, "Saving action cache"); + } + + runtime.getEventBus().post(new CachesSavedEvent( + actionCacheSaveTime, actionCacheSizeInBytes)); + } + + private ActionInputFileCache createBuildSingleFileCache(Path execRoot) { + String cwd = execRoot.getPathString(); + FileSystem fs = runtime.getDirectories().getFileSystem(); + + ActionInputFileCache cache = null; + for (BlazeModule module : runtime.getBlazeModules()) { + ActionInputFileCache pluggable = module.createActionInputCache(cwd, fs); + if (pluggable != null) { + Preconditions.checkState(cache == null); + cache = pluggable; + } + } + + if (cache == null) { + cache = new SingleBuildFileCache(cwd, fs); + } + return cache; + } + + private Reporter getReporter() { + return runtime.getReporter(); + } + + private EventBus getEventBus() { + return runtime.getEventBus(); + } + + private BuildView getView() { + return runtime.getView(); + } + + private Path getWorkspace() { + return runtime.getWorkspace(); + } + + private Path getExecRoot() { + return runtime.getExecRoot(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java b/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java new file mode 100644 index 0000000000..7890b2258a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/LocalEnvironmentException.java @@ -0,0 +1,45 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; + +/** + * An exception that signals that something is wrong with the user's environment + * that he can fix. Used to report the problem of having no free space left in + * the blaze output directory. + * + * <p>Note that this is a much higher level exception then the similarly named + * EnvironmentExecException, which is thrown from the base Client and Strategy + * layers of Blaze. + * + * <p>This exception is only thrown when we've decided that the build has, in + * fact, failed and we should exit. + */ +public class LocalEnvironmentException extends AbruptExitException { + + public LocalEnvironmentException(String message) { + super(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR); + } + + public LocalEnvironmentException(Throwable cause) { + super(ExitCode.LOCAL_ENVIRONMENTAL_ERROR, cause); + } + + public LocalEnvironmentException(String message, Throwable cause) { + super(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, cause); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java new file mode 100644 index 0000000000..094b7bca52 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/OutputDirectoryLinksUtils.java @@ -0,0 +1,184 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.base.Joiner; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Symlinks; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Static utilities for managing output directory symlinks. + */ +public class OutputDirectoryLinksUtils { + public static final String OUTPUT_SYMLINK_NAME = Constants.PRODUCT_NAME + "-out"; + + // Used in getPrettyPath() method below. + private static final String[] LINKS = { "bin", "genfiles", "includes" }; + + private static final String NO_CREATE_SYMLINKS_PREFIX = "/"; + + private static String execRootSymlink(String workspaceName) { + return Constants.PRODUCT_NAME + "-" + workspaceName; + } + /** + * Attempts to create convenience symlinks in the workspaceDirectory and in + * execRoot to the output area and to the configuration-specific output + * directories. Issues a warning if it fails, e.g. because workspaceDirectory + * is readonly. + */ + public static void createOutputDirectoryLinks(String workspaceName, + Path workspace, Path execRoot, Path outputPath, + EventHandler eventHandler, BuildConfiguration targetConfig, String symlinkPrefix) { + if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) { + return; + } + List<String> failures = new ArrayList<>(); + + // Make the two non-specific links from the workspace to the output area, + // and the configuration-specific links in both the workspace and the execution root dirs. + // NB! Keep in sync with removeOutputDirectoryLinks below. + createLink(workspace, OUTPUT_SYMLINK_NAME, outputPath, failures); + + // Points to execroot + createLink(workspace, execRootSymlink(workspaceName), execRoot, failures); + createLink(workspace, symlinkPrefix + "bin", targetConfig.getBinDirectory().getPath(), + failures); + createLink(workspace, symlinkPrefix + "testlogs", targetConfig.getTestLogsDirectory().getPath(), + failures); + createLink(workspace, symlinkPrefix + "genfiles", targetConfig.getGenfilesDirectory().getPath(), + failures); + if (!failures.isEmpty()) { + eventHandler.handle(Event.warn(String.format( + "failed to create one or more convenience symlinks for prefix '%s':\n %s", + symlinkPrefix, Joiner.on("\n ").join(failures)))); + } + } + + /** + * Returns a convenient path to the specified file, relativizing it and using output-dir symlinks + * if possible. Otherwise, return a path relative to the workspace directory if possible. + * Otherwise, return the absolute path. + * + * <p>This method must be called after the symlinks are created at the end of a build. If called + * before, the pretty path may be incorrect if the symlinks end up pointing somewhere new. + */ + public static PathFragment getPrettyPath(Path file, String workspaceName, + Path workspaceDirectory, String symlinkPrefix) { + for (String link : LINKS) { + PathFragment result = relativize(file, workspaceDirectory, symlinkPrefix + link); + if (result != null) { + return result; + } + } + + PathFragment result = relativize(file, workspaceDirectory, execRootSymlink(workspaceName)); + if (result != null) { + return result; + } + + result = relativize(file, workspaceDirectory, OUTPUT_SYMLINK_NAME); + if (result != null) { + return result; + } + + return file.asFragment(); + } + + // Helper to getPrettyPath. Returns file, relativized w.r.t. the referent of + // "linkname", or null if it was a not a child. + private static PathFragment relativize(Path file, Path workspaceDirectory, String linkname) { + PathFragment link = new PathFragment(linkname); + try { + Path dir = workspaceDirectory.getRelative(link); + PathFragment levelOneLinkTarget = dir.readSymbolicLink(); + if (levelOneLinkTarget.isAbsolute() && + file.startsWith(dir = file.getRelative(levelOneLinkTarget))) { + return link.getRelative(file.relativeTo(dir)); + } + } catch (IOException e) { + /* ignore */ + } + return null; + } + + /** + * Attempts to remove the convenience symlinks in the workspace directory. + * + * <p>Issues a warning if it fails, e.g. because workspaceDirectory is readonly. + * Also cleans up any child directories created by a custom prefix. + * + * @param workspace the runtime's workspace + * @param eventHandler the error eventHandler + * @param symlinkPrefix the symlink prefix which should be removed + */ + public static void removeOutputDirectoryLinks(String workspaceName, Path workspace, + EventHandler eventHandler, String symlinkPrefix) { + if (NO_CREATE_SYMLINKS_PREFIX.equals(symlinkPrefix)) { + return; + } + List<String> failures = new ArrayList<>(); + + removeLink(workspace, OUTPUT_SYMLINK_NAME, failures); + removeLink(workspace, execRootSymlink(workspaceName), failures); + removeLink(workspace, symlinkPrefix + "bin", failures); + removeLink(workspace, symlinkPrefix + "testlogs", failures); + removeLink(workspace, symlinkPrefix + "genfiles", failures); + FileSystemUtils.removeDirectoryAndParents(workspace, new PathFragment(symlinkPrefix)); + if (!failures.isEmpty()) { + eventHandler.handle(Event.warn(String.format( + "failed to remove one or more convenience symlinks for prefix '%s':\n %s", symlinkPrefix, + Joiner.on("\n ").join(failures)))); + } + } + + /** + * Helper to createOutputDirectoryLinks that creates a symlink from base + name to target. + */ + private static boolean createLink(Path base, String name, Path target, List<String> failures) { + try { + FileSystemUtils.ensureSymbolicLink(base.getRelative(name), target); + return true; + } catch (IOException e) { + failures.add(String.format("%s -> %s: %s", name, target.getPathString(), e.getMessage())); + return false; + } + } + + /** + * Helper to removeOutputDirectoryLinks that removes one of the Blaze convenience symbolic links. + */ + private static boolean removeLink(Path base, String name, List<String> failures) { + Path link = base.getRelative(name); + try { + if (link.exists(Symlinks.NOFOLLOW)) { + ExecutionTool.LOG.finest("Removing " + link); + link.delete(); + } + return true; + } catch (IOException e) { + failures.add(String.format("%s: %s", name, e.getMessage())); + return false; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java new file mode 100644 index 0000000000..779515a28b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java @@ -0,0 +1,355 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCacheChecker; +import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BuildFailedException; +import com.google.devtools.build.lib.actions.BuilderUtils; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.TestExecException; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog; +import com.google.devtools.build.lib.skyframe.ActionExecutionValue; +import com.google.devtools.build.lib.skyframe.Builder; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.TargetCompletionValue; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.skyframe.CycleInfo; +import com.google.devtools.build.skyframe.ErrorInfo; +import com.google.devtools.build.skyframe.EvaluationProgressReceiver; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; + +import java.text.NumberFormat; +import java.util.Collection; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A {@link Builder} implementation driven by Skyframe. + */ +@VisibleForTesting +public class SkyframeBuilder implements Builder { + + private final SkyframeExecutor skyframeExecutor; + private final boolean keepGoing; + private final int numJobs; + private final boolean checkOutputFiles; + private final ActionInputFileCache fileCache; + private final ActionCacheChecker actionCacheChecker; + private final int progressReportInterval; + + @VisibleForTesting + public SkyframeBuilder(SkyframeExecutor skyframeExecutor, ActionCacheChecker actionCacheChecker, + boolean keepGoing, int numJobs, boolean checkOutputFiles, + ActionInputFileCache fileCache, int progressReportInterval) { + this.skyframeExecutor = skyframeExecutor; + this.actionCacheChecker = actionCacheChecker; + this.keepGoing = keepGoing; + this.numJobs = numJobs; + this.checkOutputFiles = checkOutputFiles; + this.fileCache = fileCache; + this.progressReportInterval = progressReportInterval; + } + + @Override + public void buildArtifacts(Set<Artifact> artifacts, + Set<ConfiguredTarget> parallelTests, + Set<ConfiguredTarget> exclusiveTests, + Collection<ConfiguredTarget> targetsToBuild, + Executor executor, + Set<ConfiguredTarget> builtTargets, + boolean explain) + throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException { + skyframeExecutor.prepareExecution(checkOutputFiles); + skyframeExecutor.setFileCache(fileCache); + // Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a + // synchronized collection), so unsynchronized access to this variable is unsafe while it runs. + ExecutionProgressReceiver executionProgressReceiver = + new ExecutionProgressReceiver(Preconditions.checkNotNull(builtTargets), + countTestActions(exclusiveTests), skyframeExecutor.getEventBus()); + ResourceManager.instance().setEventBus(skyframeExecutor.getEventBus()); + + boolean success = false; + EvaluationResult<?> result; + + ActionExecutionStatusReporter statusReporter = ActionExecutionStatusReporter.create( + skyframeExecutor.getReporter(), executor, skyframeExecutor.getEventBus()); + + AtomicBoolean isBuildingExclusiveArtifacts = new AtomicBoolean(false); + ActionExecutionInactivityWatchdog watchdog = new ActionExecutionInactivityWatchdog( + executionProgressReceiver.createInactivityMonitor(statusReporter), + executionProgressReceiver.createInactivityReporter(statusReporter, + isBuildingExclusiveArtifacts), progressReportInterval); + + skyframeExecutor.setActionExecutionProgressReportingObjects(executionProgressReceiver, + executionProgressReceiver, statusReporter); + watchdog.start(); + + try { + result = skyframeExecutor.buildArtifacts(executor, artifacts, targetsToBuild, parallelTests, + /*exclusiveTesting=*/false, keepGoing, explain, numJobs, actionCacheChecker, + executionProgressReceiver); + // progressReceiver is finished, so unsynchronized access to builtTargets is now safe. + success = processResult(result, keepGoing, skyframeExecutor); + + Preconditions.checkState( + !success || result.keyNames().size() + == (artifacts.size() + targetsToBuild.size() + parallelTests.size()), + "Build reported as successful but not all artifacts and targets built: %s, %s", + result, artifacts); + + // Run exclusive tests: either tagged as "exclusive" or is run in an invocation with + // --test_output=streamed. + isBuildingExclusiveArtifacts.set(true); + for (ConfiguredTarget exclusiveTest : exclusiveTests) { + // Since only one artifact is being built at a time, we don't worry about an artifact being + // built and then the build being interrupted. + result = skyframeExecutor.buildArtifacts(executor, ImmutableSet.<Artifact>of(), + targetsToBuild, ImmutableSet.of(exclusiveTest), /*exclusiveTesting=*/true, keepGoing, + explain, numJobs, actionCacheChecker, null); + boolean exclusiveSuccess = processResult(result, keepGoing, skyframeExecutor); + Preconditions.checkState(!exclusiveSuccess || !result.keyNames().isEmpty(), + "Build reported as successful but test %s not executed: %s", + exclusiveTest, result); + success &= exclusiveSuccess; + } + } finally { + watchdog.stop(); + ResourceManager.instance().unsetEventBus(); + skyframeExecutor.setActionExecutionProgressReportingObjects(null, null, null); + statusReporter.unregisterFromEventBus(); + } + + if (!success) { + throw new BuildFailedException(); + } + } + + private static boolean resultHasCatastrophicError(EvaluationResult<?> result) { + for (ErrorInfo errorInfo : result.errorMap().values()) { + if (errorInfo.isCatastrophic()) { + return true; + } + } + // An unreported catastrophe manifests with hasError() being true but no errors visible. + return result.hasError() && result.errorMap().isEmpty(); + } + + /** + * Process the Skyframe update, taking into account the keepGoing setting. + * + * Returns false if the update() failed, but we should continue. Returns true on success. + * Throws on fail-fast failures. + */ + private static boolean processResult(EvaluationResult<?> result, boolean keepGoing, + SkyframeExecutor skyframeExecutor) throws BuildFailedException, TestExecException { + if (result.hasError()) { + boolean hasCycles = false; + for (Map.Entry<SkyKey, ErrorInfo> entry : result.errorMap().entrySet()) { + Iterable<CycleInfo> cycles = entry.getValue().getCycleInfo(); + skyframeExecutor.reportCycles(cycles, entry.getKey()); + hasCycles |= !Iterables.isEmpty(cycles); + } + if (keepGoing && !resultHasCatastrophicError(result)) { + return false; + } + if (hasCycles || result.errorMap().isEmpty()) { + // error map may be empty in the case of a catastrophe. + throw new BuildFailedException(); + } else { + // Need to wrap exception for rethrowCause. + BuilderUtils.rethrowCause( + new Exception(Preconditions.checkNotNull(result.getError().getException()))); + } + } + return true; + } + + private static int countTestActions(Iterable<ConfiguredTarget> testTargets) { + int count = 0; + for (ConfiguredTarget testTarget : testTargets) { + count += TestProvider.getTestStatusArtifacts(testTarget).size(); + } + return count; + } + + /** + * Listener for executed actions and built artifacts. We use a listener so that we have an + * accurate set of successfully run actions and built artifacts, even if the build is interrupted. + */ + private static final class ExecutionProgressReceiver implements EvaluationProgressReceiver, + SkyframeActionExecutor.ProgressSupplier, SkyframeActionExecutor.ActionCompletedReceiver { + private static final NumberFormat PROGRESS_MESSAGE_NUMBER_FORMATTER; + + // Must be thread-safe! + private final Set<ConfiguredTarget> builtTargets; + private final Set<SkyKey> enqueuedActions = Sets.newConcurrentHashSet(); + private final Set<Action> completedActions = Sets.newConcurrentHashSet(); + private final Object activityIndicator = new Object(); + /** Number of exclusive tests. To be accounted for in progress messages. */ + private final int exclusiveTestsCount; + private final EventBus eventBus; + + static { + PROGRESS_MESSAGE_NUMBER_FORMATTER = NumberFormat.getIntegerInstance(Locale.ENGLISH); + PROGRESS_MESSAGE_NUMBER_FORMATTER.setGroupingUsed(true); + } + + /** + * {@code builtTargets} is accessed through a synchronized set, and so no other access to it + * is permitted while this receiver is active. + */ + ExecutionProgressReceiver(Set<ConfiguredTarget> builtTargets, int exclusiveTestsCount, + EventBus eventBus) { + this.builtTargets = Collections.synchronizedSet(builtTargets); + this.exclusiveTestsCount = exclusiveTestsCount; + this.eventBus = eventBus; + } + + @Override + public void invalidated(SkyValue node, InvalidationState state) {} + + @Override + public void enqueueing(SkyKey skyKey) { + if (ActionExecutionValue.isReportWorthyAction(skyKey)) { + // Remember all enqueued actions for the benefit of progress reporting. + // We discover most actions early in the build, well before we start executing them. + // Some of these will be cache hits and won't be executed, so we'll need to account for them + // in the evaluated method too. + enqueuedActions.add(skyKey); + } + } + + @Override + public void evaluated(SkyKey skyKey, SkyValue node, EvaluationState state) { + SkyFunctionName type = skyKey.functionName(); + if (type == SkyFunctions.TARGET_COMPLETION) { + TargetCompletionValue val = (TargetCompletionValue) node; + ConfiguredTarget target = val.getConfiguredTarget(); + builtTargets.add(target); + eventBus.post(TargetCompleteEvent.createSuccessful(target)); + } else if (type == SkyFunctions.ACTION_EXECUTION) { + // Remember all completed actions, regardless of having been cached or really executed. + actionCompleted((Action) skyKey.argument()); + } + } + + /** + * {@inheritDoc} + * + * <p>This method adds the action to {@link #completedActions} and notifies the + * {@link #activityIndicator}. + * + * <p>We could do this only in the {@link #evaluated} method too, but as it happens the action + * executor tells the reporter about the completed action before the node is inserted into the + * graph, so the reporter would find out about the completed action sooner than we could + * have updated {@link #completedActions}, which would result in incorrect numbers on the + * progress messages. However we have to store completed actions in {@link #evaluated} too, + * because that's the only place we get notified about completed cached actions. + */ + @Override + public void actionCompleted(Action a) { + if (ActionExecutionValue.isReportWorthyAction(a)) { + completedActions.add(a); + synchronized (activityIndicator) { + activityIndicator.notifyAll(); + } + } + } + + @Override + public String getProgressString() { + return String.format("[%s / %s]", + PROGRESS_MESSAGE_NUMBER_FORMATTER.format(completedActions.size()), + PROGRESS_MESSAGE_NUMBER_FORMATTER.format(exclusiveTestsCount + enqueuedActions.size())); + } + + ActionExecutionInactivityWatchdog.InactivityMonitor createInactivityMonitor( + final ActionExecutionStatusReporter statusReporter) { + return new ActionExecutionInactivityWatchdog.InactivityMonitor() { + + @Override + public boolean hasStarted() { + return !enqueuedActions.isEmpty(); + } + + @Override + public int getPending() { + return statusReporter.getCount(); + } + + @Override + public int waitForNextCompletion(int timeoutMilliseconds) throws InterruptedException { + synchronized (activityIndicator) { + int before = completedActions.size(); + long startTime = BlazeClock.instance().currentTimeMillis(); + while (true) { + activityIndicator.wait(timeoutMilliseconds); + + int completed = completedActions.size() - before; + long now = 0; + if (completed > 0 || (startTime + timeoutMilliseconds) <= (now = BlazeClock.instance() + .currentTimeMillis())) { + // Some actions completed, or timeout fully elapsed. + return completed; + } else { + // Spurious Wakeup -- no actions completed and there's still time to wait. + timeoutMilliseconds -= now - startTime; // account for elapsed wait time + startTime = now; + } + } + } + } + }; + } + + ActionExecutionInactivityWatchdog.InactivityReporter createInactivityReporter( + final ActionExecutionStatusReporter statusReporter, + final AtomicBoolean isBuildingExclusiveArtifacts) { + return new ActionExecutionInactivityWatchdog.InactivityReporter() { + @Override + public void maybeReportInactivity() { + // Do not report inactivity if we are currently running an exclusive test or a streaming + // action (in practice only tests can stream and it implicitly makes them exclusive). + if (!isBuildingExclusiveArtifacts.get()) { + statusReporter.showCurrentlyExecutingActions( + ExecutionProgressReceiver.this.getProgressString() + " "); + } + } + }; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java b/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java new file mode 100644 index 0000000000..e6eed80da2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/TargetValidator.java @@ -0,0 +1,37 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; + +import java.util.Collection; + +/** + * Validator for targets. + * + * <p>Used in "blaze run" to make sure that we are building exactly one binary target. + */ +public interface TargetValidator { + + /** + * Hook for subclasses to validate a build request before building begins. + * Implementors should print warnings for invalid targets iff keepGoing. + * + * @param targets The targets to build. + * @throws LoadingFailedException if the request is not valid for some reason. + */ + void validateTargets(Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java new file mode 100644 index 0000000000..e9278e693a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildCompleteEvent.java @@ -0,0 +1,40 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildResult; + +/** + * This event is fired from BuildTool#stopRequest(). + */ +public final class BuildCompleteEvent { + private final BuildResult result; + + /** + * Construct the BuildStartingEvent. + * @param request the build request. + */ + public BuildCompleteEvent(BuildRequest request, BuildResult result) { + this.result = result; + } + + /** + * @return the build summary + */ + public BuildResult getResult() { + return result; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java new file mode 100644 index 0000000000..02a5d8bd8d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildInterruptedEvent.java @@ -0,0 +1,22 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +/** + * This event is fired from {@code AbstractBuildCommand#doBuild} to indicate + * that the user interrupted the build with control-C. + */ +public class BuildInterruptedEvent { +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java new file mode 100644 index 0000000000..714534d63c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/BuildStartingEvent.java @@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.devtools.build.lib.buildtool.BuildRequest; + +/** + * This event is fired from BuildTool#startRequest(). + * At this point, the set of target patters are known, but have + * yet to be parsed. + */ +public class BuildStartingEvent { + private final String outputFileSystem; + private final BuildRequest request; + + /** + * Construct the BuildStartingEvent. + * @param request the build request. + */ + public BuildStartingEvent(String outputFileSystem, BuildRequest request) { + this.outputFileSystem = outputFileSystem; + this.request = request; + } + + /** + * @return the output file system. + */ + public String getOutputFileSystem() { + return outputFileSystem; + } + + /** + * @return the active BuildRequest. + */ + public BuildRequest getRequest() { + return request; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java new file mode 100644 index 0000000000..cf5796035f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionPhaseCompleteEvent.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +/** + * This event is fired after the execution phase is complete. + */ +public class ExecutionPhaseCompleteEvent { + private final long timeInMs; + + /** + * Construct the event. + * + * @param timeInMs time for execution phase in milliseconds. + */ + public ExecutionPhaseCompleteEvent(long timeInMs) { + this.timeInMs = timeInMs; + } + + public long getTimeInMs() { + return timeInMs; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java new file mode 100644 index 0000000000..c2b4f77f79 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/ExecutionStartingEvent.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.buildtool.ExecutionTool; + +import java.util.Collection; + +/** + * This event is fired from {@link ExecutionTool#executeBuild} to indicate that the execution phase + * of the build is starting. + */ +public class ExecutionStartingEvent { + private final Collection<TransitiveInfoCollection> targets; + + /** + * Construct the event with a set of targets. + * @param targets Remaining active targets. + */ + public ExecutionStartingEvent(Collection<? extends TransitiveInfoCollection> targets) { + this.targets = ImmutableList.copyOf(targets); + } + + /** + * @return The set of active targets remaining, which is a subset + * of the targets in the user request. + */ + public Collection<TransitiveInfoCollection> getTargets() { + return targets; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java new file mode 100644 index 0000000000..c380456088 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/buildevent/TestFilteringCompleteEvent.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.buildevent; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.rules.test.TestProvider; + +import java.util.Collection; + +import javax.annotation.concurrent.Immutable; + +/** + * This event is fired after test filtering. + * + * The test filtering phase always expands test_suite rules, so + * the set of active targets should never contain test_suites. + */ +@Immutable +public class TestFilteringCompleteEvent { + private final Collection<ConfiguredTarget> targets; + private final Collection<ConfiguredTarget> testTargets; + + /** + * Construct the event. + * @param targets The set of active targets that remain. + * @param testTargets The collection of tests to be run. May be null. + */ + public TestFilteringCompleteEvent( + Collection<? extends ConfiguredTarget> targets, + Collection<? extends ConfiguredTarget> testTargets) { + this.targets = ImmutableList.copyOf(targets); + this.testTargets = testTargets == null ? null : ImmutableList.copyOf(testTargets); + if (testTargets == null) { + return; + } + + for (ConfiguredTarget testTarget : testTargets) { + Preconditions.checkState(testTarget.getProvider(TestProvider.class) != null); + } + } + + /** + * @return The set of active targets remaining. This is a subset of + * the targets that passed analysis, after test_suite expansion. + */ + public Collection<ConfiguredTarget> getTargets() { + return targets; + } + + /** + * @return The set of test targets to be run. May be null. + */ + public Collection<ConfiguredTarget> getTestTargets() { + return testTargets; + } +} |