// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.buildtool; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.BuildFailedException; import com.google.devtools.build.lib.actions.TestExecException; import com.google.devtools.build.lib.analysis.AnalysisPhaseCompleteEvent; import com.google.devtools.build.lib.analysis.BuildInfoEvent; import com.google.devtools.build.lib.analysis.BuildView; import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.LicensesProvider; import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; import com.google.devtools.build.lib.analysis.MakeEnvironmentEvent; import com.google.devtools.build.lib.analysis.StaticallyLinkedMarkerProvider; import com.google.devtools.build.lib.analysis.TargetAndConfiguration; import com.google.devtools.build.lib.analysis.ViewCreationFailedException; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.DefaultsPackage; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; import com.google.devtools.build.lib.buildeventstream.AbortedEvent; import com.google.devtools.build.lib.buildeventstream.BuildEventId; import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.Aborted.AbortReason; import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; import com.google.devtools.build.lib.buildtool.buildevent.NoAnalyzeEvent; import com.google.devtools.build.lib.buildtool.buildevent.NoExecutionEvent; import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.TargetParsingException; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.OutputFilter; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.packages.InputFile; import com.google.devtools.build.lib.packages.License; import com.google.devtools.build.lib.packages.License.DistributionType; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.NoSuchTargetException; import com.google.devtools.build.lib.packages.Target; import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.pkgcache.LoadingCallback; import com.google.devtools.build.lib.pkgcache.LoadingFailedException; import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; import com.google.devtools.build.lib.pkgcache.LoadingResult; import com.google.devtools.build.lib.profiler.ProfilePhase; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.query2.CommonQueryOptions; import com.google.devtools.build.lib.query2.ConfiguredTargetQueryEnvironment; import com.google.devtools.build.lib.query2.engine.QueryEvalResult; import com.google.devtools.build.lib.query2.engine.QueryException; import com.google.devtools.build.lib.query2.engine.QueryExpression; import com.google.devtools.build.lib.query2.engine.TargetLiteral; import com.google.devtools.build.lib.query2.engine.ThreadSafeOutputFormatterCallback; import com.google.devtools.build.lib.runtime.BlazeRuntime; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.skyframe.SkyframeExecutorWrappingWalkableGraph; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.RegexFilter; import com.google.devtools.build.skyframe.WalkableGraph; import com.google.devtools.common.options.OptionsParsingException; import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import java.util.regex.Pattern; /** * Provides the bulk of the implementation of the 'blaze build' command. * *
The various concrete build command classes handle the command options and request * setup, then delegate the handling of the request (the building of targets) to this class. * *
The main entry point is {@link #buildTargets}. * *
Most of analysis is handled in {@link BuildView}, and execution in {@link ExecutionTool}. */ public final class BuildTool { private static final Logger logger = Logger.getLogger(BuildTool.class.getName()); private final CommandEnvironment env; private final BlazeRuntime runtime; /** * Constructs a BuildTool. * * @param env a reference to the command environment of the currently executing command */ public BuildTool(CommandEnvironment env) { this.env = env; this.runtime = env.getRuntime(); } public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator) throws BuildFailedException, InterruptedException, ViewCreationFailedException, TargetParsingException, LoadingFailedException, AbruptExitException, InvalidConfigurationException, TestExecException, ConfiguredTargetQueryCommandLineException { buildTargets(request, result, validator, null); } /** * The crux of the build system. Builds the targets specified in the request using the specified * Executor. * *
Performs loading, analysis and execution for the specified set of targets, honoring the * configuration options in the BuildRequest. Also performs a query on results of analysis phase * if a query expression is specified. Returns normally iff successful, throws an exception * otherwise. * *
Callers must ensure that {@link #stopRequest} is called after this method, even if it * throws. * *
The caller is responsible for setting up and syncing the package cache. * *
During this function's execution, the actualTargets and successfulTargets fields of the * request object are set. * * @param request the build request that this build tool is servicing, which specifies various * options; during this method's execution, the actualTargets and successfulTargets fields of * the request object are populated * @param result the build result that is the mutable result of this build * @param validator target validator * @param queryExpression if a query is to run after building targets, this will be the query * expression. If not, this will be null. */ public void buildTargets( BuildRequest request, BuildResult result, TargetValidator validator, QueryExpression queryExpression) throws BuildFailedException, InterruptedException, ViewCreationFailedException, TargetParsingException, LoadingFailedException, AbruptExitException, InvalidConfigurationException, TestExecException, ConfiguredTargetQueryCommandLineException { validateOptions(request); BuildOptions buildOptions = runtime.createBuildOptions(request); // Sync the package manager before sending the BuildStartingEvent in runLoadingPhase() env.setupPackageCache(request, DefaultsPackage.getDefaultsPackageContent(buildOptions)); ExecutionTool executionTool = null; boolean catastrophe = false; try { env.getEventBus().post(new BuildStartingEvent(env, request)); logger.info("Build identifier: " + request.getId()); // Error out early if multi_cpus is set, but we're not in build or test command. if (!request.getMultiCpus().isEmpty()) { getReporter().handle(Event.warn( "The --experimental_multi_cpu option is _very_ experimental and only intended for " + "internal testing at this time. If you do not work on the build tool, then you " + "should stop now!")); if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) { throw new InvalidConfigurationException( "The experimental setting to select multiple CPUs is only supported for 'build' and " + "'test' right now!"); } } // Exit if there are any pending exceptions from modules. env.throwPendingException(); // Target pattern evaluation. LoadingResult loadingResult = evaluateTargetPatterns(request, validator); env.setWorkspaceName(loadingResult.getWorkspaceName()); executionTool = new ExecutionTool(env, request); if (needsExecutionPhase(request.getBuildOptions())) { executionTool.init(); } // Compute the heuristic instrumentation filter if needed. if (request.needsInstrumentationFilter()) { String instrumentationFilter = InstrumentationFilterSupport.computeInstrumentationFilter( env.getReporter(), loadingResult.getTestsToRun()); try { // We're modifying the buildOptions in place, which is not ideal, but we also don't want // to pay the price for making a copy. Maybe reconsider later if this turns out to be a // problem (and the performance loss may not be a big deal). buildOptions.get(BuildConfiguration.Options.class).instrumentationFilter = new RegexFilter.RegexFilterConverter().convert(instrumentationFilter); } catch (OptionsParsingException e) { throw new InvalidConfigurationException(e); } } // Exit if there are any pending exceptions from modules. env.throwPendingException(); // Configuration creation. // TODO(gregce): Consider dropping this phase and passing on-the-fly target / host configs as // needed. This requires cleaning up the invalidation in SkyframeBuildView.setConfigurations. BuildConfigurationCollection configurations = env.getSkyframeExecutor() .createConfigurations( env.getReporter(), runtime.getConfigurationFragmentFactories(), buildOptions, request.getMultiCpus(), request.getKeepGoing()); env.throwPendingException(); if (configurations.getTargetConfigurations().size() == 1) { // TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case // where there's only a single configuration, but we don't send an event in the multi-config // case. Can we do better? [multi-config] env.getEventBus().post(new MakeEnvironmentEvent( configurations.getTargetConfigurations().get(0).getMakeEnvironment())); } logger.info("Configurations created"); if (request.getBuildOptions().performAnalysisPhase) { AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations); result.setBuildConfigurationCollection(configurations); result.setActualTargets(analysisResult.getTargetsToBuild()); result.setTestTargets(analysisResult.getTargetsToTest()); reportTargets(analysisResult); for (ConfiguredTarget target : analysisResult.getTargetsToSkip()) { BuildConfiguration config = target.getConfiguration(); Label label = target.getLabel(); env.getEventBus().post( new AbortedEvent( BuildEventId.targetCompleted(label, config.getEventId()), AbortReason.SKIPPED, String.format("Target %s build was skipped.", label), label)); } // TODO(janakr): this query will operate over the graph as constructed by analysis, but will // also pick up any nodes that are in the graph from prior builds. This makes the results // not reproducible at the level of a single command. Either tolerate, or wipe the analysis // graph beforehand if this option is specified, or add another option to wipe if desired // (SkyframeExecutor#handleConfiguredTargetChange should be sufficient). if (queryExpression != null) { if (!env.getSkyframeExecutor().tracksStateForIncrementality()) { throw new ConfiguredTargetQueryCommandLineException( "Configured query is not allowed if incrementality state is not being kept"); } try { doConfiguredTargetQuery( request, configurations.getHostConfiguration(), analysisResult.getTopLevelTargetsWithConfigs(), queryExpression); } catch (QueryException | IOException e) { if (!request.getKeepGoing()) { throw new ViewCreationFailedException("Error doing configured target query", e); } env.getReporter().error(null, "Error doing configured target query", e); } } // Execution phase. if (needsExecutionPhase(request.getBuildOptions())) { executionTool.executeBuild( request.getId(), analysisResult, result, configurations, analysisResult.getPackageRoots(), request.getTopLevelArtifactContext()); } else { getReporter().post(new NoExecutionEvent()); } String delayedErrorMsg = analysisResult.getError(); if (delayedErrorMsg != null) { throw new BuildFailedException(delayedErrorMsg); } } else { getReporter().handle(Event.progress("Loading complete.")); getReporter().post(new NoAnalyzeEvent()); logger.info("No analysis requested, so finished"); String errorMessage = BuildView.createErrorMessage(loadingResult, null); if (errorMessage != null) { throw new BuildFailedException(errorMessage); } // Return. } } catch (RuntimeException e) { // Print an error message for unchecked runtime exceptions. This does not concern Error // subclasses such as OutOfMemoryError. request.getOutErr().printErrLn( "Unhandled exception thrown during build; message: " + e.getMessage()); catastrophe = true; throw e; } catch (Error e) { catastrophe = true; throw e; } catch (InvalidConfigurationException e) { // TODO(gregce): With "global configurations" we cannot tie a configuration creation failure // to a single target and have to halt the entire build. Once configurations are genuinely // created as part of the analysis phase they should report their error on the level of the // target(s) that triggered them. catastrophe = true; throw e; } finally { if (executionTool != null) { executionTool.shutdown(); } if (!catastrophe) { // Delete dirty nodes to ensure that they do not accumulate indefinitely. long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc; if (versionWindow != -1) { env.getSkyframeExecutor().deleteOldNodes(versionWindow); } // The workspace status actions will not run with certain flags, or if an error // occurs early in the build. Tell a lie so that the event is not missing. // If multiple build_info events are sent, only the first is kept, so this does not harm // successful runs (which use the workspace status action). env.getEventBus() .post( new BuildInfoEvent( env.getBlazeWorkspace().getWorkspaceStatusActionFactory() .createDummyWorkspaceStatus())); } } } private void reportExceptionError(Exception e) { if (e.getMessage() != null) { getReporter().handle(Event.error(e.getMessage())); } } public BuildResult processRequest(BuildRequest request, TargetValidator validator) { return processRequest(request, validator, null); } /** * The crux of the build system. Builds the targets specified in the request using the specified * Executor. * *
Performs loading, analysis and execution for the specified set of targets, honoring the * configuration options in the BuildRequest. Also performs a query on results of analysis phase * if a query expression is specified. Returns normally iff successful, throws an exception * otherwise. * *
The caller is responsible for setting up and syncing the package cache. * *
During this function's execution, the actualTargets and successfulTargets
* fields of the request object are set.
*
* @param request the build request that this build tool is servicing, which specifies various
* options; during this method's execution, the actualTargets and successfulTargets fields
* of the request object are populated
* @param validator target validator
* @param queryExpression if a query is to run after building targets, this will be the query
* expression. If not, this will be null.
* @return the result as a {@link BuildResult} object
*/
public BuildResult processRequest(
BuildRequest request, TargetValidator validator, QueryExpression queryExpression) {
BuildResult result = new BuildResult(request.getStartTime());
env.getEventBus().register(result);
maybeSetStopOnFirstFailure(request, result);
Throwable catastrophe = null;
ExitCode exitCode = ExitCode.BLAZE_INTERNAL_ERROR;
try {
buildTargets(request, result, validator, queryExpression);
exitCode = ExitCode.SUCCESS;
} catch (BuildFailedException e) {
if (e.isErrorAlreadyShown()) {
// The actual error has already been reported by the Builder.
} else {
reportExceptionError(e);
}
if (e.isCatastrophic()) {
result.setCatastrophe();
}
exitCode = e.getExitCode() != null ? e.getExitCode() : ExitCode.BUILD_FAILURE;
} catch (InterruptedException e) {
// We may have been interrupted by an error, or the user's interruption may have raced with
// an error, so check to see if we should report that error code instead.
exitCode = env.getPendingExitCode();
if (exitCode == null) {
exitCode = ExitCode.INTERRUPTED;
env.getReporter().handle(Event.error("build interrupted"));
env.getEventBus().post(new BuildInterruptedEvent());
} else {
// Report the exception from the environment - the exception we're handling here is just an
// interruption.
reportExceptionError(env.getPendingException());
result.setCatastrophe();
}
} catch (TargetParsingException | LoadingFailedException | ViewCreationFailedException e) {
exitCode = ExitCode.PARSING_FAILURE;
reportExceptionError(e);
} catch (ConfiguredTargetQueryCommandLineException e) {
exitCode = ExitCode.COMMAND_LINE_ERROR;
reportExceptionError(e);
} catch (TestExecException e) {
// ExitCode.SUCCESS means that build was successful. Real return code of program
// is going to be calculated in TestCommand.doTest().
exitCode = ExitCode.SUCCESS;
reportExceptionError(e);
} catch (InvalidConfigurationException e) {
exitCode = ExitCode.COMMAND_LINE_ERROR;
reportExceptionError(e);
// TODO(gregce): With "global configurations" we cannot tie a configuration creation failure
// to a single target and have to halt the entire build. Once configurations are genuinely
// created as part of the analysis phase they should report their error on the level of the
// target(s) that triggered them.
result.setCatastrophe();
} catch (AbruptExitException e) {
exitCode = e.getExitCode();
reportExceptionError(e);
result.setCatastrophe();
} catch (Throwable throwable) {
catastrophe = throwable;
Throwables.propagate(throwable);
} finally {
stopRequest(result, catastrophe, exitCode);
}
return result;
}
private void doConfiguredTargetQuery(
BuildRequest request,
BuildConfiguration hostConfiguration,
List
* Postcondition: On success, populates the BuildRequest's set of targets to
* build.
*
* @return null if loading / analysis phases were successful; a useful error
* message if loading or analysis phase errors were encountered and
* request.keepGoing.
* @throws InterruptedException if the current thread was interrupted.
* @throws ViewCreationFailedException if analysis failed for any reason.
*/
private AnalysisResult runAnalysisPhase(BuildRequest request, LoadingResult loadingResult,
BuildConfigurationCollection configurations)
throws InterruptedException, ViewCreationFailedException {
Stopwatch timer = Stopwatch.createStarted();
getReporter().handle(Event.progress("Loading complete. Analyzing..."));
Profiler.instance().markPhase(ProfilePhase.ANALYZE);
BuildView view = new BuildView(env.getDirectories(), runtime.getRuleClassProvider(),
env.getSkyframeExecutor(), runtime.getCoverageReportActionFactory(request));
AnalysisResult analysisResult =
view.update(
loadingResult,
configurations,
request.getAspects(),
request.getViewOptions(),
request.getKeepGoing(),
request.getLoadingPhaseThreadCount(),
request.getTopLevelArtifactContext(),
env.getReporter(),
env.getEventBus());
// TODO(bazel-team): Merge these into one event.
env.getEventBus().post(new AnalysisPhaseCompleteEvent(analysisResult.getTargetsToBuild(),
view.getTargetsVisited(), timer.stop().elapsed(TimeUnit.MILLISECONDS),
view.getAndClearPkgManagerStatistics()));
env.getEventBus().post(new TestFilteringCompleteEvent(analysisResult.getTargetsToBuild(),
analysisResult.getTargetsToTest()));
// Check licenses.
// We check licenses if the first target configuration has license checking enabled. Right now,
// it is not possible to have multiple target configurations with different settings for this
// flag, which allows us to take this short cut.
boolean checkLicenses = configurations.getTargetConfigurations().get(0).checkLicenses();
if (checkLicenses) {
Profiler.instance().markPhase(ProfilePhase.LICENSE);
validateLicensingForTargets(analysisResult.getTargetsToBuild(), request.getKeepGoing());
}
return analysisResult;
}
private static boolean needsExecutionPhase(BuildRequestOptions options) {
return options.performAnalysisPhase && options.performExecutionPhase;
}
/**
* Stops processing the specified request.
*
* This logs the build result, cleans up and stops the clock.
*
* @param crash Any unexpected RuntimeException or Error. May be null
* @param exitCondition A suggested exit condition from either the build logic or
* a thrown exception somewhere along the way.
*/
public void stopRequest(BuildResult result, Throwable crash, ExitCode exitCondition) {
Preconditions.checkState((crash == null) || !exitCondition.equals(ExitCode.SUCCESS));
result.setUnhandledThrowable(crash);
result.setExitCondition(exitCondition);
// The stop time has to be captured before we send the BuildCompleteEvent.
result.setStopTime(runtime.getClock().currentTimeMillis());
env.getEventBus()
.post(new BuildCompleteEvent(result, ImmutableList.of(BuildEventId.buildToolLogs())));
}
private void reportTargets(AnalysisResult analysisResult) {
Collection Issues warnings for the use of deprecated options, and warnings or errors for any option
* settings that conflict.
*/
@VisibleForTesting
public void validateOptions(BuildRequest request) throws InvalidConfigurationException {
for (String issue : request.validateOptions()) {
getReporter().handle(Event.warn(issue));
}
}
/**
* Takes a set of configured targets, and checks if the distribution methods
* declared for the targets are compatible with the constraints imposed by
* their prerequisites' licenses.
*
* @param configuredTargets the targets to check
* @param keepGoing if false, and a licensing error is encountered, both
* generates an error message on the reporter, and throws an
* exception. If true, then just generates a message on the reporter.
* @throws ViewCreationFailedException if the license checking failed (and not
* --keep_going)
*/
private void validateLicensingForTargets(Iterable