// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.buildtool;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
import com.google.common.base.Throwables;
import com.google.common.base.Verify;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.BuildFailedException;
import com.google.devtools.build.lib.actions.TestExecException;
import com.google.devtools.build.lib.analysis.AnalysisPhaseCompleteEvent;
import com.google.devtools.build.lib.analysis.BuildInfoEvent;
import com.google.devtools.build.lib.analysis.BuildView;
import com.google.devtools.build.lib.analysis.BuildView.AnalysisResult;
import com.google.devtools.build.lib.analysis.ConfigurationsCreatedEvent;
import com.google.devtools.build.lib.analysis.ConfiguredAttributeMapper;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.LicensesProvider;
import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense;
import com.google.devtools.build.lib.analysis.MakeEnvironmentEvent;
import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.DefaultsPackage;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.analysis.constraints.ConstraintSemantics;
import com.google.devtools.build.lib.analysis.constraints.EnvironmentCollection;
import com.google.devtools.build.lib.analysis.constraints.SupportedEnvironmentsProvider;
import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions;
import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent;
import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent;
import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.OutputFilter;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.packages.InputFile;
import com.google.devtools.build.lib.packages.License;
import com.google.devtools.build.lib.packages.License.DistributionType;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider;
import com.google.devtools.build.lib.pkgcache.LoadingCallback;
import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
import com.google.devtools.build.lib.pkgcache.LoadingResult;
import com.google.devtools.build.lib.profiler.ProfilePhase;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.runtime.BlazeRuntime;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.Collection;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import java.util.regex.Pattern;
/**
* Provides the bulk of the implementation of the 'blaze build' command.
*
*
The various concrete build command classes handle the command options and request
* setup, then delegate the handling of the request (the building of targets) to this class.
*
*
The main entry point is {@link #buildTargets}.
*
*
This class is always instantiated and managed as a singleton, being constructed and held by
* {@link BlazeRuntime}. This is so multiple kinds of build commands can share this single
* instance.
*
*
Most of analysis is handled in {@link BuildView}, and execution in {@link ExecutionTool}.
*/
public final class BuildTool {
private static final Logger LOG = Logger.getLogger(BuildTool.class.getName());
private final CommandEnvironment env;
private final BlazeRuntime runtime;
/**
* Constructs a BuildTool.
*
* @param env a reference to the command environment of the currently executing command
*/
public BuildTool(CommandEnvironment env) {
this.env = env;
this.runtime = env.getRuntime();
}
/**
* The crux of the build system. Builds the targets specified in the request using the specified
* Executor.
*
*
Performs loading, analysis and execution for the specified set of targets, honoring the
* configuration options in the BuildRequest. Returns normally iff successful, throws an exception
* otherwise.
*
*
Callers must ensure that {@link #stopRequest} is called after this method, even if it
* throws.
*
*
The caller is responsible for setting up and syncing the package cache.
*
*
During this function's execution, the actualTargets and successfulTargets
* fields of the request object are set.
*
* @param request the build request that this build tool is servicing, which specifies various
* options; during this method's execution, the actualTargets and successfulTargets fields
* of the request object are populated
* @param result the build result that is the mutable result of this build
* @param validator target validator
*/
public void buildTargets(BuildRequest request, BuildResult result, TargetValidator validator)
throws BuildFailedException, InterruptedException, ViewCreationFailedException,
TargetParsingException, LoadingFailedException, AbruptExitException,
InvalidConfigurationException, TestExecException {
validateOptions(request);
BuildOptions buildOptions = runtime.createBuildOptions(request);
// Sync the package manager before sending the BuildStartingEvent in runLoadingPhase()
env.setupPackageCache(request.getPackageCacheOptions(),
DefaultsPackage.getDefaultsPackageContent(buildOptions));
ExecutionTool executionTool = null;
LoadingResult loadingResult = null;
BuildConfigurationCollection configurations = null;
boolean catastrophe = false;
try {
env.getEventBus().post(new BuildStartingEvent(env.getOutputFileSystem(), request));
LOG.info("Build identifier: " + request.getId());
executionTool = new ExecutionTool(env, request);
if (needsExecutionPhase(request.getBuildOptions())) {
// Initialize the execution tool early if we need it. This hides the latency of setting up
// the execution backends.
executionTool.init();
}
// Loading phase.
loadingResult = runLoadingPhase(request, validator);
// Create the build configurations.
if (!request.getMultiCpus().isEmpty()) {
getReporter().handle(Event.warn(
"The --experimental_multi_cpu option is _very_ experimental and only intended for "
+ "internal testing at this time. If you do not work on the build tool, then you "
+ "should stop now!"));
if (!"build".equals(request.getCommandName()) && !"test".equals(request.getCommandName())) {
throw new InvalidConfigurationException(
"The experimental setting to select multiple CPUs is only supported for 'build' and "
+ "'test' right now!");
}
}
configurations = env.getSkyframeExecutor().createConfigurations(
env.getReporter(), runtime.getConfigurationFactory(), buildOptions,
request.getMultiCpus(), request.getViewOptions().keepGoing);
env.getEventBus().post(new ConfigurationsCreatedEvent(configurations));
env.throwPendingException();
if (configurations.getTargetConfigurations().size() == 1) {
// TODO(bazel-team): This is not optimal - we retain backwards compatibility in the case
// where there's only a single configuration, but we don't send an event in the multi-config
// case. Can we do better? [multi-config]
env.getEventBus().post(new MakeEnvironmentEvent(
configurations.getTargetConfigurations().get(0).getMakeEnvironment()));
}
LOG.info("Configurations created");
// Analysis phase.
AnalysisResult analysisResult = runAnalysisPhase(request, loadingResult, configurations);
result.setBuildConfigurationCollection(configurations);
result.setActualTargets(analysisResult.getTargetsToBuild());
result.setTestTargets(analysisResult.getTargetsToTest());
LoadedPackageProvider bridge =
new LoadedPackageProvider(env.getPackageManager(), env.getReporter());
checkTargetEnvironmentRestrictions(analysisResult.getTargetsToBuild(), bridge);
reportTargets(analysisResult);
// Execution phase.
if (needsExecutionPhase(request.getBuildOptions())) {
env.getSkyframeExecutor().injectTopLevelContext(request.getTopLevelArtifactContext());
executionTool.executeBuild(request.getId(), analysisResult, result,
configurations, transformPackageRoots(analysisResult.getPackageRoots()));
}
String delayedErrorMsg = analysisResult.getError();
if (delayedErrorMsg != null) {
throw new BuildFailedException(delayedErrorMsg);
}
} catch (RuntimeException e) {
// Print an error message for unchecked runtime exceptions. This does not concern Error
// subclasses such as OutOfMemoryError.
request.getOutErr().printErrLn("Unhandled exception thrown during build; message: " +
e.getMessage());
catastrophe = true;
throw e;
} catch (Error e) {
catastrophe = true;
throw e;
} finally {
if (!catastrophe) {
// Delete dirty nodes to ensure that they do not accumulate indefinitely.
long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc;
if (versionWindow != -1) {
env.getSkyframeExecutor().deleteOldNodes(versionWindow);
}
if (executionTool != null) {
executionTool.shutdown();
}
// The workspace status actions will not run with certain flags, or if an error
// occurs early in the build. Tell a lie so that the event is not missing.
// If multiple build_info events are sent, only the first is kept, so this does not harm
// successful runs (which use the workspace status action).
env.getEventBus()
.post(
new BuildInfoEvent(
env.getBlazeWorkspace().getWorkspaceStatusActionFactory()
.createDummyWorkspaceStatus()));
}
}
if (loadingResult != null && loadingResult.hasTargetPatternError()) {
throw new BuildFailedException("execution phase successful, but there were errors " +
"parsing the target pattern");
}
}
/**
* Checks that if this is an environment-restricted build, all top-level targets support
* the expected environments.
*
* @param topLevelTargets the build's top-level targets
* @throws ViewCreationFailedException if constraint enforcement is on, the build declares
* environment-restricted top level configurations, and any top-level target doesn't
* support the expected environments
*/
private void checkTargetEnvironmentRestrictions(Iterable topLevelTargets,
LoadedPackageProvider packageManager) throws ViewCreationFailedException {
for (ConfiguredTarget topLevelTarget : topLevelTargets) {
BuildConfiguration config = topLevelTarget.getConfiguration();
if (config == null) {
// TODO(bazel-team): support file targets (they should apply package-default constraints).
continue;
} else if (!config.enforceConstraints() || config.getTargetEnvironments().isEmpty()) {
continue;
}
// Parse and collect this configuration's environments.
EnvironmentCollection.Builder builder = new EnvironmentCollection.Builder();
for (Label envLabel : config.getTargetEnvironments()) {
try {
Target env = packageManager.getLoadedTarget(envLabel);
builder.put(ConstraintSemantics.getEnvironmentGroup(env), envLabel);
} catch (NoSuchPackageException | NoSuchTargetException
| ConstraintSemantics.EnvironmentLookupException e) {
throw new ViewCreationFailedException("invalid target environment", e);
}
}
EnvironmentCollection expectedEnvironments = builder.build();
// Now check the target against those environments.
SupportedEnvironmentsProvider provider =
Verify.verifyNotNull(topLevelTarget.getProvider(SupportedEnvironmentsProvider.class));
Collection