diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/runtime')
60 files changed, 10557 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java new file mode 100644 index 0000000000..9bf7a3f27b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/AbstractCriticalPathComponent.java @@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; + +import javax.annotation.Nullable; + +/** + * This class records the critical path for the graph of actions executed. + */ +@ThreadCompatible +public class AbstractCriticalPathComponent<C extends AbstractCriticalPathComponent<C>> { + + /** Wall time start time for the action. In milliseconds. */ + private final long startTime; + /** Wall time finish time for the action. In milliseconds. */ + private long finishTime = 0; + protected volatile boolean isRunning = true; + + /** We keep here the critical path time for the most expensive child. */ + private long childAggregatedWallTime = 0; + + /** The action for which we are storing the stat. */ + private final Action action; + + /** + * Child with the maximum critical path. + */ + @Nullable + private C child; + + public AbstractCriticalPathComponent(Action action, long startTime) { + this.action = action; + this.startTime = startTime; + } + + /** Sets the finish time for the action in milliseconds. */ + public void setFinishTimeMillis(long finishTime) { + Preconditions.checkState(isRunning, "Already stopped! %s.", action); + this.finishTime = finishTime; + isRunning = false; + } + + /** The action for which we are storing the stat. */ + public Action getAction() { + return action; + } + + /** + * Add statistics for one dependency of this action. + */ + public void addDepInfo(C dep) { + Preconditions.checkState(!dep.isRunning, + "Cannot add critical path stats when the action is not finished. %s. %s", action, + dep.getAction()); + long childAggregatedWallTime = dep.getAggregatedWallTime(); + // Replace the child if its critical path had the maximum wall time. + if (child == null || childAggregatedWallTime > this.childAggregatedWallTime) { + this.childAggregatedWallTime = childAggregatedWallTime; + child = dep; + } + } + + public long getActionWallTime() { + Preconditions.checkState(!isRunning, "Still running %s", action); + return finishTime - startTime; + } + + /** + * Returns the current critical path for the action in milliseconds. + * + * <p>Critical path is defined as : action_execution_time + max(child_critical_path). + */ + public long getAggregatedWallTime() { + Preconditions.checkState(!isRunning, "Still running %s", action); + return getActionWallTime() + childAggregatedWallTime; + } + + /** Time when the action started to execute. Milliseconds since epoch time. */ + public long getStartTime() { + return startTime; + } + + /** + * Get the child critical path component. + * + * <p>The component dependency with the maximum total critical path time. + */ + @Nullable + public C getChild() { + return child; + } + + /** + * Returns a human readable representation of the critical path stats with all the details. + */ + @Override + public String toString() { + String currentTime = "still running "; + if (!isRunning) { + currentTime = String.format("%.2f", getActionWallTime() / 1000.0) + "s "; + } + return currentTime + action.describe(); + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java new file mode 100644 index 0000000000..dd70c3520d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatedCriticalPath.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; + +/** + * Aggregates all the critical path components in one object. This allows us to easily access the + * components data and have a proper toString(). + */ +public class AggregatedCriticalPath<T extends AbstractCriticalPathComponent> { + + private final long totalTime; + private final ImmutableList<T> criticalPathComponents; + + protected AggregatedCriticalPath(long totalTime, ImmutableList<T> criticalPathComponents) { + this.totalTime = totalTime; + this.criticalPathComponents = criticalPathComponents; + } + + /** Total wall time in ms spent running the critical path actions. */ + public long totalTime() { + return totalTime; + } + + /** Returns a list of all the component stats for the critical path. */ + public ImmutableList<T> components() { + return criticalPathComponents; + } + + @Override + public String toString() { + return toString(false); + } + + /** + * Returns a summary version of the critical path stats that omits stats that are not useful + * to the user. + */ + public String toStringSummary() { + return toString(true); + } + + private String toString(boolean summary) { + StringBuilder sb = new StringBuilder("Critical Path: "); + double totalMillis = totalTime; + sb.append(String.format("%.2f", totalMillis / 1000.0)); + sb.append("s"); + if (summary || criticalPathComponents.isEmpty()) { + return sb.toString(); + } + sb.append("\n "); + Joiner.on("\n ").appendTo(sb, criticalPathComponents); + return sb.toString(); + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java new file mode 100644 index 0000000000..cc240c41ba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/AggregatingTestListener.java @@ -0,0 +1,255 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MapMaker; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.common.eventbus.AllowConcurrentEvents; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisFailureEvent; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.LabelAndConfiguration; +import com.google.devtools.build.lib.analysis.TargetCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; +import com.google.devtools.build.lib.buildtool.buildevent.TestFilteringCompleteEvent; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.events.ExceptionListener; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; + +/** + * This class aggregates and reports target-wide test statuses in real-time. + * It must be public for EventBus invocation. + */ +@ThreadSafety.ThreadSafe +public class AggregatingTestListener { + private final ConcurrentMap<Artifact, TestResult> statusMap = new MapMaker().makeMap(); + + private final TestResultAnalyzer analyzer; + private final EventBus eventBus; + private final EventHandlerPreconditions preconditionHelper; + private volatile boolean blazeHalted = false; + + + // summaryLock guards concurrent access to these two collections, which should be kept + // synchronized with each other. + private final Map<LabelAndConfiguration, TestSummary.Builder> summaries; + private final Multimap<LabelAndConfiguration, Artifact> remainingRuns; + private final Object summaryLock = new Object(); + + public AggregatingTestListener(TestResultAnalyzer analyzer, + EventBus eventBus, + ExceptionListener listener) { + this.analyzer = analyzer; + this.eventBus = eventBus; + this.preconditionHelper = new EventHandlerPreconditions(listener); + + this.summaries = Maps.newHashMap(); + this.remainingRuns = HashMultimap.create(); + } + + /** + * @return An unmodifiable copy of the map of test results. + */ + public Map<Artifact, TestResult> getStatusMap() { + return ImmutableMap.copyOf(statusMap); + } + + /** + * Populates the test summary map as soon as test filtering is complete. + * This is the earliest at which the final set of targets to test is known. + */ + @Subscribe + @AllowConcurrentEvents + public void populateTests(TestFilteringCompleteEvent event) { + // Add all target runs to the map, assuming 1:1 status artifact <-> result. + synchronized (summaryLock) { + for (ConfiguredTarget target : event.getTestTargets()) { + Iterable<Artifact> statusArtifacts = + target.getProvider(TestProvider.class).getTestParams().getTestStatusArtifacts(); + preconditionHelper.checkState(remainingRuns.putAll(asKey(target), statusArtifacts)); + + // And create an empty summary suitable for incremental analysis. + // Also has the nice side effect of mapping labels to RuleConfiguredTargets. + TestSummary.Builder summary = TestSummary.newBuilder() + .setTarget(target) + .setStatus(BlazeTestStatus.NO_STATUS); + preconditionHelper.checkState(summaries.put(asKey(target), summary) == null); + } + } + } + + /** + * Records a new test run result and incrementally updates the target status. + * This event is sent upon completion of executed test runs. + */ + @Subscribe + @AllowConcurrentEvents + public void testEvent(TestResult result) { + Preconditions.checkState( + statusMap.put(result.getTestStatusArtifact(), result) == null, + "Duplicate result reported for an individual test shard"); + + ActionOwner testOwner = result.getTestAction().getOwner(); + LabelAndConfiguration targetLabel = LabelAndConfiguration.of( + testOwner.getLabel(), result.getTestAction().getConfiguration()); + + TestSummary finalTestSummary = null; + synchronized (summaryLock) { + TestSummary.Builder summary = summaries.get(targetLabel); + preconditionHelper.checkNotNull(summary); + if (!remainingRuns.remove(targetLabel, result.getTestStatusArtifact())) { + // This can happen if a buildCompleteEvent() was processed before this event reached us. + // This situation is likely to happen if --notest_keep_going is set with multiple targets. + return; + } + + summary = analyzer.incrementalAnalyze(summary, result); + + // If all runs are processed, the target is finished and ready to report. + if (!remainingRuns.containsKey(targetLabel)) { + finalTestSummary = summary.build(); + } + } + + // Report finished targets. + if (finalTestSummary != null) { + eventBus.post(finalTestSummary); + } + } + + private void targetFailure(LabelAndConfiguration label) { + TestSummary finalSummary; + synchronized (summaryLock) { + if (!remainingRuns.containsKey(label)) { + // Blaze does not guarantee that BuildResult.getSuccessfulTargets() and posted TestResult + // events are in sync. Thus, it is possible that a test event was posted, but the target is + // not present in the set of successful targets. + return; + } + + TestSummary.Builder summary = summaries.get(label); + if (summary == null) { + // Not a test target; nothing to do. + return; + } + finalSummary = analyzer.markUnbuilt(summary, blazeHalted).build(); + + // These are never going to run; removing them marks the target complete. + remainingRuns.removeAll(label); + } + eventBus.post(finalSummary); + } + + @VisibleForTesting + void buildComplete( + Collection<ConfiguredTarget> actualTargets, Collection<ConfiguredTarget> successfulTargets) { + if (actualTargets == null || successfulTargets == null) { + return; + } + + for (ConfiguredTarget target: Sets.difference( + ImmutableSet.copyOf(actualTargets), ImmutableSet.copyOf(successfulTargets))) { + targetFailure(asKey(target)); + } + } + + @Subscribe + public void buildCompleteEvent(BuildCompleteEvent event) { + if (event.getResult().wasCatastrophe()) { + blazeHalted = true; + } + buildComplete(event.getResult().getActualTargets(), event.getResult().getSuccessfulTargets()); + } + + @Subscribe + public void analysisFailure(AnalysisFailureEvent event) { + targetFailure(event.getFailedTarget()); + } + + @Subscribe + @AllowConcurrentEvents + public void buildInterrupted(BuildInterruptedEvent event) { + blazeHalted = true; + } + + /** + * Called when a build action is not executed (e.g. because a dependency failed to build). We want + * to catch such events in order to determine when a test target has failed to build. + */ + @Subscribe + @AllowConcurrentEvents + public void targetComplete(TargetCompleteEvent event) { + if (event.failed()) { + targetFailure(new LabelAndConfiguration(event.getTarget())); + } + } + + /** + * Returns the known aggregate results for the given target at the current moment. + */ + public TestSummary.Builder getCurrentSummary(ConfiguredTarget target) { + synchronized (summaryLock) { + return summaries.get(asKey(target)); + } + } + + /** + * Returns all test status artifacts associated with a given target + * whose runs have yet to finish. + */ + public Collection<Artifact> getIncompleteRuns(ConfiguredTarget target) { + synchronized (summaryLock) { + return Collections.unmodifiableCollection(remainingRuns.get(asKey(target))); + } + } + + /** + * Returns true iff all runs of the target are accounted for. + */ + public boolean targetReported(ConfiguredTarget target) { + synchronized (summaryLock) { + return summaries.containsKey(asKey(target)) && !remainingRuns.containsKey(asKey(target)); + } + } + + /** + * Returns the {@link TestResultAnalyzer} associated with this listener. + */ + public TestResultAnalyzer getAnalyzer() { + return analyzer; + } + + private LabelAndConfiguration asKey(ConfiguredTarget target) { + return new LabelAndConfiguration(target); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java new file mode 100644 index 0000000000..61f46a8713 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommand.java @@ -0,0 +1,63 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * Interface implemented by Blaze commands. In addition to implementing this interface, each + * command must be annotated with a {@link Command} annotation. + */ +public interface BlazeCommand { + /** + * This method provides the imperative portion of the command. It takes + * a {@link OptionsProvider} instance {@code options}, which provides access + * to the options instances via {@link OptionsProvider#getOptions(Class)}, + * and access to the residue (the remainder of the command line) via + * {@link OptionsProvider#getResidue()}. The framework parses and makes + * available exactly the options that the command class specifies via the + * annotation {@link Command#options()}. The command may write to standard + * out and standard error via {@code outErr}. It indicates success / failure + * via its return value, which becomes the Unix exit status of the Blaze + * client process. It may indicate a shutdown request by throwing + * {@link BlazeCommandDispatcher.ShutdownBlazeServerException}. In that case, + * the Blaze server process (the memory resident portion of Blaze) will + * shut down and the exit status will be 0 (in case the shutdown succeeds + * without error). + * + * @param runtime The Blaze runtime requesting the execution of the command + * @param options A parsed options instance initialized with the values for + * the options specified in {@link Command#options()}. + * + * @return The Unix exit status for the Blaze client. + * @throws BlazeCommandDispatcher.ShutdownBlazeServerException Indicates + * that the command wants to shutdown the Blaze server. + */ + ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws BlazeCommandDispatcher.ShutdownBlazeServerException; + + /** + * Allows the command to provide command-specific option defaults and/or + * requirements. This method is called after all command-line and rc file options have been + * parsed. + * + * @param runtime The Blaze runtime requesting the execution of the command + * + * @throws AbruptExitException if something went wrong + */ + void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) throws AbruptExitException; +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java new file mode 100644 index 0000000000..cee47ee2fd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java @@ -0,0 +1,692 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.io.Flushables; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.AnsiStrippingOutputStream; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.DelegatingOutErr; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.logging.Level; + +/** + * Dispatches to the Blaze commands; that is, given a command line, this + * abstraction looks up the appropriate command object, parses the options + * required by the object, and calls its exec method. Also, this object provides + * the runtime state (BlazeRuntime) to the commands. + */ +public class BlazeCommandDispatcher { + + // Keep in sync with options added in OptionProcessor::AddRcfileArgsAndOptions() + private static final Set<String> INTERNAL_COMMAND_OPTIONS = ImmutableSet.of( + "rc_source", "default_override", "isatty", "terminal_columns", "ignore_client_env", + "client_env", "client_cwd"); + + private static final ImmutableList<String> HELP_COMMAND = ImmutableList.of("help"); + + private static final Set<String> ALL_HELP_OPTIONS = ImmutableSet.of("--help", "-help", "-h"); + + /** + * By throwing this exception, a command indicates that it wants to shutdown + * the Blaze server process. + * See {@link BlazeCommandDispatcher#exec(List, OutErr, long)}. + */ + public static class ShutdownBlazeServerException extends Exception { + private final int exitStatus; + + public ShutdownBlazeServerException(int exitStatus, Throwable cause) { + super(cause); + this.exitStatus = exitStatus; + } + + public ShutdownBlazeServerException(int exitStatus) { + this.exitStatus = exitStatus; + } + + public int getExitStatus() { + return exitStatus; + } + } + + private final BlazeRuntime runtime; + private final Map<String, BlazeCommand> commandsByName = new LinkedHashMap<>(); + + private OutputStream logOutputStream = null; + + /** + * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime} + * instance, and no default options, and delegates to {@code commands} as + * appropriate. + */ + @VisibleForTesting + public BlazeCommandDispatcher(BlazeRuntime runtime, BlazeCommand... commands) { + this(runtime, ImmutableList.copyOf(commands)); + } + + /** + * Create a Blaze dispatcher that uses the specified {@code BlazeRuntime} + * instance, and delegates to {@code commands} as appropriate. + */ + public BlazeCommandDispatcher(BlazeRuntime runtime, Iterable<BlazeCommand> commands) { + this.runtime = runtime; + for (BlazeCommand command : commands) { + addCommandByName(command); + } + + for (BlazeModule module : runtime.getBlazeModules()) { + for (BlazeCommand command : module.getCommands()) { + addCommandByName(command); + } + } + + runtime.setCommandMap(commandsByName); + } + + /** + * Adds the given command under the given name to the map of commands. + * + * @throws AssertionError if the name is already used by another command. + */ + private void addCommandByName(BlazeCommand command) { + String name = command.getClass().getAnnotation(Command.class).name(); + if (commandsByName.containsKey(name)) { + throw new IllegalStateException("Command name or alias " + name + " is already used."); + } + commandsByName.put(name, command); + } + + /** + * Only some commands work if cwd != workspaceSuffix in Blaze. In that case, also check if Blaze + * was called from the output directory and fail if it was. + */ + private ExitCode checkCwdInWorkspace(Command commandAnnotation, String commandName, + OutErr outErr) { + if (!commandAnnotation.mustRunInWorkspace()) { + return ExitCode.SUCCESS; + } + + if (!runtime.inWorkspace()) { + outErr.printErrLn("The '" + commandName + "' command is only supported from within a " + + "workspace."); + return ExitCode.COMMAND_LINE_ERROR; + } + + Path workspace = runtime.getWorkspace(); + Path doNotBuild = workspace.getParentDirectory().getRelative( + BlazeRuntime.DO_NOT_BUILD_FILE_NAME); + if (doNotBuild.exists()) { + if (!commandAnnotation.canRunInOutputDirectory()) { + outErr.printErrLn(getNotInRealWorkspaceError(doNotBuild)); + return ExitCode.COMMAND_LINE_ERROR; + } else { + outErr.printErrLn("WARNING: Blaze is run from output directory. This is unsound."); + } + } + return ExitCode.SUCCESS; + } + + private CommonCommandOptions checkOptions(OptionsParser optionsParser, + Command commandAnnotation, List<String> args, List<String> rcfileNotes, OutErr outErr) + throws OptionsParsingException { + Function<String, String> commandOptionSourceFunction = new Function<String, String>() { + @Override + public String apply(String input) { + if (INTERNAL_COMMAND_OPTIONS.contains(input)) { + return "options generated by Blaze launcher"; + } else { + return "command line options"; + } + } + }; + + // Explicit command-line options: + List<String> cmdLineAfterCommand = args.subList(1, args.size()); + optionsParser.parseWithSourceFunction(OptionPriority.COMMAND_LINE, + commandOptionSourceFunction, cmdLineAfterCommand); + + // Command-specific options from .blazerc passed in via --default_override + // and --rc_source. A no-op if none are provided. + CommonCommandOptions rcFileOptions = optionsParser.getOptions(CommonCommandOptions.class); + List<Pair<String, ListMultimap<String, String>>> optionsMap = + getOptionsMap(outErr, rcFileOptions.rcSource, rcFileOptions.optionsOverrides, + commandsByName.keySet()); + + parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap, null); + + // Fix-point iteration until all configs are loaded. + List<String> configsLoaded = ImmutableList.of(); + CommonCommandOptions commonOptions = optionsParser.getOptions(CommonCommandOptions.class); + while (!commonOptions.configs.equals(configsLoaded)) { + Set<String> missingConfigs = new LinkedHashSet<>(commonOptions.configs); + missingConfigs.removeAll(configsLoaded); + parseOptionsForCommand(rcfileNotes, commandAnnotation, optionsParser, optionsMap, + missingConfigs); + configsLoaded = commonOptions.configs; + commonOptions = optionsParser.getOptions(CommonCommandOptions.class); + } + + return commonOptions; + } + + /** + * Sends {@code EventKind.{STDOUT|STDERR}} messages to the given {@link OutErr}. + * + * <p>This is necessary because we cannot delete the output files from the previous Blaze run + * because there can be processes spawned by the previous invocation that are still processing + * them, in which case we need to print a warning message about that. + * + * <p>Thus, messages sent to {@link Reporter#getOutErr} get sent to this event handler, then + * to its {@link OutErr}. We need to go deeper! + */ + private static class OutErrEventHandler implements EventHandler { + private final OutErr outErr; + + private OutErrEventHandler(OutErr outErr) { + this.outErr = outErr; + } + + @Override + public void handle(Event event) { + try { + switch (event.getKind()) { + case STDOUT: + outErr.getOutputStream().write(event.getMessageBytes()); + break; + case STDERR: + outErr.getErrorStream().write(event.getMessageBytes()); + break; + } + } catch (IOException e) { + // We cannot do too much here -- ErrorEventListener#handle does not provide us with ways to + // report an error. + } + } + } + + /** + * Executes a single command. Returns the Unix exit status for the Blaze + * client process, or throws {@link ShutdownBlazeServerException} to + * indicate that a command wants to shutdown the Blaze server. + */ + public int exec(List<String> args, OutErr originalOutErr, long firstContactTime) + throws ShutdownBlazeServerException { + // Record the start time for the profiler and the timestamp granularity monitor. Do not put + // anything before this! + long execStartTimeNanos = runtime.getClock().nanoTime(); + + // Record the command's starting time for use by the commands themselves. + runtime.recordCommandStartTime(firstContactTime); + + // Record the command's starting time again, for use by + // TimestampGranularityMonitor.waitForTimestampGranularity(). + // This should be done as close as possible to the start of + // the command's execution - that's why we do this separately, + // rather than in runtime.beforeCommand(). + runtime.getTimestampGranularityMonitor().setCommandStartTime(); + runtime.initEventBus(); + + // Give a chance for module.beforeCommand() to report an errors to stdout and stderr. + // Once we can close the old streams, this event handler is removed. + OutErrEventHandler originalOutErrEventHandler = + new OutErrEventHandler(originalOutErr); + runtime.getReporter().addHandler(originalOutErrEventHandler); + OutErr outErr = originalOutErr; + runtime.getReporter().removeHandler(originalOutErrEventHandler); + + if (args.isEmpty()) { // Default to help command if no arguments specified. + args = HELP_COMMAND; + } + String commandName = args.get(0); + + // Be gentle to users who want to find out about Blaze invocation. + if (ALL_HELP_OPTIONS.contains(commandName)) { + commandName = "help"; + } + + BlazeCommand command = commandsByName.get(commandName); + if (command == null) { + outErr.printErrLn("Command '" + commandName + "' not found. " + "Try 'blaze help'."); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } + Command commandAnnotation = command.getClass().getAnnotation(Command.class); + + AbruptExitException exitCausingException = null; + for (BlazeModule module : runtime.getBlazeModules()) { + try { + module.beforeCommand(runtime, commandAnnotation); + } catch (AbruptExitException e) { + // Don't let one module's complaints prevent the other modules from doing necessary + // setup. We promised to call beforeCommand exactly once per-module before each command + // and will be calling afterCommand soon in the future - a module's afterCommand might + // rightfully assume its beforeCommand has already been called. + outErr.printErrLn(e.getMessage()); + // It's not ideal but we can only return one exit code, so we just pick the code of the + // last exception. + exitCausingException = e; + } + } + if (exitCausingException != null) { + return exitCausingException.getExitCode().getNumericExitCode(); + } + + try { + Path commandLog = getCommandLogPath(runtime.getOutputBase()); + + // Unlink old command log from previous build, if present, so scripts + // reading it don't conflate it with the command log we're about to write. + commandLog.delete(); + + logOutputStream = commandLog.getOutputStream(); + outErr = tee(originalOutErr, OutErr.create(logOutputStream, logOutputStream)); + } catch (IOException ioException) { + LoggingUtil.logToRemote( + Level.WARNING, "Unable to delete or open command.log", ioException); + } + + // Create the UUID for this command. + runtime.setCommandId(UUID.randomUUID()); + + ExitCode result = checkCwdInWorkspace(commandAnnotation, commandName, outErr); + if (result != ExitCode.SUCCESS) { + return result.getNumericExitCode(); + } + + OptionsParser optionsParser; + CommonCommandOptions commonOptions; + // Delay output of notes regarding the parsed rc file, so it's possible to disable this in the + // rc file. + List<String> rcfileNotes = new ArrayList<>(); + try { + optionsParser = createOptionsParser(command); + commonOptions = checkOptions(optionsParser, commandAnnotation, args, rcfileNotes, outErr); + } catch (OptionsParsingException e) { + for (String note : rcfileNotes) { + outErr.printErrLn("INFO: " + note); + } + outErr.printErrLn(e.getMessage()); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } + + // Setup log filtering + BlazeCommandEventHandler.Options eventHandlerOptions = + optionsParser.getOptions(BlazeCommandEventHandler.Options.class); + if (!eventHandlerOptions.useColor()) { + if (!commandAnnotation.binaryStdOut()) { + outErr = ansiStripOut(outErr); + } + + if (!commandAnnotation.binaryStdErr()) { + outErr = ansiStripErr(outErr); + } + } + + BlazeRuntime.setupLogging(commonOptions.verbosity); + + // Do this before an actual crash so we don't have to worry about + // allocating memory post-crash. + String[] crashData = runtime.getCrashData(); + int numericExitCode = ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); + PrintStream savedOut = System.out; + PrintStream savedErr = System.err; + + EventHandler handler = createEventHandler(outErr, eventHandlerOptions); + Reporter reporter = runtime.getReporter(); + reporter.addHandler(handler); + try { + // While a Blaze command is active, direct all errors to the client's + // event handler (and out/err streams). + OutErr reporterOutErr = reporter.getOutErr(); + System.setOut(new PrintStream(reporterOutErr.getOutputStream(), /*autoflush=*/true)); + System.setErr(new PrintStream(reporterOutErr.getErrorStream(), /*autoflush=*/true)); + + if (commonOptions.announceRcOptions) { + for (String note : rcfileNotes) { + reporter.handle(Event.info(note)); + } + } + + try { + // Notify the BlazeRuntime, so it can do some initial setup. + runtime.beforeCommand(commandName, optionsParser, commonOptions, execStartTimeNanos); + // Allow the command to edit options after parsing: + command.editOptions(runtime, optionsParser); + } catch (AbruptExitException e) { + reporter.handle(Event.error(e.getMessage())); + return e.getExitCode().getNumericExitCode(); + } + + // Print warnings for odd options usage + for (String warning : optionsParser.getWarnings()) { + reporter.handle(Event.warn(warning)); + } + + ExitCode outcome = command.exec(runtime, optionsParser); + outcome = runtime.precompleteCommand(outcome); + numericExitCode = outcome.getNumericExitCode(); + return numericExitCode; + } catch (ShutdownBlazeServerException e) { + numericExitCode = e.getExitStatus(); + throw e; + } catch (Throwable e) { + BugReport.printBug(outErr, e); + BugReport.sendBugReport(e, args, crashData); + numericExitCode = e instanceof OutOfMemoryError + ? ExitCode.OOM_ERROR.getNumericExitCode() + : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); + throw new ShutdownBlazeServerException(numericExitCode, e); + } finally { + runtime.afterCommand(numericExitCode); + // Swallow IOException, as we are already in a finally clause + Flushables.flushQuietly(outErr.getOutputStream()); + Flushables.flushQuietly(outErr.getErrorStream()); + + System.setOut(savedOut); + System.setErr(savedErr); + reporter.removeHandler(handler); + releaseHandler(handler); + runtime.getTimestampGranularityMonitor().waitForTimestampGranularity(outErr); + } + } + + /** + * For testing ONLY. Same as {@link #exec(List, OutErr, long)}, but automatically uses the current + * time. + */ + @VisibleForTesting + public int exec(List<String> args, OutErr originalOutErr) throws ShutdownBlazeServerException { + return exec(args, originalOutErr, runtime.getClock().currentTimeMillis()); + } + + /** + * Parses the options from .rc files for a command invocation. It works in one of two modes; + * either it loads the non-config options, or the config options that are specified in the {@code + * configs} parameter. + * + * <p>This method adds every option pertaining to the specified command to the options parser. To + * do that, it needs the command -> option mapping that is generated from the .rc files. + * + * <p>It is not as trivial as simply taking the list of options for the specified command because + * commands can inherit arguments from each other, and we have to respect that (e.g. if an option + * is specified for 'build', it needs to take effect for the 'test' command, too). + * + * <p>Note that the order in which the options are parsed is well-defined: all options from the + * same rc file are parsed at the same time, and the rc files are handled in the order in which + * they were passed in from the client. + * + * @param rcfileNotes note message that would be printed during parsing + * @param commandAnnotation the command for which options should be parsed. + * @param optionsParser parser to receive parsed options. + * @param optionsMap .rc files in structured format: a list of pairs, where the first part is the + * name of the rc file, and the second part is a multimap of command name (plus config, if + * present) to the list of options for that command + * @param configs the configs for which to parse options; if {@code null}, non-config options are + * parsed + * @throws OptionsParsingException + */ + protected static void parseOptionsForCommand(List<String> rcfileNotes, Command commandAnnotation, + OptionsParser optionsParser, List<Pair<String, ListMultimap<String, String>>> optionsMap, + Iterable<String> configs) throws OptionsParsingException { + for (String commandToParse : getCommandNamesToParse(commandAnnotation)) { + for (Pair<String, ListMultimap<String, String>> entry : optionsMap) { + List<String> allOptions = new ArrayList<>(); + if (configs == null) { + allOptions.addAll(entry.second.get(commandToParse)); + } else { + for (String config : configs) { + allOptions.addAll(entry.second.get(commandToParse + ":" + config)); + } + } + processOptionList(optionsParser, commandToParse, + commandAnnotation.name(), rcfileNotes, entry.first, allOptions); + if (allOptions.isEmpty()) { + continue; + } + } + } + } + + // Processes the option list for an .rc file - command pair. + private static void processOptionList(OptionsParser optionsParser, String commandToParse, + String originalCommand, List<String> rcfileNotes, String rcfile, List<String> rcfileOptions) + throws OptionsParsingException { + if (!rcfileOptions.isEmpty()) { + String inherited = commandToParse.equals(originalCommand) ? "" : "Inherited "; + rcfileNotes.add("Reading options for '" + originalCommand + + "' from " + rcfile + ":\n" + + " " + inherited + "'" + commandToParse + "' options: " + + Joiner.on(' ').join(rcfileOptions)); + optionsParser.parse(OptionPriority.RC_FILE, rcfile, rcfileOptions); + } + } + + private static List<String> getCommandNamesToParse(Command commandAnnotation) { + List<String> result = new ArrayList<>(); + getCommandNamesToParseHelper(commandAnnotation, result); + result.add("common"); + // TODO(bazel-team): This statement is a NO-OP: Lists.reverse(result); + return result; + } + + private static void getCommandNamesToParseHelper(Command commandAnnotation, + List<String> accumulator) { + for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) { + getCommandNamesToParseHelper(base.getAnnotation(Command.class), accumulator); + } + accumulator.add(commandAnnotation.name()); + } + + private OutErr ansiStripOut(OutErr outErr) { + OutputStream wrappedOut = new AnsiStrippingOutputStream(outErr.getOutputStream()); + return OutErr.create(wrappedOut, outErr.getErrorStream()); + } + + private OutErr ansiStripErr(OutErr outErr) { + OutputStream wrappedErr = new AnsiStrippingOutputStream(outErr.getErrorStream()); + return OutErr.create(outErr.getOutputStream(), wrappedErr); + } + + private String getNotInRealWorkspaceError(Path doNotBuildFile) { + String message = "Blaze should not be called from a Blaze output directory. "; + try { + String realWorkspace = + new String(FileSystemUtils.readContentAsLatin1(doNotBuildFile)); + message += String.format("The pertinent workspace directory is: '%s'", + realWorkspace); + } catch (IOException e) { + // We are exiting anyway. + } + + return message; + } + + /** + * For a given output_base directory, returns the command log file path. + */ + public static Path getCommandLogPath(Path outputBase) { + return outputBase.getRelative("command.log"); + } + + private OutErr tee(OutErr outErr1, OutErr outErr2) { + DelegatingOutErr outErr = new DelegatingOutErr(); + outErr.addSink(outErr1); + outErr.addSink(outErr2); + return outErr; + } + + private void closeSilently(OutputStream logOutputStream) { + if (logOutputStream != null) { + try { + logOutputStream.close(); + } catch (IOException e) { + LoggingUtil.logToRemote(Level.WARNING, "Unable to close command.log", e); + } + } + } + + /** + * Creates an option parser using the common options classes and the + * command-specific options classes. + * + * <p>An overriding method should first call this method and can then + * override default values directly or by calling {@link + * #parseOptionsForCommand} for command-specific options. + * + * @throws OptionsParsingException + */ + protected OptionsParser createOptionsParser(BlazeCommand command) + throws OptionsParsingException { + Command annotation = command.getClass().getAnnotation(Command.class); + List<Class<? extends OptionsBase>> allOptions = Lists.newArrayList(); + allOptions.addAll(BlazeCommandUtils.getOptions( + command.getClass(), getRuntime().getBlazeModules(), getRuntime().getRuleClassProvider())); + OptionsParser parser = OptionsParser.newOptionsParser(allOptions); + parser.setAllowResidue(annotation.allowResidue()); + return parser; + } + + /** + * Convert a list of option override specifications to a more easily digestible + * form. + * + * @param overrides list of option override specifications + */ + @VisibleForTesting + static List<Pair<String, ListMultimap<String, String>>> getOptionsMap( + OutErr outErr, + List<String> rcFiles, + List<CommonCommandOptions.OptionOverride> overrides, + Set<String> validCommands) { + List<Pair<String, ListMultimap<String, String>>> result = new ArrayList<>(); + + String lastRcFile = null; + ListMultimap<String, String> lastMap = null; + for (CommonCommandOptions.OptionOverride override : overrides) { + if (override.blazeRc < 0 || override.blazeRc >= rcFiles.size()) { + outErr.printErrLn("WARNING: inconsistency in generated command line " + + "args. Ignoring bogus argument\n"); + continue; + } + String rcFile = rcFiles.get(override.blazeRc); + + String command = override.command; + int index = command.indexOf(':'); + if (index > 0) { + command = command.substring(0, index); + } + if (!validCommands.contains(command) && !command.equals("common")) { + outErr.printErrLn("WARNING: while reading option defaults file '" + + rcFile + "':\n" + + " invalid command name '" + override.command + "'."); + continue; + } + + if (!rcFile.equals(lastRcFile)) { + if (lastRcFile != null) { + result.add(Pair.of(lastRcFile, lastMap)); + } + lastRcFile = rcFile; + lastMap = ArrayListMultimap.create(); + } + lastMap.put(override.command, override.option); + } + if (lastRcFile != null) { + result.add(Pair.of(lastRcFile, lastMap)); + } + + return result; + } + + /** + * Returns the event handler to use for this Blaze command. + */ + private EventHandler createEventHandler(OutErr outErr, + BlazeCommandEventHandler.Options eventOptions) { + EventHandler eventHandler; + if ((eventOptions.useColor() || eventOptions.useCursorControl())) { + eventHandler = new FancyTerminalEventHandler(outErr, eventOptions); + } else { + eventHandler = new BlazeCommandEventHandler(outErr, eventOptions); + } + + return RateLimitingEventHandler.create(eventHandler, eventOptions.showProgressRateLimit); + } + + /** + * Unsets the event handler. + */ + private void releaseHandler(EventHandler eventHandler) { + if (eventHandler instanceof FancyTerminalEventHandler) { + // Make sure that the terminal state of the old event handler is clear + // before creating a new one. + ((FancyTerminalEventHandler)eventHandler).resetTerminal(); + } + } + + /** + * Returns the runtime instance shared by the commands that this dispatcher + * dispatches to. + */ + public BlazeRuntime getRuntime() { + return runtime; + } + + /** + * The map from command names to commands that this dispatcher dispatches to. + */ + Map<String, BlazeCommand> getCommandsByName() { + return Collections.unmodifiableMap(commandsByName); + } + + /** + * Shuts down all the registered commands to give them a chance to cleanup or + * close resources. Should be called by the owner of this command dispatcher + * in all termination cases. + */ + public void shutdown() { + closeSilently(logOutputStream); + logOutputStream = null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java new file mode 100644 index 0000000000..603b0bef46 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandEventHandler.java @@ -0,0 +1,246 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.events.Location; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.util.EnumSet; +import java.util.Set; + +/** + * BlazeCommandEventHandler: an event handler established for the duration of a + * single Blaze command. + */ +public class BlazeCommandEventHandler implements EventHandler { + + public enum UseColor { YES, NO, AUTO } + public enum UseCurses { YES, NO, AUTO } + + public static class UseColorConverter extends EnumConverter<UseColor> { + public UseColorConverter() { + super(UseColor.class, "--color setting"); + } + } + + public static class UseCursesConverter extends EnumConverter<UseCurses> { + public UseCursesConverter() { + super(UseCurses.class, "--curses setting"); + } + } + + public static class Options extends OptionsBase { + + @Option(name = "show_progress", + defaultValue = "true", + category = "verbosity", + help = "Display progress messages during a build.") + public boolean showProgress; + + @Option(name = "show_task_finish", + defaultValue = "false", + category = "verbosity", + help = "Display progress messages when tasks complete, not just when they start.") + public boolean showTaskFinish; + + @Option(name = "show_progress_rate_limit", + defaultValue = "0.03", // A nice middle ground; snappy but not too spammy in logs. + category = "verbosity", + help = "Minimum number of seconds between progress messages in the output.") + public double showProgressRateLimit; + + @Option(name = "color", + defaultValue = "auto", + converter = UseColorConverter.class, + category = "verbosity", + help = "Use terminal controls to colorize output.") + public UseColor useColorEnum; + + @Option(name = "curses", + defaultValue = "auto", + converter = UseCursesConverter.class, + category = "verbosity", + help = "Use terminal cursor controls to minimize scrolling output") + public UseCurses useCursesEnum; + + @Option(name = "terminal_columns", + defaultValue = "80", + category = "hidden", + help = "A system-generated parameter which specifies the terminal " + + " width in columns.") + public int terminalColumns; + + @Option(name = "isatty", + defaultValue = "false", + category = "hidden", + help = "A system-generated parameter which is used to notify the " + + "server whether this client is running in a terminal. " + + "If this is set to false, then '--color=auto' will be treated as '--color=no'. " + + "If this is set to true, then '--color=auto' will be treated as '--color=yes'.") + public boolean isATty; + + // This lives here (as opposed to the more logical BuildRequest.Options) + // because the client passes it to the server *always*. We don't want the + // client to have to figure out when it should or shouldn't to send it. + @Option(name = "emacs", + defaultValue = "false", + category = "undocumented", + help = "A system-generated parameter which is true iff EMACS=t in the environment of " + + "the client. This option controls certain display features.") + public boolean runningInEmacs; + + @Option(name = "show_timestamps", + defaultValue = "false", + category = "verbosity", + help = "Include timestamps in messages") + public boolean showTimestamp; + + @Option(name = "progress_in_terminal_title", + defaultValue = "false", + category = "verbosity", + help = "Show the command progress in the terminal title. " + + "Useful to see what blaze is doing when having multiple terminal tabs.") + public boolean progressInTermTitle; + + + public boolean useColor() { + return useColorEnum == UseColor.YES || (useColorEnum == UseColor.AUTO && isATty); + } + + public boolean useCursorControl() { + return useCursesEnum == UseCurses.YES || (useCursesEnum == UseCurses.AUTO && isATty); + } + } + + private static final DateTimeFormatter TIMESTAMP_FORMAT = + DateTimeFormat.forPattern("(MM-dd HH:mm:ss.SSS) "); + + protected final OutErr outErr; + + private final PrintStream errPrintStream; + + protected final Set<EventKind> eventMask = + EnumSet.copyOf(EventKind.ERRORS_WARNINGS_AND_INFO_AND_OUTPUT); + + protected final boolean showTimestamp; + + public BlazeCommandEventHandler(OutErr outErr, Options eventOptions) { + this.outErr = outErr; + this.errPrintStream = new PrintStream(outErr.getErrorStream(), true); + if (eventOptions.showProgress) { + eventMask.add(EventKind.PROGRESS); + eventMask.add(EventKind.START); + } else { + // Skip PASS events if --noshow_progress is requested. + eventMask.remove(EventKind.PASS); + } + if (eventOptions.showTaskFinish) { + eventMask.add(EventKind.FINISH); + } + eventMask.add(EventKind.SUBCOMMAND); + this.showTimestamp = eventOptions.showTimestamp; + } + + /** See EventHandler.handle. */ + @Override + public void handle(Event event) { + if (!eventMask.contains(event.getKind())) { + return; + } + String prefix; + switch (event.getKind()) { + case STDOUT: + putOutput(outErr.getOutputStream(), event); + return; + case STDERR: + putOutput(outErr.getErrorStream(), event); + return; + case PASS: + case FAIL: + case TIMEOUT: + case ERROR: + case WARNING: + case DEPCHECKER: + prefix = event.getKind() + ": "; + break; + case SUBCOMMAND: + prefix = ">>>>>>>>> "; + break; + case INFO: + case PROGRESS: + case START: + case FINISH: + prefix = "____"; + break; + default: + throw new IllegalStateException("" + event.getKind()); + } + StringBuilder buf = new StringBuilder(); + buf.append(prefix); + + if (showTimestamp) { + buf.append(timestamp()); + } + + Location location = event.getLocation(); + if (location != null) { + buf.append(location.print()).append(": "); + } + + buf.append(event.getMessage()); + if (event.getKind() == EventKind.FINISH) { + buf.append(" DONE"); + } + + // Add a trailing period for ERROR and WARNING messages, which are + // typically English sentences composed from exception messages. + if (event.getKind() == EventKind.WARNING || + event.getKind() == EventKind.ERROR) { + buf.append('.'); + } + + // Event messages go to stderr; results (e.g. 'blaze query') go to stdout. + errPrintStream.println(buf); + } + + private void putOutput(OutputStream out, Event event) { + try { + out.write(event.getMessageBytes()); + out.flush(); + } catch (IOException e) { + // This can happen in server mode if the blaze client has exited, + // or if output is redirected to a file and the disk is full, etc. + // Ignore. + } + } + + /** + * @return a string representing the current time, eg "04-26 13:47:32.124". + */ + protected String timestamp() { + return TIMESTAMP_FORMAT.print(System.currentTimeMillis()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java new file mode 100644 index 0000000000..ff738db529 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandUtils.java @@ -0,0 +1,166 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.util.ResourceFileLoader; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Utility class for functionality related to Blaze commands. + */ +public class BlazeCommandUtils { + /** + * Options classes used as startup options in Blaze core. + */ + private static final List<Class<? extends OptionsBase>> DEFAULT_STARTUP_OPTIONS = + ImmutableList.<Class<? extends OptionsBase>>of( + BlazeServerStartupOptions.class, + HostJvmStartupOptions.class); + + /** + * The set of option-classes that are common to all Blaze commands. + */ + private static final Collection<Class<? extends OptionsBase>> COMMON_COMMAND_OPTIONS = + ImmutableList.of(CommonCommandOptions.class, BlazeCommandEventHandler.Options.class); + + + private BlazeCommandUtils() {} + + public static ImmutableList<Class<? extends OptionsBase>> getStartupOptions( + Iterable<BlazeModule> modules) { + Set<Class<? extends OptionsBase>> options = new HashSet<>(); + options.addAll(DEFAULT_STARTUP_OPTIONS); + for (BlazeModule blazeModule : modules) { + Iterables.addAll(options, blazeModule.getStartupOptions()); + } + + return ImmutableList.copyOf(options); + } + + /** + * Returns the set of all options (including those inherited directly and + * transitively) for this AbstractCommand's @Command annotation. + * + * <p>Why does metaprogramming always seem like such a bright idea in the + * beginning? + */ + public static ImmutableList<Class<? extends OptionsBase>> getOptions( + Class<? extends BlazeCommand> clazz, + Iterable<BlazeModule> modules, + ConfiguredRuleClassProvider ruleClassProvider) { + Command commandAnnotation = clazz.getAnnotation(Command.class); + if (commandAnnotation == null) { + throw new IllegalStateException("@Command missing for " + clazz.getName()); + } + + Set<Class<? extends OptionsBase>> options = new HashSet<>(); + options.addAll(COMMON_COMMAND_OPTIONS); + Collections.addAll(options, commandAnnotation.options()); + + if (commandAnnotation.usesConfigurationOptions()) { + options.addAll(ruleClassProvider.getConfigurationOptions()); + } + + for (BlazeModule blazeModule : modules) { + Iterables.addAll(options, blazeModule.getCommandOptions(commandAnnotation)); + } + + for (Class<? extends BlazeCommand> base : commandAnnotation.inherits()) { + options.addAll(getOptions(base, modules, ruleClassProvider)); + } + return ImmutableList.copyOf(options); + } + + /** + * Returns the expansion of the specified help topic. + * + * @param topic the name of the help topic; used in %{command} expansion. + * @param help the text template of the help message. Certain %{x} variables + * will be expanded. A prefix of "resource:" means use the .jar + * resource of that name. + * @param categoryDescriptions a mapping from option category names to + * descriptions, passed to {@link OptionsParser#describeOptions}. + * @param helpVerbosity a tri-state verbosity option selecting between just + * names, names and syntax, and full description. + */ + public static final String expandHelpTopic(String topic, String help, + Class<? extends BlazeCommand> commandClass, + Collection<Class<? extends OptionsBase>> options, + Map<String, String> categoryDescriptions, + OptionsParser.HelpVerbosity helpVerbosity) { + OptionsParser parser = OptionsParser.newOptionsParser(options); + + String template; + if (help.startsWith("resource:")) { + String resourceName = help.substring("resource:".length()); + try { + template = ResourceFileLoader.loadResource(commandClass, resourceName); + } catch (IOException e) { + throw new IllegalStateException("failed to load help resource '" + resourceName + + "' due to I/O error: " + e.getMessage(), e); + } + } else { + template = help; + } + + if (!template.contains("%{options}")) { + throw new IllegalStateException("Help template for '" + topic + "' omits %{options}!"); + } + + return template. + replace("%{command}", topic). + replace("%{options}", parser.describeOptions(categoryDescriptions, helpVerbosity)). + trim() + + "\n\n" + + (helpVerbosity == OptionsParser.HelpVerbosity.MEDIUM + ? "(Use 'help --long' for full details or --short to just enumerate options.)\n" + : ""); + } + + /** + * The help page for this command. + * + * @param categoryDescriptions a mapping from option category names to + * descriptions, passed to {@link OptionsParser#describeOptions}. + * @param verbosity a tri-state verbosity option selecting between just names, + * names and syntax, and full description. + */ + public static String getUsage( + Class<? extends BlazeCommand> commandClass, + Map<String, String> categoryDescriptions, + OptionsParser.HelpVerbosity verbosity, + Iterable<BlazeModule> blazeModules, + ConfiguredRuleClassProvider ruleClassProvider) { + Command commandAnnotation = commandClass.getAnnotation(Command.class); + return BlazeCommandUtils.expandHelpTopic( + commandAnnotation.name(), + commandAnnotation.help(), + commandClass, + BlazeCommandUtils.getOptions(commandClass, blazeModules, ruleClassProvider), + categoryDescriptions, + verbosity); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java new file mode 100644 index 0000000000..6855cbd149 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java @@ -0,0 +1,420 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.exec.OutputService; +import com.google.devtools.build.lib.packages.MakeEnvironment; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.PackageFactory.PackageArgument; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory; +import com.google.devtools.build.lib.skyframe.DiffAwareness; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.annotation.Nullable; + +/** + * A module Blaze can load at the beginning of its execution. Modules are supplied with extension + * points to augment the functionality at specific, well-defined places. + * + * <p>The constructors of individual Blaze modules should be empty. All work should be done in the + * methods (e.g. {@link #blazeStartup}). + */ +public abstract class BlazeModule { + + /** + * Returns the extra startup options this module contributes. + * + * <p>This method will be called at the beginning of Blaze startup (before #blazeStartup). + */ + public Iterable<Class<? extends OptionsBase>> getStartupOptions() { + return ImmutableList.of(); + } + + /** + * Called before {@link #getFileSystem} and {@link #blazeStartup}. + * + * <p>This method will be called at the beginning of Blaze startup. + */ + @SuppressWarnings("unused") + public void globalInit(OptionsProvider startupOptions) throws AbruptExitException { + } + + /** + * Returns the file system implementation used by Blaze. It is an error if more than one module + * returns a file system. If all return null, the default unix file system is used. + * + * <p>This method will be called at the beginning of Blaze startup (in-between #globalInit and + * #blazeStartup). + */ + @SuppressWarnings("unused") + public FileSystem getFileSystem(OptionsProvider startupOptions, PathFragment outputPath) { + return null; + } + + /** + * Called when Blaze starts up. + */ + @SuppressWarnings("unused") + public void blazeStartup(OptionsProvider startupOptions, + BlazeVersionInfo versionInfo, UUID instanceId, BlazeDirectories directories, + Clock clock) throws AbruptExitException { + } + + /** + * Returns the set of directories under which blaze may assume all files are immutable. + */ + public Set<Path> getImmutableDirectories() { + return ImmutableSet.<Path>of(); + } + + /** + * May yield a supplier that provides factories for the Preprocessor to apply. Only one of the + * configured modules may return non-null. + * + * The factory yielded by the supplier will be checked with + * {@link Preprocessor.Factory#isStillValid} at the beginning of each incremental build. This + * allows modules to have preprocessors customizable by flags. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Preprocessor.Factory.Supplier getPreprocessorFactorySupplier() { + return null; + } + + /** + * Adds the rule classes supported by this module. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + @SuppressWarnings("unused") + public void initializeRuleClasses(ConfiguredRuleClassProvider.Builder builder) { + } + + /** + * Returns the list of commands this module contributes to Blaze. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Iterable<? extends BlazeCommand> getCommands() { + return ImmutableList.of(); + } + + /** + * Returns the list of query output formatters this module provides. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Iterable<OutputFormatter> getQueryOutputFormatters() { + return ImmutableList.of(); + } + + /** + * Returns the {@link DiffAwareness} strategies this module contributes. These will be used to + * determine which files, if any, changed between Blaze commands. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + @SuppressWarnings("unused") + public Iterable<? extends DiffAwareness.Factory> getDiffAwarenessFactories(boolean watchFS) { + return ImmutableList.of(); + } + + /** + * Returns the workspace status action factory contributed by this module. + * + * <p>There should always be exactly one of these in a Blaze instance. + */ + public WorkspaceStatusAction.Factory getWorkspaceStatusActionFactory() { + return null; + } + + /** + * PlatformSet is a group of platforms characterized by a regular expression. For example, the + * entry "oldlinux": "i[34]86-libc[345]-linux" might define a set of platforms representing + * certain older linux releases. + * + * <p>Platform-set names are used in BUILD files in the third argument to <tt>vardef</tt>, to + * define per-platform tweaks to variables such as CFLAGS. + * + * <p>vardef is a legacy mechanism: it needs explicit support in the rule implementations, + * and cannot express conditional dependencies, only conditional attribute values. This + * mechanism will be supplanted by configuration dependent attributes, and its effect can + * usually also be achieved with abi_deps. + * + * <p>This method will be called during Blaze startup (after #blazeStartup). + */ + public Map<String, String> getPlatformSetRegexps() { + return ImmutableMap.<String, String>of(); + } + + /** + * Services provided for Blaze modules via BlazeRuntime. + */ + public interface ModuleEnvironment { + /** + * Gets a file from the depot based on its label and returns the {@link Path} where it can + * be found. + */ + Path getFileFromDepot(Label label) + throws NoSuchThingException, InterruptedException, IOException; + + /** + * Exits Blaze as early as possible. This is currently a hack and should only be called in + * event handlers for {@code BuildStartingEvent}, {@code GotOptionsEvent} and + * {@code LoadingPhaseCompleteEvent}. + */ + void exit(AbruptExitException exception); + } + + /** + * Called before each command. + */ + @SuppressWarnings("unused") + public void beforeCommand(BlazeRuntime blazeRuntime, Command command) + throws AbruptExitException { + } + + /** + * Returns the output service to be used. It is an error if more than one module returns an + * output service. + * + * <p>This method will be called at the beginning of each command (after #beforeCommand). + */ + @SuppressWarnings("unused") + public OutputService getOutputService() throws AbruptExitException { + return null; + } + + /** + * Returns the extra options this module contributes to a specific command. + * + * <p>This method will be called at the beginning of each command (after #beforeCommand). + */ + @SuppressWarnings("unused") + public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) { + return ImmutableList.of(); + } + + /** + * Returns a map of option categories to descriptive strings. This is used by {@code HelpCommand} + * to show a more readable list of flags. + */ + public Map<String, String> getOptionCategories() { + return ImmutableMap.of(); + } + + /** + * A item that is returned by "blaze info". + */ + public interface InfoItem { + /** + * The name of the info key. + */ + String getName(); + + /** + * The help description of the info key. + */ + String getDescription(); + + /** + * Whether the key is printed when "blaze info" is invoked without arguments. + * + * <p>This is usually true for info keys that take multiple lines, thus, cannot really be + * included in the output of argumentless "blaze info". + */ + boolean isHidden(); + + /** + * Returns the value of the info key. The return value is directly printed to stdout. + */ + byte[] get(Supplier<BuildConfiguration> configurationSupplier) throws AbruptExitException; + } + + /** + * Returns the additional information this module provides to "blaze info". + * + * <p>This method will be called at the beginning of each "blaze info" command (after + * #beforeCommand). + */ + public Iterable<InfoItem> getInfoItems() { + return ImmutableList.of(); + } + + /** + * Returns the list of query functions this module provides to "blaze query". + * + * <p>This method will be called at the beginning of each "blaze query" command (after + * #beforeCommand). + */ + public Iterable<QueryFunction> getQueryFunctions() { + return ImmutableList.of(); + } + + /** + * Returns the action context provider the module contributes to Blaze, if any. + * + * <p>This method will be called at the beginning of the execution phase, e.g. of the + * "blaze build" command. + */ + public ActionContextProvider getActionContextProvider() { + return null; + } + + /** + * Returns the action context consumer that pulls in action contexts required by this module, + * if any. + * + * <p>This method will be called at the beginning of the execution phase, e.g. of the + * "blaze build" command. + */ + public ActionContextConsumer getActionContextConsumer() { + return null; + } + + /** + * Called after each command. + */ + public void afterCommand() { + } + + /** + * Called when Blaze shuts down. + */ + public void blazeShutdown() { + } + + /** + * Action inputs are allowed to be missing for all inputs where this predicate returns true. + */ + public Predicate<PathFragment> getAllowedMissingInputs() { + return null; + } + + /** + * Optionally specializes the cache that ensures source files are looked at just once during + * a build. Only one module may do so. + */ + public ActionInputFileCache createActionInputCache(String cwd, FileSystem fs) { + return null; + } + + /** + * Returns the extensions this module contributes to the global namespace of the BUILD language. + */ + public PackageFactory.EnvironmentExtension getPackageEnvironmentExtension() { + return new PackageFactory.EnvironmentExtension() { + @Override + public void update( + Environment environment, MakeEnvironment.Builder pkgMakeEnv, Label buildFileLabel) { + } + + @Override + public Iterable<PackageArgument<?>> getPackageArguments() { + return ImmutableList.of(); + } + }; + } + + /** + * Returns a factory for creating {@link SkyframeExecutor} objects. If the module does not + * provide any SkyframeExecutorFactory, it returns null. Note that only one factory per + * Bazel/Blaze runtime is allowed. + */ + public SkyframeExecutorFactory getSkyframeExecutorFactory() { + return null; + } + + /** Returns a map of "extra" SkyFunctions for SkyValues that this module may want to build. */ + public ImmutableMap<SkyFunctionName, SkyFunction> getSkyFunctions(BlazeDirectories directories) { + return ImmutableMap.of(); + } + + /** + * Returns the extra precomputed values that the module makes available in Skyframe. + * + * <p>This method is called once per Blaze instance at the very beginning of its life. + * If it creates the injected values by using a {@code com.google.common.base.Supplier}, + * that supplier is asked for the value it contains just before the loading phase begins. This + * functionality can be used to implement precomputed values that are not constant during the + * lifetime of a Blaze instance (naturally, they must be constant over the course of a build) + * + * <p>The following things must be done in order to define a new precomputed values: + * <ul> + * <li> Create a public static final variable of type + * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed} + * <li> Set its value by adding an {@link Injected} in this method (it can be created using the + * aforementioned variable and the value or a supplier of the value) + * <li> Reference the value in Skyframe functions by calling get {@code get} method on the + * {@link com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed} variable. This + * will never return null, because its value will have been injected before most of the Skyframe + * values are computed. + * </ul> + */ + public Iterable<Injected> getPrecomputedSkyframeValues() { + return ImmutableList.of(); + } + + /** + * Optionally returns a provider for project files that can be used to bundle targets and + * command-line options. + */ + @Nullable + public ProjectFile.Provider createProjectFileProvider() { + return null; + } + + /** + * Optionally returns a factory to create coverage report actions. + */ + @Nullable + public CoverageReportActionFactory getCoverageReportFactory() { + return null; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java new file mode 100644 index 0000000000..0251e83772 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java @@ -0,0 +1,1795 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.SubscriberExceptionContext; +import com.google.common.eventbus.SubscriberExceptionHandler; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache; +import com.google.devtools.build.lib.actions.cache.NullActionCache; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.config.BinTools; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfigurationKey; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; +import com.google.devtools.build.lib.analysis.config.DefaultsPackage; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.buildtool.BuildTool; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.OutputFilter; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.OutputService; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.PackageFactory; +import com.google.devtools.build.lib.packages.Preprocessor; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadedPackageProvider; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PackageManager; +import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator; +import com.google.devtools.build.lib.profiler.MemoryProfiler; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.Profiler.ProfiledTaskKinds; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory; +import com.google.devtools.build.lib.runtime.commands.BuildCommand; +import com.google.devtools.build.lib.runtime.commands.CanonicalizeCommand; +import com.google.devtools.build.lib.runtime.commands.CleanCommand; +import com.google.devtools.build.lib.runtime.commands.HelpCommand; +import com.google.devtools.build.lib.runtime.commands.InfoCommand; +import com.google.devtools.build.lib.runtime.commands.ProfileCommand; +import com.google.devtools.build.lib.runtime.commands.QueryCommand; +import com.google.devtools.build.lib.runtime.commands.RunCommand; +import com.google.devtools.build.lib.runtime.commands.ShutdownCommand; +import com.google.devtools.build.lib.runtime.commands.SkylarkCommand; +import com.google.devtools.build.lib.runtime.commands.TestCommand; +import com.google.devtools.build.lib.runtime.commands.VersionCommand; +import com.google.devtools.build.lib.server.RPCServer; +import com.google.devtools.build.lib.server.ServerCommand; +import com.google.devtools.build.lib.server.signal.InterruptSignalHandler; +import com.google.devtools.build.lib.skyframe.DiffAwareness; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutorFactory; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.skyframe.SkyframeExecutorFactory; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.ThreadUtils; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.UnixFileSystem; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; +import com.google.devtools.common.options.TriState; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * The BlazeRuntime class encapsulates the runtime settings and services that + * are available to most parts of any Blaze application for the duration of the + * batch run or server lifetime. A single instance of this runtime will exist + * and will be passed around as needed. + */ +public final class BlazeRuntime { + /** + * The threshold for memory reserved by a 32-bit JVM before trouble may be expected. + * + * <p>After the JVM starts, it reserves memory for heap (controlled by -Xmx) and non-heap + * (code, PermGen, etc.). Furthermore, as Blaze spawns threads, each thread reserves memory + * for the stack (controlled by -Xss). Thus even if Blaze starts fine, with high memory settings + * it will die from a stack allocation failure in the middle of a build. We prefer failing + * upfront by setting a safe threshold. + * + * <p>This does not apply to 64-bit VMs. + */ + private static final long MAX_BLAZE32_RESERVED_MEMORY = 3400 * 1048576L; + + // Less than this indicates tampering with -Xmx settings. + private static final long MIN_BLAZE32_HEAP_SIZE = 3000 * 1000000L; + + public static final String DO_NOT_BUILD_FILE_NAME = "DO_NOT_BUILD_HERE"; + + private static final Pattern suppressFromLog = Pattern.compile(".*(auth|pass|cookie).*", + Pattern.CASE_INSENSITIVE); + + private static final Logger LOG = Logger.getLogger(BlazeRuntime.class.getName()); + + private final BlazeDirectories directories; + private Path workingDirectory; + private long commandStartTime; + + // Application-specified constants + private final PathFragment runfilesPrefix; + + private final SkyframeExecutor skyframeExecutor; + + private final Reporter reporter; + private EventBus eventBus; + private final LoadingPhaseRunner loadingPhaseRunner; + private final PackageFactory packageFactory; + private final ConfigurationFactory configurationFactory; + private final ConfiguredRuleClassProvider ruleClassProvider; + private final BuildView view; + private ActionCache actionCache; + private final TimestampGranularityMonitor timestampGranularityMonitor; + private final Clock clock; + private final BuildTool buildTool; + + private OutputService outputService; + + private final Iterable<BlazeModule> blazeModules; + private final BlazeModule.ModuleEnvironment blazeModuleEnvironment; + + private UUID commandId; // Unique identifier for the command being run + + private final AtomicInteger storedExitCode = new AtomicInteger(); + + private final Map<String, String> clientEnv; + + // We pass this through here to make it available to the MasterLogWriter. + private final OptionsProvider startupOptionsProvider; + + private String outputFileSystem; + private Map<String, BlazeCommand> commandMap; + + private AbruptExitException pendingException; + + private final SubscriberExceptionHandler eventBusExceptionHandler; + + private final BinTools binTools; + + private final WorkspaceStatusAction.Factory workspaceStatusActionFactory; + + private final ProjectFile.Provider projectFileProvider; + + private class BlazeModuleEnvironment implements BlazeModule.ModuleEnvironment { + @Override + public Path getFileFromDepot(Label label) + throws NoSuchThingException, InterruptedException, IOException { + Target target = getPackageManager().getTarget(reporter, label); + return (outputService != null) + ? outputService.stageTool(target) + : target.getPackage().getPackageDirectory().getRelative(target.getName()); + } + + @Override + public void exit(AbruptExitException exception) { + Preconditions.checkState(pendingException == null); + pendingException = exception; + } + } + + private BlazeRuntime(BlazeDirectories directories, Reporter reporter, + WorkspaceStatusAction.Factory workspaceStatusActionFactory, + final SkyframeExecutor skyframeExecutor, + PackageFactory pkgFactory, ConfiguredRuleClassProvider ruleClassProvider, + ConfigurationFactory configurationFactory, PathFragment runfilesPrefix, Clock clock, + OptionsProvider startupOptionsProvider, Iterable<BlazeModule> blazeModules, + Map<String, String> clientEnv, + TimestampGranularityMonitor timestampGranularityMonitor, + SubscriberExceptionHandler eventBusExceptionHandler, + BinTools binTools, ProjectFile.Provider projectFileProvider) { + this.workspaceStatusActionFactory = workspaceStatusActionFactory; + this.directories = directories; + this.workingDirectory = directories.getWorkspace(); + this.reporter = reporter; + this.runfilesPrefix = runfilesPrefix; + this.packageFactory = pkgFactory; + this.binTools = binTools; + this.projectFileProvider = projectFileProvider; + + this.skyframeExecutor = skyframeExecutor; + this.loadingPhaseRunner = new LoadingPhaseRunner( + skyframeExecutor.getPackageManager(), + pkgFactory.getRuleClassNames()); + + this.clientEnv = clientEnv; + + this.blazeModules = blazeModules; + this.ruleClassProvider = ruleClassProvider; + this.configurationFactory = configurationFactory; + this.view = new BuildView(directories, getPackageManager(), ruleClassProvider, + skyframeExecutor, binTools, getCoverageReportActionFactory(blazeModules)); + this.clock = clock; + this.timestampGranularityMonitor = Preconditions.checkNotNull(timestampGranularityMonitor); + this.startupOptionsProvider = startupOptionsProvider; + + this.eventBusExceptionHandler = eventBusExceptionHandler; + this.blazeModuleEnvironment = new BlazeModuleEnvironment(); + this.buildTool = new BuildTool(this); + initEventBus(); + + if (inWorkspace()) { + writeOutputBaseReadmeFile(); + writeOutputBaseDoNotBuildHereFile(); + } + setupExecRoot(); + } + + @Nullable private CoverageReportActionFactory getCoverageReportActionFactory( + Iterable<BlazeModule> blazeModules) { + CoverageReportActionFactory firstFactory = null; + for (BlazeModule module : blazeModules) { + CoverageReportActionFactory factory = module.getCoverageReportFactory(); + if (factory != null) { + Preconditions.checkState(firstFactory == null, + "only one Blaze Module can have a Coverage Report Factory"); + firstFactory = factory; + } + } + return firstFactory; + } + + /** + * Figures out what file system we are writing output to. Here we use + * outputBase instead of outputPath because we need a file system to create the latter. + */ + private String determineOutputFileSystem() { + if (getOutputService() != null) { + return getOutputService().getFilesSystemName(); + } + long startTime = Profiler.nanoTimeMaybe(); + String fileSystem = FileSystemUtils.getFileSystem(getOutputBase()); + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Finding output file system"); + return fileSystem; + } + + public String getOutputFileSystem() { + return outputFileSystem; + } + + @VisibleForTesting + public void initEventBus() { + setEventBus(new EventBus(eventBusExceptionHandler)); + } + + private void clearEventBus() { + // EventBus does not have an unregister() method, so this is how we release memory associated + // with handlers. + setEventBus(null); + } + + private void setEventBus(EventBus eventBus) { + this.eventBus = eventBus; + skyframeExecutor.setEventBus(eventBus); + } + + /** + * Conditionally enable profiling. + */ + private final boolean initProfiler(CommonCommandOptions options, + UUID buildID, long execStartTimeNanos) { + OutputStream out = null; + boolean recordFullProfilerData = false; + ProfiledTaskKinds profiledTasks = ProfiledTaskKinds.NONE; + + try { + if (options.profilePath != null) { + Path profilePath = getWorkspace().getRelative(options.profilePath); + + recordFullProfilerData = options.recordFullProfilerData; + out = new BufferedOutputStream(profilePath.getOutputStream(), 1024 * 1024); + getReporter().handle(Event.info("Writing profile data to '" + profilePath + "'")); + profiledTasks = ProfiledTaskKinds.ALL; + } else if (options.alwaysProfileSlowOperations) { + recordFullProfilerData = false; + out = null; + profiledTasks = ProfiledTaskKinds.SLOWEST; + } + if (profiledTasks != ProfiledTaskKinds.NONE) { + Profiler.instance().start(profiledTasks, out, + "Blaze profile for " + getOutputBase() + " at " + new Date() + + ", build ID: " + buildID, + recordFullProfilerData, clock, execStartTimeNanos); + return true; + } + } catch (IOException e) { + getReporter().handle(Event.error("Error while creating profile file: " + e.getMessage())); + } + return false; + } + + /** + * Generates a README file in the output base directory. This README file + * contains the name of the workspace directory, so that users can figure out + * which output base directory corresponds to which workspace. + */ + private void writeOutputBaseReadmeFile() { + Preconditions.checkNotNull(getWorkspace()); + Path outputBaseReadmeFile = getOutputBase().getRelative("README"); + try { + FileSystemUtils.writeIsoLatin1(outputBaseReadmeFile, "WORKSPACE: " + getWorkspace(), "", + "The first line of this file is intentionally easy to parse for various", + "interactive scripting and debugging purposes. But please DO NOT write programs", + "that exploit it, as they will be broken by design: it is not possible to", + "reverse engineer the set of source trees or the --package_path from the output", + "tree, and if you attempt it, you will fail, creating subtle and", + "hard-to-diagnose bugs, that will no doubt get blamed on changes made by the", + "Blaze team.", "", "This directory was generated by Blaze.", + "Do not attempt to modify or delete any files in this directory.", + "Among other issues, Blaze's file system caching assumes that", + "only Blaze will modify this directory and the files in it,", + "so if you change anything here you may mess up Blaze's cache."); + } catch (IOException e) { + LOG.warning("Couldn't write to '" + outputBaseReadmeFile + "': " + e.getMessage()); + } + } + + private void writeOutputBaseDoNotBuildHereFile() { + Preconditions.checkNotNull(getWorkspace()); + Path filePath = getOutputBase().getRelative(DO_NOT_BUILD_FILE_NAME); + try { + FileSystemUtils.writeContent(filePath, ISO_8859_1, getWorkspace().toString()); + } catch (IOException e) { + LOG.warning("Couldn't write to '" + filePath + "': " + e.getMessage()); + } + } + + /** + * Creates the execRoot dir under outputBase. + */ + private void setupExecRoot() { + try { + FileSystemUtils.createDirectoryAndParents(directories.getExecRoot()); + } catch (IOException e) { + LOG.warning("failed to create execution root '" + directories.getExecRoot() + "': " + + e.getMessage()); + } + } + + public void recordCommandStartTime(long commandStartTime) { + this.commandStartTime = commandStartTime; + } + + public long getCommandStartTime() { + return commandStartTime; + } + + public String getWorkspaceName() { + Path workspace = directories.getWorkspace(); + if (workspace == null) { + return ""; + } + return workspace.getBaseName(); + } + + /** + * Returns any prefix to be inserted between relative source paths and the runfiles directory. + */ + public PathFragment getRunfilesPrefix() { + return runfilesPrefix; + } + + /** + * Returns the Blaze directories object for this runtime. + */ + public BlazeDirectories getDirectories() { + return directories; + } + + /** + * Returns the working directory of the server. + * + * <p>This is often the first entry on the {@code --package_path}, but not always. + * Callers should certainly not make this assumption. The Path returned may be null. + * + * @see #getWorkingDirectory() + */ + public Path getWorkspace() { + return directories.getWorkspace(); + } + + /** + * Returns the working directory of the {@code blaze} client process. + * + * <p>This may be equal to {@code getWorkspace()}, or beneath it. + * + * @see #getWorkspace() + */ + public Path getWorkingDirectory() { + return workingDirectory; + } + + /** + * Returns if the client passed a valid workspace to be used for the build. + */ + public boolean inWorkspace() { + return directories.inWorkspace(); + } + + /** + * Returns the output base directory associated with this Blaze server + * process. This is the base directory for shared Blaze state as well as tool + * and strategy specific subdirectories. + */ + public Path getOutputBase() { + return directories.getOutputBase(); + } + + /** + * Returns the output path associated with this Blaze server process.. + */ + public Path getOutputPath() { + return directories.getOutputPath(); + } + + /** + * The directory in which blaze stores the server state - that is, the socket + * file and a log. + */ + public Path getServerDirectory() { + return getOutputBase().getChild("server"); + } + + /** + * Returns the execution root directory associated with this Blaze server + * process. This is where all input and output files visible to the actual + * build reside. + */ + public Path getExecRoot() { + return directories.getExecRoot(); + } + + /** + * Returns the reporter for events. + */ + public Reporter getReporter() { + return reporter; + } + + /** + * Returns the current event bus. Only valid within the scope of a single Blaze command. + */ + public EventBus getEventBus() { + return eventBus; + } + + public BinTools getBinTools() { + return binTools; + } + + /** + * Returns the skyframe executor. + */ + public SkyframeExecutor getSkyframeExecutor() { + return skyframeExecutor; + } + + /** + * Returns the package factory. + */ + public PackageFactory getPackageFactory() { + return packageFactory; + } + + /** + * Returns the build tool. + */ + public BuildTool getBuildTool() { + return buildTool; + } + + public ImmutableList<OutputFormatter> getQueryOutputFormatters() { + ImmutableList.Builder<OutputFormatter> result = ImmutableList.builder(); + result.addAll(OutputFormatter.getDefaultFormatters()); + for (BlazeModule module : blazeModules) { + result.addAll(module.getQueryOutputFormatters()); + } + + return result.build(); + } + + /** + * Returns the package manager. + */ + public PackageManager getPackageManager() { + return skyframeExecutor.getPackageManager(); + } + + public WorkspaceStatusAction.Factory getworkspaceStatusActionFactory() { + return workspaceStatusActionFactory; + } + + public BlazeModule.ModuleEnvironment getBlazeModuleEnvironment() { + return blazeModuleEnvironment; + } + + /** + * Returns the rule class provider. + */ + public ConfiguredRuleClassProvider getRuleClassProvider() { + return ruleClassProvider; + } + + public LoadingPhaseRunner getLoadingPhaseRunner() { + return loadingPhaseRunner; + } + + /** + * Returns the build view. + */ + public BuildView getView() { + return view; + } + + public Iterable<BlazeModule> getBlazeModules() { + return blazeModules; + } + + @SuppressWarnings("unchecked") + public <T extends BlazeModule> T getBlazeModule(Class<T> moduleClass) { + for (BlazeModule module : blazeModules) { + if (module.getClass() == moduleClass) { + return (T) module; + } + } + + return null; + } + + public ConfigurationFactory getConfigurationFactory() { + return configurationFactory; + } + + /** + * Returns the target pattern parser. + */ + public TargetPatternEvaluator getTargetPatternEvaluator() { + return loadingPhaseRunner.getTargetPatternEvaluator(); + } + + /** + * Returns reference to the lazily instantiated persistent action cache + * instance. Note, that method may recreate instance between different build + * requests, so return value should not be cached. + */ + public ActionCache getPersistentActionCache() throws IOException { + if (actionCache == null) { + if (OS.getCurrent() == OS.WINDOWS) { + // TODO(bazel-team): Add support for a persistent action cache on Windows. + actionCache = new NullActionCache(); + return actionCache; + } + long startTime = Profiler.nanoTimeMaybe(); + try { + actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock); + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to load action cache: " + e.getMessage(), e); + LoggingUtil.logToRemote(Level.WARNING, "Failed to load action cache: " + + e.getMessage(), e); + getReporter().handle( + Event.error("Error during action cache initialization: " + e.getMessage() + + ". Corrupted files were renamed to '" + getCacheDirectory() + "/*.bad'. " + + "Blaze will now reset action cache data, causing a full rebuild")); + actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock); + } finally { + Profiler.instance().logSimpleTask(startTime, ProfilerTask.INFO, "Loading action cache"); + } + } + return actionCache; + } + + /** + * Removes in-memory caches. + */ + public void clearCaches() throws IOException { + clearSkyframeRelevantCaches(); + actionCache = null; + FileSystemUtils.deleteTree(getCacheDirectory()); + } + + /** Removes skyframe cache and other caches that must be kept synchronized with skyframe. */ + private void clearSkyframeRelevantCaches() { + skyframeExecutor.resetEvaluator(); + view.clear(); + } + + /** + * Returns the TimestampGranularityMonitor. The same monitor object is used + * across multiple Blaze commands, but it doesn't hold any persistent state + * across different commands. + */ + public TimestampGranularityMonitor getTimestampGranularityMonitor() { + return timestampGranularityMonitor; + } + + /** + * Returns path to the cache directory. Path must be inside output base to + * ensure that users can run concurrent instances of blaze in different + * clients without attempting to concurrently write to the same action cache + * on disk, which might not be safe. + */ + private Path getCacheDirectory() { + return getOutputBase().getChild("action_cache"); + } + + /** + * Returns a provider for project file objects. Can be null if no such provider was set by any of + * the modules. + */ + @Nullable + public ProjectFile.Provider getProjectFileProvider() { + return projectFileProvider; + } + + /** + * Hook method called by the BlazeCommandDispatcher prior to the dispatch of + * each command. + * + * @param options The CommonCommandOptions used by every command. + * @throws AbruptExitException if this command is unsuitable to be run as specified + */ + void beforeCommand(String commandName, OptionsParser optionsParser, + CommonCommandOptions options, long execStartTimeNanos) + throws AbruptExitException { + commandStartTime -= options.startupTime; + + eventBus.post(new GotOptionsEvent(startupOptionsProvider, + optionsParser)); + throwPendingException(); + + outputService = null; + BlazeModule outputModule = null; + for (BlazeModule module : blazeModules) { + OutputService moduleService = module.getOutputService(); + if (moduleService != null) { + if (outputService != null) { + throw new IllegalStateException(String.format( + "More than one module (%s and %s) returns an output service", + module.getClass(), outputModule.getClass())); + } + outputService = moduleService; + outputModule = module; + } + } + + skyframeExecutor.setBatchStatter(outputService == null + ? null + : outputService.getBatchStatter()); + + outputFileSystem = determineOutputFileSystem(); + + // Ensure that the working directory will be under the workspace directory. + Path workspace = getWorkspace(); + if (inWorkspace()) { + workingDirectory = workspace.getRelative(options.clientCwd); + } else { + workspace = FileSystemUtils.getWorkingDirectory(directories.getFileSystem()); + workingDirectory = workspace; + } + updateClientEnv(options.clientEnv, options.ignoreClientEnv); + loadingPhaseRunner.updatePatternEvaluator(workingDirectory.relativeTo(workspace)); + + // Fail fast in the case where a Blaze command forgets to install the package path correctly. + skyframeExecutor.setActive(false); + // Let skyframe figure out if it needs to store graph edges for this build. + skyframeExecutor.decideKeepIncrementalState( + startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).batch, + optionsParser.getOptions(BuildView.Options.class)); + + // Conditionally enable profiling + // We need to compensate for launchTimeNanos (measurements taken outside of the jvm). + long startupTimeNanos = options.startupTime * 1000000L; + if (initProfiler(options, this.getCommandId(), execStartTimeNanos - startupTimeNanos)) { + Profiler profiler = Profiler.instance(); + + // Instead of logEvent() we're calling the low level function to pass the timings we took in + // the launcher. We're setting the INIT phase marker so that it follows immediately the LAUNCH + // phase. + profiler.logSimpleTaskDuration(execStartTimeNanos - startupTimeNanos, 0, ProfilerTask.PHASE, + ProfilePhase.LAUNCH.description); + profiler.logSimpleTaskDuration(execStartTimeNanos, 0, ProfilerTask.PHASE, + ProfilePhase.INIT.description); + } + + if (options.memoryProfilePath != null) { + Path memoryProfilePath = getWorkingDirectory().getRelative(options.memoryProfilePath); + try { + MemoryProfiler.instance().start(memoryProfilePath.getOutputStream()); + } catch (IOException e) { + getReporter().handle( + Event.error("Error while creating memory profile file: " + e.getMessage())); + } + } + + eventBus.post(new CommandStartEvent(commandName, commandId, clientEnv, workingDirectory)); + // Initialize exit code to dummy value for afterCommand. + storedExitCode.set(ExitCode.RESERVED.getNumericExitCode()); + } + + /** + * Hook method called by the BlazeCommandDispatcher right before the dispatch + * of each command ends (while its outcome can still be modified). + */ + ExitCode precompleteCommand(ExitCode originalExit) { + eventBus.post(new CommandPrecompleteEvent(originalExit)); + // If Blaze did not suffer an infrastructure failure, check for errors in modules. + ExitCode exitCode = originalExit; + if (!originalExit.isInfrastructureFailure()) { + if (pendingException != null) { + exitCode = pendingException.getExitCode(); + } + } + pendingException = null; + return exitCode; + } + + /** + * Posts the {@link CommandCompleteEvent}, so that listeners can tidy up. Called by {@link + * #afterCommand}, and by BugReport when crashing from an exception in an async thread. + */ + public void notifyCommandComplete(int exitCode) { + if (!storedExitCode.compareAndSet(ExitCode.RESERVED.getNumericExitCode(), exitCode)) { + // This command has already been called, presumably because there is a race between the main + // thread and a worker thread that crashed. Don't try to arbitrate the dispute. If the main + // thread won the race (unlikely, but possible), this may be incorrectly logged as a success. + return; + } + eventBus.post(new CommandCompleteEvent(exitCode)); + } + + /** + * Hook method called by the BlazeCommandDispatcher after the dispatch of each + * command. + */ + @VisibleForTesting + public void afterCommand(int exitCode) { + // Remove any filters that the command might have added to the reporter. + getReporter().setOutputFilter(OutputFilter.OUTPUT_EVERYTHING); + + notifyCommandComplete(exitCode); + + for (BlazeModule module : blazeModules) { + module.afterCommand(); + } + + clearEventBus(); + + try { + Profiler.instance().stop(); + MemoryProfiler.instance().stop(); + } catch (IOException e) { + getReporter().handle(Event.error("Error while writing profile file: " + e.getMessage())); + } + } + + // Make sure we keep a strong reference to this logger, so that the + // configuration isn't lost when the gc kicks in. + private static Logger templateLogger = Logger.getLogger("com.google.devtools.build"); + + /** + * Configures "com.google.devtools.build.*" loggers to the given + * {@code level}. Note: This code relies on static state. + */ + public static void setupLogging(Level level) { + templateLogger.setLevel(level); + templateLogger.info("Log level: " + templateLogger.getLevel()); + } + + /** + * Return an unmodifiable view of the blaze client's environment when it + * invoked the most recent command. Updates from future requests will be + * accessible from this view. + */ + public Map<String, String> getClientEnv() { + return Collections.unmodifiableMap(clientEnv); + } + + @VisibleForTesting + void updateClientEnv(List<Map.Entry<String, String>> clientEnvList, boolean ignoreClientEnv) { + clientEnv.clear(); + + Collection<Map.Entry<String, String>> env = + ignoreClientEnv ? System.getenv().entrySet() : clientEnvList; + for (Map.Entry<String, String> entry : env) { + clientEnv.put(entry.getKey(), entry.getValue()); + } + } + + /** + * Returns the Clock-instance used for the entire build. Before, + * individual classes (such as Profiler) used to specify the type + * of clock (e.g. EpochClock) they wanted to use. This made it + * difficult to get Blaze working on Windows as some of the clocks + * available for Linux aren't (directly) available on Windows. + * Setting the Blaze-wide clock upon construction of BlazeRuntime + * allows injecting whatever Clock instance should be used from + * BlazeMain. + * + * @return The Blaze-wide clock + */ + public Clock getClock() { + return clock; + } + + public OptionsProvider getStartupOptionsProvider() { + return startupOptionsProvider; + } + + /** + * An array of String values useful if Blaze crashes. + * For now, just returns the size of the action cache and the build id. + */ + public String[] getCrashData() { + return new String[]{ + getFileSizeString(CompactPersistentActionCache.cacheFile(getCacheDirectory()), + "action cache"), + commandIdString(), + }; + } + + private String commandIdString() { + UUID uuid = getCommandId(); + return (uuid == null) + ? "no build id" + : uuid + " (build id)"; + } + + /** + * @return the OutputService in use, or null if none. + */ + public OutputService getOutputService() { + return outputService; + } + + private String getFileSizeString(Path path, String type) { + try { + return String.format("%d bytes (%s)", path.getFileSize(), type); + } catch (IOException e) { + return String.format("unknown file size (%s)", type); + } + } + + /** + * Returns the UUID that Blaze uses to identify everything + * logged from the current build command. + */ + public UUID getCommandId() { + return commandId; + } + + void setCommandMap(Map<String, BlazeCommand> commandMap) { + this.commandMap = ImmutableMap.copyOf(commandMap); + } + + public Map<String, BlazeCommand> getCommandMap() { + return commandMap; + } + + /** + * Sets the UUID that Blaze uses to identify everything + * logged from the current build command. + */ + @VisibleForTesting + public void setCommandId(UUID runId) { + commandId = runId; + } + + /** + * Constructs a build configuration key for the given options. + */ + public BuildConfigurationKey getBuildConfigurationKey(BuildOptions buildOptions, + ImmutableSortedSet<String> multiCpu) { + return new BuildConfigurationKey(buildOptions, directories, clientEnv, multiCpu); + } + + /** + * This method only exists for the benefit of InfoCommand, which needs to construct a {@link + * BuildConfigurationCollection} without running a full loading phase. Don't add any more clients; + * instead, we should change info so that it doesn't need the configuration. + */ + public BuildConfigurationCollection getConfigurations(OptionsProvider optionsProvider) + throws InvalidConfigurationException, InterruptedException { + BuildConfigurationKey configurationKey = getBuildConfigurationKey( + createBuildOptions(optionsProvider), ImmutableSortedSet.<String>of()); + boolean keepGoing = optionsProvider.getOptions(BuildView.Options.class).keepGoing; + LoadedPackageProvider loadedPackageProvider = + loadingPhaseRunner.loadForConfigurations(reporter, + ImmutableSet.copyOf(configurationKey.getLabelsToLoadUnconditionally().values()), + keepGoing); + if (loadedPackageProvider == null) { + throw new InvalidConfigurationException("Configuration creation failed"); + } + return skyframeExecutor.createConfigurations(keepGoing, configurationFactory, + configurationKey); + } + + /** + * Initializes the package cache using the given options, and syncs the package cache. Also + * injects a defaults package using the options for the {@link BuildConfiguration}. + * + * @see DefaultsPackage + */ + public void setupPackageCache(PackageCacheOptions packageCacheOptions, + String defaultsPackageContents) throws InterruptedException, AbruptExitException { + if (!skyframeExecutor.hasIncrementalState()) { + clearSkyframeRelevantCaches(); + } + skyframeExecutor.sync(packageCacheOptions, getWorkingDirectory(), + defaultsPackageContents, getCommandId()); + } + + public void shutdown() { + for (BlazeModule module : blazeModules) { + module.blazeShutdown(); + } + } + + /** + * Throws the exception currently queued by a Blaze module. + * + * <p>This should be called as often as is practical so that errors are reported as soon as + * possible. Ideally, we'd not need this, but the event bus swallows exceptions so we raise + * the exception this way. + */ + public void throwPendingException() throws AbruptExitException { + if (pendingException != null) { + AbruptExitException exception = pendingException; + pendingException = null; + throw exception; + } + } + + /** + * Returns the defaults package for the default settings. Should only be called by commands that + * do <i>not</i> process {@link BuildOptions}, since build options can alter the contents of the + * defaults package, which will not be reflected here. + */ + public String getDefaultsPackageContent() { + return ruleClassProvider.getDefaultsPackageContent(); + } + + /** + * Returns the defaults package for the given options taken from an optionsProvider. + */ + public String getDefaultsPackageContent(OptionsClassProvider optionsProvider) { + return ruleClassProvider.getDefaultsPackageContent(optionsProvider); + } + + /** + * Creates a BuildOptions class for the given options taken from an optionsProvider. + */ + public BuildOptions createBuildOptions(OptionsClassProvider optionsProvider) { + return ruleClassProvider.createBuildOptions(optionsProvider); + } + + /** + * An EventBus exception handler that will report the exception to a remote server, if a + * handler is registered. + */ + public static final class RemoteExceptionHandler implements SubscriberExceptionHandler { + @Override + public void handleException(Throwable exception, SubscriberExceptionContext context) { + LoggingUtil.logToRemote(Level.SEVERE, "Failure in EventBus subscriber.", exception); + } + } + + /** + * An EventBus exception handler that will call BugReport.handleCrash exiting + * the current thread. + */ + public static final class BugReportingExceptionHandler implements SubscriberExceptionHandler { + @Override + public void handleException(Throwable exception, SubscriberExceptionContext context) { + BugReport.handleCrash(exception); + } + } + + /** + * Main method for the Blaze server startup. Note: This method logs + * exceptions to remote servers. Do not add this to a unittest. + */ + public static void main(Iterable<Class<? extends BlazeModule>> moduleClasses, String[] args) { + setupUncaughtHandler(args); + List<BlazeModule> modules = createModules(moduleClasses); + if (args.length >= 1 && args[0].equals("--batch")) { + // Run Blaze in batch mode. + System.exit(batchMain(modules, args)); + } + LOG.info("Starting Blaze server with args " + Arrays.toString(args)); + try { + // Run Blaze in server mode. + System.exit(serverMain(modules, OutErr.SYSTEM_OUT_ERR, args)); + } catch (RuntimeException | Error e) { // A definite bug... + BugReport.printBug(OutErr.SYSTEM_OUT_ERR, e); + BugReport.sendBugReport(e, Arrays.asList(args)); + System.exit(ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode()); + throw e; // Shouldn't get here. + } + } + + @VisibleForTesting + public static List<BlazeModule> createModules( + Iterable<Class<? extends BlazeModule>> moduleClasses) { + ImmutableList.Builder<BlazeModule> result = ImmutableList.builder(); + for (Class<? extends BlazeModule> moduleClass : moduleClasses) { + try { + BlazeModule module = moduleClass.newInstance(); + result.add(module); + } catch (Throwable e) { + throw new IllegalStateException("Cannot instantiate module " + moduleClass.getName(), e); + } + } + + return result.build(); + } + + /** + * Generates a string form of a request to be written to the logs, + * filtering the user environment to remove anything that looks private. + * The current filter criteria removes any variable whose name includes + * "auth", "pass", or "cookie". + * + * @param requestStrings + * @return the filtered request to write to the log. + */ + @VisibleForTesting + public static String getRequestLogString(List<String> requestStrings) { + StringBuilder buf = new StringBuilder(); + buf.append('['); + String sep = ""; + for (String s : requestStrings) { + buf.append(sep); + if (s.startsWith("--client_env")) { + int varStart = "--client_env=".length(); + int varEnd = s.indexOf('=', varStart); + String varName = s.substring(varStart, varEnd); + if (suppressFromLog.matcher(varName).matches()) { + buf.append("--client_env="); + buf.append(varName); + buf.append("=__private_value_removed__"); + } else { + buf.append(s); + } + } else { + buf.append(s); + } + sep = ", "; + } + buf.append(']'); + return buf.toString(); + } + + /** + * Command line options split in to two parts: startup options and everything else. + */ + @VisibleForTesting + static class CommandLineOptions { + private final List<String> startupArgs; + private final List<String> otherArgs; + + CommandLineOptions(List<String> startupArgs, List<String> otherArgs) { + this.startupArgs = ImmutableList.copyOf(startupArgs); + this.otherArgs = ImmutableList.copyOf(otherArgs); + } + + public List<String> getStartupArgs() { + return startupArgs; + } + + public List<String> getOtherArgs() { + return otherArgs; + } + } + + /** + * Splits given arguments into two lists - arguments matching options defined in this class + * and everything else, while preserving order in each list. + */ + static CommandLineOptions splitStartupOptions( + Iterable<BlazeModule> modules, String... args) { + List<String> prefixes = new ArrayList<>(); + List<Field> startupFields = Lists.newArrayList(); + for (Class<? extends OptionsBase> defaultOptions + : BlazeCommandUtils.getStartupOptions(modules)) { + startupFields.addAll(ImmutableList.copyOf(defaultOptions.getFields())); + } + + for (Field field : startupFields) { + if (field.isAnnotationPresent(Option.class)) { + prefixes.add("--" + field.getAnnotation(Option.class).name()); + if (field.getType() == boolean.class || field.getType() == TriState.class) { + prefixes.add("--no" + field.getAnnotation(Option.class).name()); + } + } + } + + List<String> startupArgs = new ArrayList<>(); + List<String> otherArgs = Lists.newArrayList(args); + + for (Iterator<String> argi = otherArgs.iterator(); argi.hasNext(); ) { + String arg = argi.next(); + if (!arg.startsWith("--")) { + break; // stop at command - all startup options would be specified before it. + } + for (String prefix : prefixes) { + if (arg.startsWith(prefix)) { + startupArgs.add(arg); + argi.remove(); + break; + } + } + } + return new CommandLineOptions(startupArgs, otherArgs); + } + + private static void captureSigint() { + final Thread mainThread = Thread.currentThread(); + final AtomicInteger numInterrupts = new AtomicInteger(); + + final Runnable interruptWatcher = new Runnable() { + @Override + public void run() { + int count = 0; + // Not an actual infinite loop because it's run in a daemon thread. + while (true) { + count++; + Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); + LOG.warning("Slow interrupt number " + count + " in batch mode"); + ThreadUtils.warnAboutSlowInterrupt(); + } + } + }; + + new InterruptSignalHandler() { + @Override + public void run() { + LOG.info("User interrupt"); + OutErr.SYSTEM_OUT_ERR.printErrLn("Blaze received an interrupt"); + mainThread.interrupt(); + + int curNumInterrupts = numInterrupts.incrementAndGet(); + if (curNumInterrupts == 1) { + Thread interruptWatcherThread = new Thread(interruptWatcher, "interrupt-watcher"); + interruptWatcherThread.setDaemon(true); + interruptWatcherThread.start(); + } else if (curNumInterrupts == 2) { + LOG.warning("Second --batch interrupt: Reverting to JVM SIGINT handler"); + uninstall(); + } + } + }; + } + + /** + * A main method that runs blaze commands in batch mode. The return value indicates the desired + * exit status of the program. + */ + private static int batchMain(Iterable<BlazeModule> modules, String[] args) { + captureSigint(); + CommandLineOptions commandLineOptions = splitStartupOptions(modules, args); + LOG.info("Running Blaze in batch mode with startup args " + + commandLineOptions.getStartupArgs()); + + String memoryWarning = validateJvmMemorySettings(); + if (memoryWarning != null) { + OutErr.SYSTEM_OUT_ERR.printErrLn(memoryWarning); + } + + BlazeRuntime runtime; + try { + runtime = newRuntime(modules, parseOptions(modules, commandLineOptions.getStartupArgs())); + } catch (OptionsParsingException e) { + OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage()); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } catch (AbruptExitException e) { + OutErr.SYSTEM_OUT_ERR.printErr(e.getMessage()); + return e.getExitCode().getNumericExitCode(); + } + + BlazeCommandDispatcher dispatcher = + new BlazeCommandDispatcher(runtime, getBuiltinCommandList()); + + try { + LOG.info(getRequestLogString(commandLineOptions.getOtherArgs())); + return dispatcher.exec(commandLineOptions.getOtherArgs(), OutErr.SYSTEM_OUT_ERR, + runtime.getClock().currentTimeMillis()); + } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) { + return e.getExitStatus(); + } finally { + runtime.shutdown(); + dispatcher.shutdown(); + } + } + + /** + * A main method that does not send email. The return value indicates the desired exit status of + * the program. + */ + private static int serverMain(Iterable<BlazeModule> modules, OutErr outErr, String[] args) { + try { + createBlazeRPCServer(modules, Arrays.asList(args)).serve(); + return ExitCode.SUCCESS.getNumericExitCode(); + } catch (OptionsParsingException e) { + outErr.printErr(e.getMessage()); + return ExitCode.COMMAND_LINE_ERROR.getNumericExitCode(); + } catch (IOException e) { + outErr.printErr("I/O Error: " + e.getMessage()); + return ExitCode.BUILD_FAILURE.getNumericExitCode(); + } catch (AbruptExitException e) { + outErr.printErr(e.getMessage()); + return e.getExitCode().getNumericExitCode(); + } + } + + private static FileSystem fileSystemImplementation() { + // The JNI-based UnixFileSystem is faster, but on Windows it is not available. + return OS.getCurrent() == OS.WINDOWS ? new JavaIoFileSystem() : new UnixFileSystem(); + } + + /** + * Creates and returns a new Blaze RPCServer. Call {@link RPCServer#serve()} to start the server. + */ + private static RPCServer createBlazeRPCServer(Iterable<BlazeModule> modules, List<String> args) + throws IOException, OptionsParsingException, AbruptExitException { + OptionsProvider options = parseOptions(modules, args); + BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class); + + final BlazeRuntime runtime = newRuntime(modules, options); + final BlazeCommandDispatcher dispatcher = + new BlazeCommandDispatcher(runtime, getBuiltinCommandList()); + final String memoryWarning = validateJvmMemorySettings(); + + final ServerCommand blazeCommand; + + // Adaptor from RPC mechanism to BlazeCommandDispatcher: + blazeCommand = new ServerCommand() { + private boolean shutdown = false; + + @Override + public int exec(List<String> args, OutErr outErr, long firstContactTime) { + LOG.info(getRequestLogString(args)); + if (memoryWarning != null) { + outErr.printErrLn(memoryWarning); + } + + try { + return dispatcher.exec(args, outErr, firstContactTime); + } catch (BlazeCommandDispatcher.ShutdownBlazeServerException e) { + if (e.getCause() != null) { + StringWriter message = new StringWriter(); + message.write("Shutting down due to exception:\n"); + PrintWriter writer = new PrintWriter(message, true); + e.printStackTrace(writer); + writer.flush(); + LOG.severe(message.toString()); + } + shutdown = true; + runtime.shutdown(); + dispatcher.shutdown(); + return e.getExitStatus(); + } + } + + @Override + public boolean shutdown() { + return shutdown; + } + }; + + RPCServer server = RPCServer.newServerWith(runtime.getClock(), blazeCommand, + runtime.getServerDirectory(), runtime.getWorkspace(), startupOptions.maxIdleSeconds); + return server; + } + + private static Function<String, String> sourceFunctionForMap(final Map<String, String> map) { + return new Function<String, String>() { + @Override + public String apply(String input) { + if (!map.containsKey(input)) { + return "default"; + } + + if (map.get(input).isEmpty()) { + return "command line"; + } + + return map.get(input); + } + }; + } + + /** + * Parses the command line arguments into a {@link OptionsParser} object. + * + * <p>This function needs to parse the --option_sources option manually so that the real option + * parser can set the source for every option correctly. If that cannot be parsed or is missing, + * we just report an unknown source for every startup option. + */ + private static OptionsProvider parseOptions( + Iterable<BlazeModule> modules, List<String> args) throws OptionsParsingException { + Set<Class<? extends OptionsBase>> optionClasses = Sets.newHashSet(); + optionClasses.addAll(BlazeCommandUtils.getStartupOptions(modules)); + // First parse the command line so that we get the option_sources argument + OptionsParser parser = OptionsParser.newOptionsParser(optionClasses); + parser.setAllowResidue(false); + parser.parse(OptionPriority.COMMAND_LINE, null, args); + Function<? super String, String> sourceFunction = + sourceFunctionForMap(parser.getOptions(BlazeServerStartupOptions.class).optionSources); + + // Then parse the command line again, this time with the correct option sources + parser = OptionsParser.newOptionsParser(optionClasses); + parser.setAllowResidue(false); + parser.parseWithSourceFunction(OptionPriority.COMMAND_LINE, sourceFunction, args); + return parser; + } + + /** + * Creates a new blaze runtime, given the install and output base directories. + * + * <p>Note: This method can and should only be called once per startup, as it also creates the + * filesystem object that will be used for the runtime. So it should only ever be called from the + * main method of the Blaze program. + * + * @param options Blaze startup options. + * + * @return a new BlazeRuntime instance initialized with the given filesystem and directories, and + * an error string that, if not null, describes a fatal initialization failure that makes + * this runtime unsuitable for real commands + */ + private static BlazeRuntime newRuntime( + Iterable<BlazeModule> blazeModules, OptionsProvider options) throws AbruptExitException { + for (BlazeModule module : blazeModules) { + module.globalInit(options); + } + + BlazeServerStartupOptions startupOptions = options.getOptions(BlazeServerStartupOptions.class); + PathFragment workspaceDirectory = startupOptions.workspaceDirectory; + PathFragment installBase = startupOptions.installBase; + PathFragment outputBase = startupOptions.outputBase; + + OsUtils.maybeForceJNI(installBase); // Must be before first use of JNI. + + // From the point of view of the Java program --install_base and --output_base + // are mandatory options, despite the comment in their declarations. + if (installBase == null || !installBase.isAbsolute()) { // (includes "" default case) + throw new IllegalArgumentException( + "Bad --install_base option specified: '" + installBase + "'"); + } + if (outputBase != null && !outputBase.isAbsolute()) { // (includes "" default case) + throw new IllegalArgumentException( + "Bad --output_base option specified: '" + outputBase + "'"); + } + + PathFragment outputPathFragment = BlazeDirectories.outputPathFromOutputBase( + outputBase, workspaceDirectory); + FileSystem fs = null; + for (BlazeModule module : blazeModules) { + FileSystem moduleFs = module.getFileSystem(options, outputPathFragment); + if (moduleFs != null) { + Preconditions.checkState(fs == null, "more than one module returns a file system"); + fs = moduleFs; + } + } + + if (fs == null) { + fs = fileSystemImplementation(); + } + Path.setFileSystemForSerialization(fs); + + Path installBasePath = fs.getPath(installBase); + Path outputBasePath = fs.getPath(outputBase); + Path workspaceDirectoryPath = null; + if (!workspaceDirectory.equals(PathFragment.EMPTY_FRAGMENT)) { + workspaceDirectoryPath = fs.getPath(workspaceDirectory); + } + + BlazeDirectories directories = + new BlazeDirectories(installBasePath, outputBasePath, workspaceDirectoryPath); + + Clock clock = BlazeClock.instance(); + + BinTools binTools; + try { + binTools = BinTools.forProduction(directories); + } catch (IOException e) { + throw new AbruptExitException( + "Cannot enumerate embedded binaries: " + e.getMessage(), + ExitCode.LOCAL_ENVIRONMENTAL_ERROR); + } + + BlazeRuntime.Builder runtimeBuilder = new BlazeRuntime.Builder().setDirectories(directories) + .setStartupOptionsProvider(options) + .setBinTools(binTools) + .setClock(clock) + // TODO(bazel-team): Make BugReportingExceptionHandler the default. + // See bug "Make exceptions in EventBus subscribers fatal" + .setEventBusExceptionHandler( + startupOptions.fatalEventBusExceptions || !BlazeVersionInfo.instance().isReleasedBlaze() + ? new BlazeRuntime.BugReportingExceptionHandler() + : new BlazeRuntime.RemoteExceptionHandler()); + + runtimeBuilder.setRunfilesPrefix(new PathFragment(Constants.RUNFILES_PREFIX)); + for (BlazeModule blazeModule : blazeModules) { + runtimeBuilder.addBlazeModule(blazeModule); + } + + BlazeRuntime runtime = runtimeBuilder.build(); + BugReport.setRuntime(runtime); + return runtime; + } + + /** + * Returns null if JVM memory settings are considered safe, and an error string otherwise. + */ + private static String validateJvmMemorySettings() { + boolean is64BitVM = "64".equals(System.getProperty("sun.arch.data.model")); + if (is64BitVM) { + return null; + } + MemoryMXBean mem = ManagementFactory.getMemoryMXBean(); + long heapSize = mem.getHeapMemoryUsage().getMax(); + long nonHeapSize = mem.getNonHeapMemoryUsage().getMax(); + if (heapSize == -1 || nonHeapSize == -1) { + return null; + } + + if (heapSize + nonHeapSize > MAX_BLAZE32_RESERVED_MEMORY) { + return String.format( + "WARNING: JVM reserved %d MB of virtual memory (above threshold of %d MB). " + + "This may result in OOMs at runtime. Use lower values of MaxPermSize " + + "or switch to blaze64.", + (heapSize + nonHeapSize) >> 20, MAX_BLAZE32_RESERVED_MEMORY >> 20); + } else if (heapSize < MIN_BLAZE32_HEAP_SIZE) { + return String.format( + "WARNING: JVM heap size is %d MB. You probably have a custom -Xmx setting in your " + + "local Blaze configuration. This may result in OOMs. Removing overrides of -Xmx " + + "settings is advised.", + heapSize >> 20); + } else { + return null; + } + } + + /** + * Make sure async threads cannot be orphaned. This method makes sure bugs are reported to + * telemetry and the proper exit code is reported. + */ + private static void setupUncaughtHandler(final String[] args) { + Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread thread, Throwable throwable) { + BugReport.handleCrash(throwable, args); + } + }); + } + + + /** + * Returns an immutable list containing new instances of each Blaze command. + */ + @VisibleForTesting + public static List<BlazeCommand> getBuiltinCommandList() { + return ImmutableList.of( + new BuildCommand(), + new CanonicalizeCommand(), + new CleanCommand(), + new HelpCommand(), + new SkylarkCommand(), + new InfoCommand(), + new ProfileCommand(), + new QueryCommand(), + new RunCommand(), + new ShutdownCommand(), + new TestCommand(), + new VersionCommand()); + } + + /** + * A builder for {@link BlazeRuntime} objects. The only required fields are the {@link + * BlazeDirectories}, and the {@link RuleClassProvider} (except for testing). All other fields + * have safe default values. + * + * <p>If a {@link ConfigurationFactory} is set, then the builder ignores the host system flag. + * <p>The default behavior of the BlazeRuntime's EventBus is to exit when a subscriber throws + * an exception. Please plan appropriately. + */ + public static class Builder { + + private PathFragment runfilesPrefix = PathFragment.EMPTY_FRAGMENT; + private BlazeDirectories directories; + private Reporter reporter; + private ConfigurationFactory configurationFactory; + private Clock clock; + private OptionsProvider startupOptionsProvider; + private final List<BlazeModule> blazeModules = Lists.newArrayList(); + private SubscriberExceptionHandler eventBusExceptionHandler = + new RemoteExceptionHandler(); + private BinTools binTools; + private UUID instanceId; + + public BlazeRuntime build() throws AbruptExitException { + Preconditions.checkNotNull(directories); + Preconditions.checkNotNull(startupOptionsProvider); + Reporter reporter = (this.reporter == null) ? new Reporter() : this.reporter; + + Clock clock = (this.clock == null) ? BlazeClock.instance() : this.clock; + UUID instanceId = (this.instanceId == null) ? UUID.randomUUID() : this.instanceId; + + Preconditions.checkNotNull(clock); + Map<String, String> clientEnv = new HashMap<>(); + TimestampGranularityMonitor timestampMonitor = new TimestampGranularityMonitor(clock); + + Preprocessor.Factory.Supplier preprocessorFactorySupplier = null; + SkyframeExecutorFactory skyframeExecutorFactory = null; + for (BlazeModule module : blazeModules) { + module.blazeStartup(startupOptionsProvider, + BlazeVersionInfo.instance(), instanceId, directories, clock); + Preprocessor.Factory.Supplier modulePreprocessorFactorySupplier = + module.getPreprocessorFactorySupplier(); + if (modulePreprocessorFactorySupplier != null) { + Preconditions.checkState(preprocessorFactorySupplier == null, + "more than one module defines a preprocessor factory supplier"); + preprocessorFactorySupplier = modulePreprocessorFactorySupplier; + } + SkyframeExecutorFactory skyFactory = module.getSkyframeExecutorFactory(); + if (skyFactory != null) { + Preconditions.checkState(skyframeExecutorFactory == null, + "At most one skyframe factory supported. But found two: %s and %s", skyFactory, + skyframeExecutorFactory); + skyframeExecutorFactory = skyFactory; + } + } + if (skyframeExecutorFactory == null) { + skyframeExecutorFactory = new SequencedSkyframeExecutorFactory(); + } + if (preprocessorFactorySupplier == null) { + preprocessorFactorySupplier = Preprocessor.Factory.Supplier.NullSupplier.INSTANCE; + } + + ConfiguredRuleClassProvider.Builder ruleClassBuilder = + new ConfiguredRuleClassProvider.Builder(); + for (BlazeModule module : blazeModules) { + module.initializeRuleClasses(ruleClassBuilder); + } + + Map<String, String> platformRegexps = null; + { + ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>(); + for (BlazeModule module : blazeModules) { + builder.putAll(module.getPlatformSetRegexps()); + } + platformRegexps = builder.build(); + if (platformRegexps.isEmpty()) { + platformRegexps = null; // Use the default. + } + } + + Set<Path> immutableDirectories = null; + { + ImmutableSet.Builder<Path> builder = new ImmutableSet.Builder<>(); + for (BlazeModule module : blazeModules) { + builder.addAll(module.getImmutableDirectories()); + } + immutableDirectories = builder.build(); + } + + Iterable<DiffAwareness.Factory> diffAwarenessFactories = null; + { + ImmutableList.Builder<DiffAwareness.Factory> builder = new ImmutableList.Builder<>(); + boolean watchFS = startupOptionsProvider != null + && startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).watchFS; + for (BlazeModule module : blazeModules) { + builder.addAll(module.getDiffAwarenessFactories(watchFS)); + } + diffAwarenessFactories = builder.build(); + } + + // Merge filters from Blaze modules that allow some action inputs to be missing. + Predicate<PathFragment> allowedMissingInputs = null; + for (BlazeModule module : blazeModules) { + Predicate<PathFragment> modulePredicate = module.getAllowedMissingInputs(); + if (modulePredicate != null) { + Preconditions.checkArgument(allowedMissingInputs == null, + "More than one Blaze module allows missing inputs."); + allowedMissingInputs = modulePredicate; + } + } + if (allowedMissingInputs == null) { + allowedMissingInputs = Predicates.alwaysFalse(); + } + + ConfiguredRuleClassProvider ruleClassProvider = ruleClassBuilder.build(); + WorkspaceStatusAction.Factory workspaceStatusActionFactory = null; + for (BlazeModule module : blazeModules) { + WorkspaceStatusAction.Factory candidate = module.getWorkspaceStatusActionFactory(); + if (candidate != null) { + Preconditions.checkState(workspaceStatusActionFactory == null, + "more than one module defines a workspace status action factory"); + workspaceStatusActionFactory = candidate; + } + } + + List<PackageFactory.EnvironmentExtension> extensions = new ArrayList<>(); + for (BlazeModule module : blazeModules) { + extensions.add(module.getPackageEnvironmentExtension()); + } + + // We use an immutable map builder for the nice side effect that it throws if a duplicate key + // is inserted. + ImmutableMap.Builder<SkyFunctionName, SkyFunction> skyFunctions = ImmutableMap.builder(); + for (BlazeModule module : blazeModules) { + skyFunctions.putAll(module.getSkyFunctions(directories)); + } + + ImmutableList.Builder<PrecomputedValue.Injected> precomputedValues = ImmutableList.builder(); + for (BlazeModule module : blazeModules) { + precomputedValues.addAll(module.getPrecomputedSkyframeValues()); + } + + final PackageFactory pkgFactory = + new PackageFactory(ruleClassProvider, platformRegexps, extensions); + SkyframeExecutor skyframeExecutor = skyframeExecutorFactory.create(reporter, pkgFactory, + timestampMonitor, directories, workspaceStatusActionFactory, + ruleClassProvider.getBuildInfoFactories(), immutableDirectories, diffAwarenessFactories, + allowedMissingInputs, preprocessorFactorySupplier, skyFunctions.build(), + precomputedValues.build()); + + if (configurationFactory == null) { + configurationFactory = new ConfigurationFactory( + ruleClassProvider.getConfigurationCollectionFactory(), + ruleClassProvider.getConfigurationFragments()); + } + + ProjectFile.Provider projectFileProvider = null; + for (BlazeModule module : blazeModules) { + ProjectFile.Provider candidate = module.createProjectFileProvider(); + if (candidate != null) { + Preconditions.checkState(projectFileProvider == null, + "more than one module defines a project file provider"); + projectFileProvider = candidate; + } + } + + return new BlazeRuntime(directories, reporter, workspaceStatusActionFactory, skyframeExecutor, + pkgFactory, ruleClassProvider, configurationFactory, + runfilesPrefix == null ? PathFragment.EMPTY_FRAGMENT : runfilesPrefix, + clock, startupOptionsProvider, ImmutableList.copyOf(blazeModules), + clientEnv, timestampMonitor, + eventBusExceptionHandler, binTools, projectFileProvider); + } + + public Builder setRunfilesPrefix(PathFragment prefix) { + this.runfilesPrefix = prefix; + return this; + } + + public Builder setBinTools(BinTools binTools) { + this.binTools = binTools; + return this; + } + + public Builder setDirectories(BlazeDirectories directories) { + this.directories = directories; + return this; + } + + /** + * Creates and sets a new {@link BlazeDirectories} instance with the given + * parameters. + */ + public Builder setDirectories(Path installBase, Path outputBase, + Path workspace) { + this.directories = new BlazeDirectories(installBase, outputBase, workspace); + return this; + } + + public Builder setReporter(Reporter reporter) { + this.reporter = reporter; + return this; + } + + public Builder setConfigurationFactory(ConfigurationFactory configurationFactory) { + this.configurationFactory = configurationFactory; + return this; + } + + public Builder setClock(Clock clock) { + this.clock = clock; + return this; + } + + public Builder setStartupOptionsProvider(OptionsProvider startupOptionsProvider) { + this.startupOptionsProvider = startupOptionsProvider; + return this; + } + + public Builder addBlazeModule(BlazeModule blazeModule) { + blazeModules.add(blazeModule); + return this; + } + + public Builder setInstanceId(UUID id) { + instanceId = id; + return this; + } + + @VisibleForTesting + public Builder setEventBusExceptionHandler( + SubscriberExceptionHandler eventBusExceptionHandler) { + this.eventBusExceptionHandler = eventBusExceptionHandler; + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java new file mode 100644 index 0000000000..1f9bcea008 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeServerStartupOptions.java @@ -0,0 +1,225 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.util.Map; + +/** + * Options that will be evaluated by the blaze client startup code and passed + * to the blaze server upon startup. + * + * <h4>IMPORTANT</h4> These options and their defaults must be kept in sync with those in the + * source of the launcher. The latter define the actual default values; this class exists only to + * provide the help message, which displays the default values. + * + * The same relationship holds between {@link HostJvmStartupOptions} and the launcher. + */ +public class BlazeServerStartupOptions extends OptionsBase { + /** + * Converter for the <code>option_sources</code> option. Takes a string in the form of + * "option_name1:source1:option_name2:source2:.." and converts it into an option name to + * source map. + */ + public static class OptionSourcesConverter implements Converter<Map<String, String>> { + private String unescape(String input) { + return input.replace("_C", ":").replace("_U", "_"); + } + + @Override + public Map<String, String> convert(String input) { + ImmutableMap.Builder<String, String> builder = ImmutableMap.builder(); + if (input.isEmpty()) { + return builder.build(); + } + + String[] elements = input.split(":"); + for (int i = 0; i < (elements.length + 1) / 2; i++) { + String name = elements[i * 2]; + String value = ""; + if (elements.length > i * 2 + 1) { + value = elements[i * 2 + 1]; + } + builder.put(unescape(name), unescape(value)); + } + return builder.build(); + } + + @Override + public String getTypeDescription() { + return "a list of option-source pairs"; + } + } + + /* Passed from the client to the server, specifies the installation + * location. The location should be of the form: + * $OUTPUT_BASE/_blaze_${USER}/install/${MD5_OF_INSTALL_MANIFEST}. + * The server code will only accept a non-empty path; it's the + * responsibility of the client to compute a proper default if + * necessary. + */ + @Option(name = "install_base", + defaultValue = "", // NOTE: purely decorative! See class docstring. + category = "hidden", + converter = OptionsUtils.PathFragmentConverter.class, + help = "This launcher option is intended for use only by tests.") + public PathFragment installBase; + + /* Note: The help string in this option applies to the client code; not + * the server code. The server code will only accept a non-empty path; it's + * the responsibility of the client to compute a proper default if + * necessary. + */ + @Option(name = "output_base", + defaultValue = "null", // NOTE: purely decorative! See class docstring. + category = "server startup", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, specifies the output location to which all build output will be written. " + + "Otherwise, the location will be " + + "${OUTPUT_ROOT}/_blaze_${USER}/${MD5_OF_WORKSPACE_ROOT}. Note: If you specify a " + + "different option from one to the next Blaze invocation for this value, you'll likely " + + "start up a new, additional Blaze server. Blaze starts exactly one server per " + + "specified output base. Typically there is one output base per workspace--however, " + + "with this option you may have multiple output bases per workspace and thereby run " + + "multiple builds for the same client on the same machine concurrently. See " + + "'blaze help shutdown' on how to shutdown a Blaze server.") + public PathFragment outputBase; + + /* Note: This option is only used by the C++ client, never by the Java server. + * It is included here to make sure that the option is documented in the help + * output, which is auto-generated by Java code. + */ + @Option(name = "output_user_root", + defaultValue = "null", // NOTE: purely decorative! See class docstring. + category = "server startup", + converter = OptionsUtils.PathFragmentConverter.class, + help = "The user-specific directory beneath which all build outputs are written; " + + "by default, this is a function of $USER, but by specifying a constant, build outputs " + + "can be shared between collaborating users.") + public PathFragment outputUserRoot; + + @Option(name = "workspace_directory", + defaultValue = "", + category = "hidden", + converter = OptionsUtils.PathFragmentConverter.class, + help = "The root of the workspace, that is, the directory that Blaze uses as the root of the " + + "build. This flag is only to be set by the blaze client.") + public PathFragment workspaceDirectory; + + @Option(name = "max_idle_secs", + defaultValue = "" + (3 * 3600), // NOTE: purely decorative! See class docstring. + category = "server startup", + help = "The number of seconds the build server will wait idling " + + "before shutting down. Note: Blaze will ignore this option " + + "unless you are starting a new instance. See also 'blaze help " + + "shutdown'.") + public int maxIdleSeconds; + + @Option(name = "batch", + defaultValue = "false", // NOTE: purely decorative! See class docstring. + category = "server startup", + help = "If set, Blaze will be run in batch mode, instead of " + + "the standard client/server. Doing so may provide " + + "more predictable semantics with respect to signal handling and job control, " + + "Batch mode retains proper queueing semantics within the same output_base. " + + "That is, simultaneous invocations will be processed in order, without overlap. " + + "If a batch mode Blaze is run on a client with a running server, it first kills " + + "the server before processing the command." + + "Blaze will run slower in batch mode, compared to client/server mode. " + + "Among other things, the build file cache is memory-resident, so it is not " + + "preserved between sequential batch invocations. Therefore, using batch mode " + + "often makes more sense in cases where performance is less critical, " + + "such as continuous builds.") + public boolean batch; + + @Option(name = "block_for_lock", + defaultValue = "true", // NOTE: purely decorative! See class docstring. + category = "server startup", + help = "If set, Blaze will exit immediately instead of waiting for other " + + "Blaze commands holding the server lock to complete.") + public boolean noblock_for_lock; + + @Option(name = "io_nice_level", + defaultValue = "-1", // NOTE: purely decorative! + category = "server startup", + help = "Set a level from 0-7 for best-effort IO scheduling. 0 is highest priority, " + + "7 is lowest. The anticipatory scheduler may only honor up to priority 4. " + + "Negative values are ignored.") + public int ioNiceLevel; + + @Option(name = "batch_cpu_scheduling", + defaultValue = "false", // NOTE: purely decorative! + category = "server startup", + help = "Use 'batch' CPU scheduling for Blaze. This policy is useful for workloads that " + + "are non-interactive, but do not want to lower their nice value. " + + "See 'man 2 sched_setscheduler'.") + public boolean batchCpuScheduling; + + @Option(name = "blazerc", + // NOTE: purely decorative! + defaultValue = "In the current directory, then in the user's home directory, the file named " + + ".$(basename $0)rc (i.e. .bazelrc for Bazel or .blazerc for Blaze)", + category = "misc", + help = "The location of the .bazelrc/.blazerc file containing default values of " + + "Blaze command options. Use /dev/null to disable the search for a " + + "blazerc file, e.g. in release builds.") + public String blazerc; + + @Option(name = "master_blazerc", + defaultValue = "true", // NOTE: purely decorative! + category = "misc", + help = "If this option is false, the master blazerc/bazelrc next to the binary " + + "is not read.") + public boolean masterBlazerc; + + @Option(name = "skyframe", + defaultValue = "full", + category = "undocumented", + help = "Unused.") + public String unusedSkyframe; + + @Option(name = "fatal_event_bus_exceptions", + defaultValue = "false", // NOTE: purely decorative! + category = "undocumented", + help = "Whether or not to allow EventBus exceptions to be fatal. Experimental.") + public boolean fatalEventBusExceptions; + + @Option(name = "option_sources", + converter = OptionSourcesConverter.class, + defaultValue = "", + category = "hidden", + help = "") + public Map<String, String> optionSources; + + // TODO(bazel-team): In order to make it easier to have local watchers in open source Bazel, + // turn this into a non-startup option. + @Option(name = "watchfs", + defaultValue = "false", + category = "undocumented", + help = "If true, Blaze tries to use the operating system's file watch service for local " + + "changes instead of scanning every file for a change.") + public boolean watchFS; + + @Option(name = "use_webstatusserver", + defaultValue = "0", + category = "server startup", + help = "Specifies port to run web status server on (0 to disable, which is default).") + public int useWebStatusServer; +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java new file mode 100644 index 0000000000..ee1e429ac0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BugReport.java @@ -0,0 +1,141 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.io.OutErr; + +import java.io.PrintStream; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility methods for sending bug reports. + * + * <p> Note, code in this class must be extremely robust. There's nothing + * worse than a crash-handler that itself crashes! + */ +public abstract class BugReport { + + private BugReport() {} + + private static Logger LOG = Logger.getLogger(BugReport.class.getName()); + + private static BlazeVersionInfo versionInfo = BlazeVersionInfo.instance(); + + private static BlazeRuntime runtime = null; + + public static void setRuntime(BlazeRuntime newRuntime) { + Preconditions.checkNotNull(newRuntime); + Preconditions.checkState(runtime == null, "runtime already set: %s, %s", runtime, newRuntime); + runtime = newRuntime; + } + + /** + * Logs the unhandled exception with a special prefix signifying that this was a crash. + * + * @param exception the unhandled exception to display. + * @param args additional values to record in the message. + * @param values Additional string values to clarify the exception. + */ + public static void sendBugReport(Throwable exception, List<String> args, String... values) { + if (!versionInfo.isReleasedBlaze()) { + LOG.info("(Not a released binary; not logged.)"); + return; + } + + logException(exception, filterClientEnv(args), values); + } + + /** + * Print and send a bug report, and exit with the proper Blaze code. + */ + public static void handleCrash(Throwable throwable, String... args) { + BugReport.sendBugReport(throwable, Arrays.asList(args)); + BugReport.printBug(OutErr.SYSTEM_OUT_ERR, throwable); + System.err.println("Blaze crash in async thread:"); + throwable.printStackTrace(); + int exitCode = + (throwable instanceof OutOfMemoryError) ? ExitCode.OOM_ERROR.getNumericExitCode() + : ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); + if (runtime != null) { + runtime.notifyCommandComplete(exitCode); + // We don't call runtime#shutDown() here because all it does is shut down the modules, and who + // knows if they can be trusted. + } + System.exit(exitCode); + } + + private static void printThrowableTo(OutErr outErr, Throwable e) { + PrintStream err = new PrintStream(outErr.getErrorStream()); + e.printStackTrace(err); + err.flush(); + LOG.log(Level.SEVERE, "Blaze crashed", e); + } + + /** + * Print user-helpful information about the bug/crash to the output. + * + * @param outErr where to write the output + * @param e the exception thrown + */ + public static void printBug(OutErr outErr, Throwable e) { + if (e instanceof OutOfMemoryError) { + outErr.printErr(e.getMessage() + "\n\n" + + "Blaze ran out of memory and crashed.\n"); + } else { + printThrowableTo(outErr, e); + } + } + + /** + * Filters {@code args} by removing any item that starts with "--client_env", + * then returns this as an immutable list. + * + * <p>The client's environment variables may contain sensitive data, so we filter it out. + */ + private static List<String> filterClientEnv(Iterable<String> args) { + if (args == null) { + return null; + } + + ImmutableList.Builder<String> filteredArgs = ImmutableList.builder(); + for (String arg : args) { + if (arg != null && !arg.startsWith("--client_env")) { + filteredArgs.add(arg); + } + } + return filteredArgs.build(); + } + + // Log the exception. Because this method is only called in a blaze release, + // this will result in a report being sent to a remote logging service. + private static void logException(Throwable exception, List<String> args, String... values) { + // The preamble is used in the crash watcher, so don't change it + // unless you know what you're doing. + String preamble = exception instanceof OutOfMemoryError + ? "Blaze OOMError: " + : "Blaze crashed with args: "; + + LoggingUtil.logToRemote(Level.SEVERE, preamble + Joiner.on(' ').join(args), exception, + values); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java new file mode 100644 index 0000000000..5175a15c49 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildPhase.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +/** + * Represents how far into the build a given target has gone. + * Used primarily for master log status reporting and representation. + */ +public enum BuildPhase { + PARSING("parsing-failed", false), + LOADING("loading-failed", false), + ANALYSIS("analysis-failed", false), + TEST_FILTERING("test-filtered", true), + TARGET_FILTERING("target-filtered", true), + NOT_BUILT("not-built", false), + NOT_ANALYZED("not-analyzed", false), + EXECUTION("build-failed", false), + BLAZE_HALTED("blaze-halted", false), + COMPLETE("built", true); + + private final String msg; + private final boolean success; + + BuildPhase(String msg, boolean success) { + this.msg = msg; + this.success = success; + } + + public String getMessage() { + return msg; + } + + public boolean getSuccess() { + return success; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java new file mode 100644 index 0000000000..8b072c775a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BuildSummaryStatsModule.java @@ -0,0 +1,88 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; +import com.google.devtools.build.lib.buildtool.buildevent.ExecutionStartingEvent; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.util.BlazeClock; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; + +/** + * Blaze module for the build summary message that reports various stats to the user. + */ +public class BuildSummaryStatsModule extends BlazeModule { + + private static final Logger LOG = Logger.getLogger(BuildSummaryStatsModule.class.getName()); + + private SimpleCriticalPathComputer criticalPathComputer; + private EventBus eventBus; + private Reporter reporter; + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.reporter = runtime.getReporter(); + this.eventBus = runtime.getEventBus(); + eventBus.register(this); + } + + @Subscribe + public void executionPhaseStarting(ExecutionStartingEvent event) { + criticalPathComputer = new SimpleCriticalPathComputer(BlazeClock.instance()); + eventBus.register(criticalPathComputer); + } + + @Subscribe + public void buildComplete(BuildCompleteEvent event) { + try { + // We might want to make this conditional on a flag; it can sometimes be a bit of a nuisance. + List<String> items = new ArrayList<>(); + items.add(String.format("Elapsed time: %.3fs", event.getResult().getElapsedSeconds())); + + if (criticalPathComputer != null) { + Profiler.instance().startTask(ProfilerTask.CRITICAL_PATH, "Critical path"); + AggregatedCriticalPath<SimpleCriticalPathComponent> criticalPath = + criticalPathComputer.aggregate(); + items.add(criticalPath.toStringSummary()); + LOG.info(criticalPath.toString()); + LOG.info("Slowest actions:\n " + Joiner.on("\n ") + .join(criticalPathComputer.getSlowestComponents())); + // We reverse the critical path because the profiler expect events ordered by the time + // when the actions were executed while critical path computation is stored in the reverse + // way. + for (SimpleCriticalPathComponent stat : criticalPath.components().reverse()) { + Profiler.instance().logSimpleTaskDuration( + TimeUnit.MILLISECONDS.toNanos(stat.getStartTime()), + TimeUnit.MILLISECONDS.toNanos(stat.getActionWallTime()), + ProfilerTask.CRITICAL_PATH_COMPONENT, stat.getAction()); + } + Profiler.instance().completeTask(ProfilerTask.CRITICAL_PATH); + } + + reporter.handle(Event.info(Joiner.on(", ").join(items))); + } finally { + criticalPathComputer = null; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/Command.java b/src/main/java/com/google/devtools/build/lib/runtime/Command.java new file mode 100644 index 0000000000..1797cd3b46 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/Command.java @@ -0,0 +1,108 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation that lets blaze commands specify their options and their help. + * The annotations are processed by {@link BlazeCommand}. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface Command { + /** + * The name of the command, as the user would type it. + */ + String name(); + + /** + * Options processed by the command, indicated by options interfaces. + * These interfaces must contain methods annotated with {@link Option}. + */ + Class<? extends OptionsBase>[] options() default {}; + + /** + * The set of other Blaze commands that this annotation's command "inherits" + * options from. These classes must be annotated with {@link Command}. + */ + Class<? extends BlazeCommand>[] inherits() default {}; + + /** + * A short description, which appears in 'blaze help'. + */ + String shortDescription(); + + /** + * True if the configuration-specific options should be available for this command. + */ + boolean usesConfigurationOptions() default false; + + /** + * True if the command runs a build. + */ + boolean builds() default false; + + /** + * True if the command should not be shown in the output of 'blaze help'. + */ + boolean hidden() default false; + + /** + * Specifies whether this command allows a residue after the parsed options. + * For example, a command might expect a list of targets to build in the + * residue. + */ + boolean allowResidue() default false; + + /** + * Returns true if this command wants to write binary data to stdout. + * Enabling this flag will disable ANSI escape stripping for this command. + */ + boolean binaryStdOut() default false; + + /** + * Returns true if this command wants to write binary data to stderr. + * Enabling this flag will disable ANSI escape stripping for this command. + */ + boolean binaryStdErr() default false; + + /** + * The help message for this command. If the value starts with "resource:", + * the remainder is interpreted as the name of a text file resource (in the + * .jar file that provides the Command implementation class). + */ + String help(); + + /** + * Returns true iff this command may only be run from within a Blaze workspace. Broadly, this + * should be true for any command that interprets the package-path, since it's potentially + * confusing otherwise. + */ + boolean mustRunInWorkspace() default true; + + /** + * Returns true iff this command is allowed to run in the output directory, + * i.e. $OUTPUT_BASE/_blaze_$USER/$MD5/... . No command should be allowed to run here, + * but there are some legacy uses of 'blaze query'. + */ + boolean canRunInOutputDirectory() default false; + +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java new file mode 100644 index 0000000000..fb92781ad1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandCompleteEvent.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +/** + * This event is fired when the Blaze command is complete + * (clean, build, test, etc.). + */ +public class CommandCompleteEvent extends CommandEvent { + + private final int exitCode; + + /** + * @param exitCode the exit code of the blaze command + */ + public CommandCompleteEvent(int exitCode) { + this.exitCode = exitCode; + } + + /** + * @return the exit code of the blaze command + */ + public int getExitCode() { + return exitCode; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java new file mode 100644 index 0000000000..3e59dce22f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEvent.java @@ -0,0 +1,68 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.BlazeClock; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.util.Date; + +/** + * Base class for Command events that includes some resource fields. + */ +public abstract class CommandEvent { + + private final long eventTimeInNanos; + private final long eventTimeInEpochTime; + private final long gcTimeInMillis; + + protected CommandEvent() { + eventTimeInNanos = BlazeClock.nanoTime(); + eventTimeInEpochTime = new Date().getTime(); + gcTimeInMillis = collectGcTimeInMillis(); + } + + /** + * Returns time spent in garbage collection since the start of the JVM process. + */ + private static long collectGcTimeInMillis() { + long gcTime = 0; + for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcTime += gcBean.getCollectionTime(); + } + return gcTime; + } + + /** + * Get the time-stamp in ns for the event. + */ + public long getEventTimeInNanos() { + return eventTimeInNanos; + } + + /** + * Get the time-stamp as epoch-time for the event. + */ + public long getEventTimeInEpochTime() { + return eventTimeInEpochTime; + } + + /** + * Get the cumulative GC time for the event. + */ + public long getGCTimeInMillis() { + return gcTimeInMillis; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java new file mode 100644 index 0000000000..9a4408613a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandPrecompleteEvent.java @@ -0,0 +1,38 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.ExitCode; + +/** + * This message is fired right before the Blaze command completes, + * and can be used to modify the command's exit code. + */ +public class CommandPrecompleteEvent { + private final ExitCode exitCode; + + /** + * @param exitCode the exit code of the blaze command + */ + public CommandPrecompleteEvent(ExitCode exitCode) { + this.exitCode = exitCode; + } + + /** + * @return the exit code of the blaze command + */ + public ExitCode getExitCode() { + return exitCode; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java new file mode 100644 index 0000000000..32834a2d7a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandStartEvent.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.vfs.Path; + +import java.util.Map; +import java.util.UUID; + +/** + * This event is fired when the Blaze command is started (clean, build, test, + * etc.). + */ +public class CommandStartEvent extends CommandEvent { + private final String commandName; + private final UUID commandId; + private final Map<String, String> clientEnv; + private final Path workingDirectory; + + /** + * @param commandName the name of the command + */ + public CommandStartEvent(String commandName, UUID commandId, Map<String, String> clientEnv, + Path workingDirectory) { + this.commandName = commandName; + this.commandId = commandId; + this.clientEnv = clientEnv; + this.workingDirectory = workingDirectory; + } + + public String getCommandName() { + return commandName; + } + + public UUID getCommandId() { + return commandId; + } + + public Map<String, String> getClientEnv() { + return clientEnv; + } + + public Path getWorkingDirectory() { + return workingDirectory; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java new file mode 100644 index 0000000000..7054975ac2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommonCommandOptions.java @@ -0,0 +1,250 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.List; +import java.util.Map; +import java.util.logging.Level; + +/** + * Options common to all commands. + */ +public class CommonCommandOptions extends OptionsBase { + /** + * A class representing a blazerc option. blazeRc is serial number of the rc + * file this option came from, option is the name of the option and value is + * its value (or null if not specified). + */ + public static class OptionOverride { + final int blazeRc; + final String command; + final String option; + + public OptionOverride(int blazeRc, String command, String option) { + this.blazeRc = blazeRc; + this.command = command; + this.option = option; + } + + @Override + public String toString() { + return String.format("%d:%s=%s", blazeRc, command, option); + } + } + + /** + * Converter for --default_override. The format is: + * --default_override=blazerc:command=option. + */ + public static class OptionOverrideConverter implements Converter<OptionOverride> { + static final String ERROR_MESSAGE = "option overrides must be in form " + + " rcfile:command=option, where rcfile is a nonzero integer"; + + public OptionOverrideConverter() {} + + @Override + public OptionOverride convert(String input) throws OptionsParsingException { + int colonPos = input.indexOf(':'); + int assignmentPos = input.indexOf('='); + + if (colonPos < 0) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + if (assignmentPos <= colonPos + 1) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + int blazeRc; + try { + blazeRc = Integer.valueOf(input.substring(0, colonPos)); + } catch (NumberFormatException e) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + if (blazeRc < 0) { + throw new OptionsParsingException(ERROR_MESSAGE); + } + + String command = input.substring(colonPos + 1, assignmentPos); + String option = input.substring(assignmentPos + 1); + + return new OptionOverride(blazeRc, command, option); + } + + @Override + public String getTypeDescription() { + return "blazerc option override"; + } + } + + + @Option(name = "config", + defaultValue = "", + category = "misc", + allowMultiple = true, + help = "Selects additional config sections from the rc files; for every <command>, it " + + "also pulls in the options from <command>:<config> if such a section exists. " + + "Note that it is currently only possible to provide these options on the " + + "command line, not in the rc files. The config sections and flag combinations " + + "they are equivalent to are located in the tools/*.blazerc config files.") + public List<String> configs; + + @Option(name = "logging", + defaultValue = "3", // Level.INFO + category = "verbosity", + converter = Converters.LogLevelConverter.class, + help = "The logging level.") + public Level verbosity; + + @Option(name = "client_env", + defaultValue = "", + category = "hidden", + converter = Converters.AssignmentConverter.class, + allowMultiple = true, + help = "A system-generated parameter which specifies the client's environment") + public List<Map.Entry<String, String>> clientEnv; + + @Option(name = "ignore_client_env", + defaultValue = "false", + category = "hidden", + help = "If true, ignore the '--client_env' flag, and use the JVM environment instead") + public boolean ignoreClientEnv; + + @Option(name = "client_cwd", + defaultValue = "", + category = "hidden", + converter = OptionsUtils.PathFragmentConverter.class, + help = "A system-generated parameter which specifies the client's working directory") + public PathFragment clientCwd; + + @Option(name = "announce_rc", + defaultValue = "false", + category = "verbosity", + help = "Whether to announce rc options.") + public boolean announceRcOptions; + + /** + * These are the actual default overrides. + * Each value is a pair of (command name, value). + * + * For example: "--default_override=build=--cpu=piii" + */ + @Option(name = "default_override", + defaultValue = "", + allowMultiple = true, + category = "hidden", + converter = OptionOverrideConverter.class, + help = "") + public List<OptionOverride> optionsOverrides; + + /** + * This is the filename that the Blaze client parsed. + */ + @Option(name = "rc_source", + defaultValue = "", + allowMultiple = true, + category = "hidden", + help = "") + public List<String> rcSource; + + @Option(name = "always_profile_slow_operations", + defaultValue = "true", + category = "undocumented", + help = "Whether profiling slow operations is always turned on") + public boolean alwaysProfileSlowOperations; + + @Option(name = "profile", + defaultValue = "null", + category = "misc", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, profile Blaze and write data to the specified " + + "file. Use blaze analyze-profile to analyze the profile.") + public PathFragment profilePath; + + @Option(name = "record_full_profiler_data", + defaultValue = "false", + category = "undocumented", + help = "By default, Blaze profiler will record only aggregated data for fast but numerous " + + "events (such as statting the file). If this option is enabled, profiler will record " + + "each event - resulting in more precise profiling data but LARGE performance " + + "hit. Option only has effect if --profile used as well.") + public boolean recordFullProfilerData; + + @Option(name = "memory_profile", + defaultValue = "null", + category = "undocumented", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, write memory usage data to the specified " + + "file at phase ends.") + public PathFragment memoryProfilePath; + + @Option(name = "gc_watchdog", + defaultValue = "false", + category = "undocumented", + deprecationWarning = "Ignoring: this option is no longer supported", + help = "Deprecated.") + public boolean gcWatchdog; + + @Option(name = "startup_time", + defaultValue = "0", + category = "hidden", + help = "The time in ms the launcher spends before sending the request to the blaze server.") + public long startupTime; + + @Option(name = "extract_data_time", + defaultValue = "0", + category = "hidden", + help = "The time spend on extracting the new blaze version.") + public long extractDataTime; + + @Option(name = "command_wait_time", + defaultValue = "0", + category = "hidden", + help = "The time in ms a command had to wait on a busy Blaze server process.") + public long waitTime; + + @Option(name = "tool_tag", + defaultValue = "", + allowMultiple = true, + category = "misc", + help = "A tool name to attribute this Blaze invocation to.") + public List<String> toolTag; + + @Option(name = "restart_reason", + defaultValue = "no_restart", + category = "hidden", + help = "The reason for the server restart.") + public String restartReason; + + @Option(name = "binary_path", + defaultValue = "", + category = "hidden", + help = "The absolute path of the blaze binary.") + public String binaryPath; + + @Option(name = "experimental_allow_project_files", + defaultValue = "false", + category = "hidden", + help = "Enable processing of +<file> parameters.") + public boolean allowProjectFiles; +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java new file mode 100644 index 0000000000..2546492e66 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/CriticalPathComputer.java @@ -0,0 +1,231 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionCompletionEvent; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.ActionMiddlemanEvent; +import com.google.devtools.build.lib.actions.ActionStartedEvent; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.CachedActionEvent; +import com.google.devtools.build.lib.util.Clock; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * Computes the critical path in the action graph based on events published to the event bus. + * + * <p>After instantiation, this object needs to be registered on the event bus to work. + */ +@ThreadSafe +public abstract class CriticalPathComputer<C extends AbstractCriticalPathComponent<C>, + A extends AggregatedCriticalPath<C>> { + + /** Number of top actions to record. */ + static final int SLOWEST_COMPONENTS_SIZE = 30; + // outputArtifactToComponent is accessed from multiple event handlers. + protected final ConcurrentMap<Artifact, C> outputArtifactToComponent = Maps.newConcurrentMap(); + + /** Maximum critical path found. */ + private C maxCriticalPath; + private final Clock clock; + + /** + * The list of slowest individual components, ignoring the time to build dependencies. + * + * <p>This data is a useful metric when running non highly incremental builds, where multiple + * tasks could run un parallel and critical path would only record the longest path. + */ + private final PriorityQueue<C> slowestComponents = new PriorityQueue<>(SLOWEST_COMPONENTS_SIZE, + new Comparator<C>() { + @Override + public int compare(C o1, C o2) { + return Long.compare(o1.getActionWallTime(), o2.getActionWallTime()); + } + } + ); + + private final Object lock = new Object(); + + protected CriticalPathComputer(Clock clock) { + this.clock = clock; + maxCriticalPath = null; + } + + /** + * Creates a critical path component for an action. + * @param action the action for the critical path component + * @param startTimeMillis time when the action started to run + */ + protected abstract C createComponent(Action action, long startTimeMillis); + + /** + * Return the critical path stats for the current command execution. + * + * <p>This method allows us to calculate lazily the aggregate statistics of the critical path, + * avoiding the memory and cpu penalty for doing it for all the actions executed. + */ + public abstract A aggregate(); + + /** + * Record an action that has started to run. + * + * @param event information about the started action + */ + @Subscribe + public void actionStarted(ActionStartedEvent event) { + Action action = event.getAction(); + C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart())); + for (Artifact output : action.getOutputs()) { + C old = outputArtifactToComponent.put(output, component); + Preconditions.checkState(old == null, "Duplicate output artifact found. This could happen" + + " if a previous event registered the action %s. Artifact: %s", action, output); + } + } + + /** + * Record a middleman action execution. Even if middleman are almost instant, we record them + * because they depend on other actions and we need them for constructing the critical path. + * + * <p>For some rules with incorrect configuration transitions we might get notified several times + * for the same middleman. This should only happen if the actions are shared. + */ + @Subscribe + public void middlemanAction(ActionMiddlemanEvent event) { + Action action = event.getAction(); + C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart())); + boolean duplicate = false; + for (Artifact output : action.getOutputs()) { + C old = outputArtifactToComponent.putIfAbsent(output, component); + if (old != null) { + if (!Actions.canBeShared(action, old.getAction())) { + throw new IllegalStateException("Duplicate output artifact found for middleman." + + "This could happen if a previous event registered the action.\n" + + "Old action: " + old.getAction() + "\n\n" + + "New action: " + action + "\n\n" + + "Artifact: " + output + "\n"); + } + duplicate = true; + } + } + if (!duplicate) { + finalizeActionStat(action, component); + } + } + + /** + * Record an action that was not executed because it was in the (disk) cache. This is needed so + * that we can calculate correctly the dependencies tree if we have some cached actions in the + * middle of the critical path. + */ + @Subscribe + public void actionCached(CachedActionEvent event) { + Action action = event.getAction(); + C component = createComponent(action, TimeUnit.NANOSECONDS.toMillis(event.getNanoTimeStart())); + for (Artifact output : action.getOutputs()) { + outputArtifactToComponent.put(output, component); + } + finalizeActionStat(action, component); + } + + /** + * Records the elapsed time stats for the action. For each input artifact, it finds the real + * dependent artifacts and records the critical path stats. + */ + @Subscribe + public void actionComplete(ActionCompletionEvent event) { + ActionMetadata action = event.getActionMetadata(); + C component = Preconditions.checkNotNull( + outputArtifactToComponent.get(action.getPrimaryOutput())); + finalizeActionStat(action, component); + } + + /** Maximum critical path component found during the build. */ + protected C getMaxCriticalPath() { + synchronized (lock) { + return maxCriticalPath; + } + } + + /** + * The list of slowest individual components, ignoring the time to build dependencies. + */ + public ImmutableList<C> getSlowestComponents() { + ArrayList<C> list; + synchronized (lock) { + list = new ArrayList<>(slowestComponents); + Collections.sort(list, slowestComponents.comparator()); + } + return ImmutableList.copyOf(list).reverse(); + } + + private void finalizeActionStat(ActionMetadata action, C component) { + component.setFinishTimeMillis(getTime()); + for (Artifact input : action.getInputs()) { + addArtifactDependency(component, input); + } + + synchronized (lock) { + if (isBiggestCriticalPath(component)) { + maxCriticalPath = component; + } + + if (slowestComponents.size() == SLOWEST_COMPONENTS_SIZE) { + // The new component is faster than any of the slow components, avoid insertion. + if (slowestComponents.peek().getActionWallTime() >= component.getActionWallTime()) { + return; + } + // Remove the head element to make space (The fastest component in the queue). + slowestComponents.remove(); + } + slowestComponents.add(component); + } + } + + private long getTime() { + return TimeUnit.NANOSECONDS.toMillis(clock.nanoTime()); + } + + private boolean isBiggestCriticalPath(C newCriticalPath) { + synchronized (lock) { + return maxCriticalPath == null + || maxCriticalPath.getAggregatedWallTime() < newCriticalPath.getAggregatedWallTime(); + } + } + + /** + * If "input" is a generated artifact, link its critical path to the one we're building. + */ + private void addArtifactDependency(C actionStats, Artifact input) { + C depComponent = outputArtifactToComponent.get(input); + if (depComponent != null) { + actionStats.addDepInfo(depComponent); + } + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java new file mode 100644 index 0000000000..f4ef8e3f22 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/EventHandlerPreconditions.java @@ -0,0 +1,143 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.events.ExceptionListener; +import com.google.devtools.build.lib.util.LoggingUtil; + +import java.util.logging.Level; + +/** + * Reports precondition failures from within an event handler. + * Necessary because the EventBus silently ignores exceptions thrown from within a handler. + * This class logs the exceptions and creates some noise when a precondition check fails. + */ +public class EventHandlerPreconditions { + + private final ExceptionListener listener; + + /** + * Creates a new precondition helper which outputs errors to the given reporter. + */ + public EventHandlerPreconditions(ExceptionListener listener) { + this.listener = listener; + } + + /** + * Verifies that the given condition (a check on an argument) is true, + * throwing an IllegalArgumentException if not. + * + * @param condition a condition to check for truth. + * @throws IllegalArgumentException if the condition is false. + */ + @SuppressWarnings("unused") + public void checkArgument(boolean condition) { + checkArgument(condition, null); + } + + /** + * Verifies that the given condition (a check on an argument) is true, + * throwing an IllegalArgumentException with the given message if not. + * + * @param condition a condition to check for truth. + * @param message extra information to output if the condition is false. + * @throws IllegalArgumentException if the condition is false. + */ + public void checkArgument(boolean condition, String message) { + try { + Preconditions.checkArgument(condition, message); + } catch (IllegalArgumentException iae) { + String error = "Event handler argument check failed"; + LoggingUtil.logToRemote(Level.SEVERE, error, iae); + listener.error(null, error, iae); + throw iae; // Still terminate the handler. + } + } + + /** + * Verifies that the given condition (a check against the program's current state) is true, + * throwing an IllegalStateException if not. + * + * @param condition a condition to check for truth. + * @throws IllegalStateException if the condition is false. + */ + public void checkState(boolean condition) { + checkState(condition, null); + } + + /** + * Verifies that the given condition (a check against the program's current state) is true, + * throwing an IllegalStateException with the given message if not. + * + * @param condition a condition to check for truth. + * @param message extra information to output if the condition is false. + * @throws IllegalStateException if the condition is false. + */ + public void checkState(boolean condition, String message) { + try { + Preconditions.checkState(condition, message); + } catch (IllegalStateException ise) { + String error = "Event handler state check failed"; + LoggingUtil.logToRemote(Level.SEVERE, error, ise); + listener.error(null, error, ise); + throw ise; // Still terminate the handler. + } + } + + /** + * Fails with an IllegalStateException when invoked. + */ + public void fail(String message) { + String error = "Event handler failed: " + message; + IllegalStateException ise = new IllegalStateException(message); + LoggingUtil.logToRemote(Level.SEVERE, error, ise); + listener.error(null, error, ise); + throw ise; + } + + /** + * Verifies that the given argument is not null, throwing a NullPointerException if it is null. + * Returns the original argument or throws. + * + * @param object an object to test for null. + * @return the reference which was checked. + * @throws NullPointerException if the object is null. + */ + public <T> T checkNotNull(T object) { + return checkNotNull(object, null); + } + + /** + * Verifies that the given argument is not null, throwing a + * NullPointerException with the given message if it is null. + * Returns the original argument or throws. + * + * @param object an object to test for null. + * @param message extra information to output if the object is null. + * @return the reference which was checked. + * @throws NullPointerException if the object is null. + */ + public <T> T checkNotNull(T object, String message) { + try { + return Preconditions.checkNotNull(object, message); + } catch (NullPointerException npe) { + String error = "Event handler not-null check failed"; + LoggingUtil.logToRemote(Level.SEVERE, error, npe); + listener.error(null, error, npe); + throw npe; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java new file mode 100644 index 0000000000..e55ad2f244 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/FancyTerminalEventHandler.java @@ -0,0 +1,355 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Splitter; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.io.AnsiTerminal; +import com.google.devtools.build.lib.util.io.OutErr; + +import java.io.IOException; +import java.util.Iterator; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * An event handler for ANSI terminals which uses control characters to + * provide eye-candy, reduce scrolling, and generally improve usability + * for users running directly from the shell. + * + * <p/> + * This event handler differs from a normal terminal because it only adds + * control characters to stderr, not stdout. All blaze status feedback + * is sent to stderr, so adding control characters just to that stream gives + * the benefits described above without modifying the normal output stream. + * For commands like build that don't generate stdout output this doesn't + * matter, but for commands like query and ide_build_info, inserting these + * control characters in stdout invalidated their output. + * + * <p/> + * The underlying streams may be either line-bufferred or unbuffered. + * Normally each event will write out a sequence of output to a single + * stream, and will end with a newline, which ensures a flush. + * But care is required when outputting incomplete lines, or when mixing + * output between the two different streams (stdout and stderr): + * it may be necessary to explicitly flush the output in those cases. + * However, we also don't want to flush too often; that can lead to + * a choppy UI experience. + */ +public class FancyTerminalEventHandler extends BlazeCommandEventHandler { + private static Logger LOG = Logger.getLogger(FancyTerminalEventHandler.class.getName()); + private static final Pattern progressPattern = Pattern.compile( + // Match strings that look like they start with progress info: + // [42%] Compiling base/base.cc + // [1,442 / 23,476] Compiling base/base.cc + "^\\[(?:(?:\\d\\d?\\d?%)|(?:[\\d+,]+ / [\\d,]+))\\] "); + private static final Splitter LINEBREAK_SPLITTER = Splitter.on('\n'); + + private final AnsiTerminal terminal; + + private final boolean useColor; + private final boolean useCursorControls; + private final boolean progressInTermTitle; + public final int terminalWidth; + + private boolean terminalClosed = false; + private boolean previousLineErasable = false; + private int numLinesPreviousErasable = 0; + + public FancyTerminalEventHandler(OutErr outErr, BlazeCommandEventHandler.Options options) { + super(outErr, options); + this.terminal = new AnsiTerminal(outErr.getErrorStream()); + this.terminalWidth = (options.terminalColumns > 0 ? options.terminalColumns : 80); + useColor = options.useColor(); + useCursorControls = options.useCursorControl(); + progressInTermTitle = options.progressInTermTitle; + } + + @Override + public void handle(Event event) { + if (terminalClosed) { + return; + } + if (!eventMask.contains(event.getKind())) { + return; + } + + try { + boolean previousLineErased = false; + if (previousLineErasable) { + previousLineErased = maybeOverwritePreviousMessage(); + } + switch (event.getKind()) { + case PROGRESS: + case START: + { + String message = event.getMessage(); + Pair<String,String> progressPair = matchProgress(message); + if (progressPair != null) { + progress(progressPair.getFirst(), progressPair.getSecond()); + } else { + progress("INFO: ", message); + } + break; + } + case FINISH: + { + String message = event.getMessage(); + Pair<String,String> progressPair = matchProgress(message); + if (progressPair != null) { + String percentage = progressPair.getFirst(); + String rest = progressPair.getSecond(); + progress(percentage, rest + " DONE"); + } else { + progress("INFO: ", message + " DONE"); + } + break; + } + case PASS: + progress("PASS: ", event.getMessage()); + break; + case INFO: + info(event); + break; + case ERROR: + case FAIL: + case TIMEOUT: + // For errors, scroll the message, so it appears above the status + // line, and highlight the word "ERROR" or "FAIL" in boldface red. + errorOrFail(event); + break; + case WARNING: + // For warnings, highlight the word "Warning" in boldface magenta, + // and scroll it. + warning(event); + break; + case SUBCOMMAND: + subcmd(event); + break; + case STDOUT: + if (previousLineErased) { + terminal.flush(); + } + previousLineErasable = false; + super.handle(event); + // We don't need to flush stdout here, because + // super.handle(event) will take care of that. + break; + case STDERR: + putOutput(event); + break; + default: + // Ignore all other event types. + break; + } + } catch (IOException e) { + // The terminal shouldn't have IO errors, unless the shell is killed, which + // should also kill the blaze client. So this isn't something that should + // occur here; it will show up in the client/server interface as a broken + // pipe. + LOG.warning("Terminal was closed during build: " + e); + terminalClosed = true; + } + } + + /** + * Displays a progress message that may be erased by subsequent messages. + * + * @param prefix a short string such as "[99%] " or "INFO: ", which will be highlighted + * @param rest the remainder of the message; may be multiple lines + */ + private void progress(String prefix, String rest) throws IOException { + previousLineErasable = true; + + if (progressInTermTitle) { + int newlinePos = rest.indexOf('\n'); + if (newlinePos == -1) { + terminal.setTitle(prefix + rest); + } else { + terminal.setTitle(prefix + rest.substring(0, newlinePos)); + } + } + + if (useColor) { + terminal.textGreen(); + } + int prefixWidth = prefix.length(); + terminal.writeString(prefix); + terminal.resetTerminal(); + if (showTimestamp) { + String timestamp = timestamp(); + prefixWidth += timestamp.length(); + terminal.writeString(timestamp); + } + int numLines = 0; + Iterator<String> lines = LINEBREAK_SPLITTER.split(rest).iterator(); + String firstLine = lines.next(); + terminal.writeString(firstLine); + // Subtract one, because when the line length is the same as the terminal + // width, the terminal doesn't line-advance, so we don't want to erase + // two lines. + numLines += (prefixWidth + firstLine.length() - 1) / terminalWidth + 1; + crlf(); + while (lines.hasNext()) { + String line = lines.next(); + terminal.writeString(line); + crlf(); + numLines += (line.length() - 1) / terminalWidth + 1; + } + numLinesPreviousErasable = numLines; + } + + /** + * Try to match a message against the "progress message" pattern. If it + * matches, return the progress percentage, and the rest of the message. + * @param message the message to match + * @return a pair containing the progress percentage, and the rest of the + * progress message, or null if the message isn't a progress message. + */ + private Pair<String,String> matchProgress(String message) { + Matcher m = progressPattern.matcher(message); + if (m.find()) { + return Pair.of(message.substring(0, m.end()), message.substring(m.end())); + } else { + return null; + } + } + + /** + * Send the terminal controls that will put the cursor on the beginning + * of the same line if cursor control is on, or the next line if not. + * @returns True if it did any output; if so, caller is responsible for + * flushing the terminal if needed. + */ + private boolean maybeOverwritePreviousMessage() throws IOException { + if (useCursorControls && numLinesPreviousErasable != 0) { + for (int i = 0; i < numLinesPreviousErasable; i++) { + terminal.cr(); + terminal.cursorUp(1); + terminal.clearLine(); + } + return true; + } else { + return false; + } + } + + private void errorOrFail(Event event) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textRed(); + terminal.textBold(); + } + terminal.writeString(event.getKind().toString() + ": "); + if (useColor) { + terminal.resetTerminal(); + } + writeTimestampAndLocation(event); + terminal.writeString(event.getMessage()); + terminal.writeString("."); + crlf(); + } + + private void warning(Event warning) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textMagenta(); + } + terminal.writeString("WARNING: "); + terminal.resetTerminal(); + writeTimestampAndLocation(warning); + terminal.writeString(warning.getMessage()); + terminal.writeString("."); + crlf(); + } + + private void info(Event event) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textGreen(); + } + terminal.writeString(event.getKind().toString() + ": "); + terminal.resetTerminal(); + writeTimestampAndLocation(event); + terminal.writeString(event.getMessage()); + // No period; info messages often end in '...'. + crlf(); + } + + private void subcmd(Event subcmd) throws IOException { + previousLineErasable = false; + if (useColor) { + terminal.textBlue(); + } + terminal.writeString(">>>>> "); + terminal.resetTerminal(); + writeTimestampAndLocation(subcmd); + terminal.writeString(subcmd.getMessage()); + crlf(); + } + + /* Handle STDERR events. */ + private void putOutput(Event event) throws IOException { + previousLineErasable = false; + terminal.writeBytes(event.getMessageBytes()); +/* + * The following code doesn't work because buildtool.TerminalTestNotifier + * writes ANSI-formatted text via this mechanism, one character at a time, + * and if we try to insert additional ANSI sequences in between the characters + * of another ANSI escape sequence, we screw things up. (?) + * TODO(bazel-team): (2009) fix this. TerminalTestNotifier should go via the Reporter + * rather than via an AnsiTerminalWriter. + */ +// terminal.resetTerminal(); +// writeTimestampAndLocation(event); +// if (useColor) { +// terminal.textNormal(); +// } +// terminal.writeBytes(event.getMessageBytes()); +// terminal.resetTerminal(); + } + + /** + * Add a carriage return, shifting to the next line on the terminal, while + * guaranteeing that the terminal control codes don't cause any strange + * effects. Without the CR before the "\n", the "\n" can cause a line-break + * moving text to the next line, where the new message will be generated. + * Emitting a "CR" before means that the actual terminal controls generated + * here are CR+CR+LF; the double-CR resets the terminal line state, which + * prevents the potentially ugly formatting issue. + */ + private void crlf() throws IOException { + terminal.cr(); + terminal.writeString("\n"); + } + + private void writeTimestampAndLocation(Event event) throws IOException { + if (showTimestamp) { + terminal.writeString(timestamp()); + } + if (event.getLocation() != null) { + terminal.writeString(event.getLocation() + ": "); + } + } + + public void resetTerminal() { + try { + terminal.resetTerminal(); + } catch (IOException e) { + LOG.warning("IO Error writing to user terminal: " + e); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java new file mode 100644 index 0000000000..48e366d6b1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/GCStatsRecorder.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import java.lang.management.GarbageCollectorMXBean; +import java.util.ArrayList; +import java.util.List; + +/** + * Record GC stats for a build. + */ +public class GCStatsRecorder { + + private final Iterable<GarbageCollectorMXBean> mxBeans; + private final ImmutableMap<String, GCStat> initialData; + + public GCStatsRecorder(Iterable<GarbageCollectorMXBean> mxBeans) { + this.mxBeans = mxBeans; + ImmutableMap.Builder<String, GCStat> initialData = ImmutableMap.builder(); + for (GarbageCollectorMXBean mxBean : mxBeans) { + String name = mxBean.getName(); + initialData.put(name, new GCStat(name, mxBean.getCollectionCount(), + mxBean.getCollectionTime())); + } + this.initialData = initialData.build(); + } + + public Iterable<GCStat> getCurrentGcStats() { + List<GCStat> stats = new ArrayList<>(); + for (GarbageCollectorMXBean mxBean : mxBeans) { + String name = mxBean.getName(); + GCStat initStat = Preconditions.checkNotNull(initialData.get(name)); + stats.add(new GCStat(name, + mxBean.getCollectionCount() - initStat.getNumCollections(), + mxBean.getCollectionTime() - initStat.getTotalTimeInMs())); + } + return stats; + } + + /** Represents the garbage collections statistics for one collector (For example CMS). */ + public static class GCStat { + + private final String name; + private final long numCollections; + private final long totalTimeInMs; + + public GCStat(String name, long numCollections, long totalTimeInMs) { + this.name = name; + this.numCollections = numCollections; + this.totalTimeInMs = totalTimeInMs; + } + + /** Name of the Collector. For example CMS. */ + public String getName() { return name; } + + /** Number of invocations for a build. */ + public long getNumCollections() { return numCollections; } + + /** + * Total time spend in GC for the collector. Note that the time does need to be exclusive (aka a + * stop-the-world GC). + */ + public long getTotalTimeInMs() { return totalTimeInMs; } + + @Override + public String toString() { + return "GC time for '" + name + "' collector: " + numCollections + + " collections using " + totalTimeInMs + "ms"; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java new file mode 100644 index 0000000000..622d112235 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/GotOptionsEvent.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.common.options.OptionsProvider; + +/** + * An event in which the command line options + * are discovered. + */ +public class GotOptionsEvent { + + private final OptionsProvider startupOptions; + private final OptionsProvider options; + + /** + * Construct the options event. + * + * @param startupOptions the parsed startup options + * @param options the parsed options + */ + public GotOptionsEvent(OptionsProvider startupOptions, OptionsProvider options) { + this.startupOptions = startupOptions; + this.options = options; + } + + /** + * @return the parsed startup options + */ + public OptionsProvider getStartupOptions() { + return startupOptions; + } + + /** + * @return the parsed options. + */ + public OptionsProvider getOptions() { + return options; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java new file mode 100644 index 0000000000..305c048727 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/HostJvmStartupOptions.java @@ -0,0 +1,54 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +/** + * Options that will be evaluated by the blaze client startup code only. + * + * The only reason we have this interface is that we'd like to print a nice + * help page for the client startup options. These options do not affect the + * server's behavior in any way. + */ +public class HostJvmStartupOptions extends OptionsBase { + + @Option(name = "host_jvm_args", + defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions. + category = "host jvm startup", + help = "Flags to pass to the JVM executing Blaze. Note: Blaze " + + "will ignore this option unless you are starting a new " + + "instance. See also 'blaze help shutdown'.") + public String hostJvmArgs; + + @Option(name = "host_jvm_profile", + defaultValue = "", // NOTE: purely decorative! See BlazeServerStartupOptions. + category = "host jvm startup", + help = "Run the JVM executing Blaze in the given profiler. " + + "Blaze will search for hardcoded paths based on the " + + "profiler. Note: Blaze will ignore this option unless you " + + "are starting a new instance. See also 'blaze help shutdown'.") + public String hostJvmProfile; + + @Option(name = "host_jvm_debug", + defaultValue = "false", // NOTE: purely decorative! See BlazeServerStartupOptions. + category = "host jvm startup", + help = "Run the JVM executing Blaze so that it listens for a " + + "connection from a JDWP-compliant debugger. Note: Blaze " + + "will ignore this option unless you are starting a new " + + "instance. See also 'blaze help shutdown'.") + public boolean hostJvmDebug; +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java new file mode 100644 index 0000000000..56747d8e7a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/ProjectFile.java @@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * A file that describes a project - for large source trees that are worked on by multiple + * independent teams, it is useful to have a larger unit than a package which combines a set of + * target patterns and a set of corresponding options. + */ +public interface ProjectFile { + + /** + * A provider for a project file - we generally expect the provider to cache parsed files + * internally and return a cached version if it can ascertain that that is still correct. + * + * <p>Note in particular that packages may be moved between different package path entries, which + * should lead to cache invalidation. + */ + public interface Provider { + /** + * Returns an (optionally cached) project file instance. If there is no such file, or if the + * file cannot be parsed, then it throws an exception. + */ + ProjectFile getProjectFile(List<Path> packagePath, PathFragment path) + throws AbruptExitException; + } + + /** + * A string name of the project file that is reported to the user. It should be in such a format + * that passing it back in on the command line works. + */ + String getName(); + + /** + * A list of strings that are parsed into the options for the command. + * + * @param command An action from the command line, e.g. "build" or "test". + * @throws UnsupportedOperationException if an unknown command is passed. + */ + List<String> getCommandLineFor(String command); +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java new file mode 100644 index 0000000000..5e90f2ecf2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/RateLimitingEventHandler.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.BlazeClock; +import com.google.devtools.build.lib.util.Clock; + +/** + * An event handler that rate limits events. + */ +public class RateLimitingEventHandler implements EventHandler { + + private final EventHandler outputHandler; + private final double intervalMillis; + private final Clock clock; + private long lastEventMillis = -1; + + /** + * Creates a new Event handler that rate limits the events of type PROGRESS + * to one per event "rateLimitation" seconds. Events that arrive too quickly are dropped; + * all others are are forwarded to the handler "delegateTo". + * + * @param delegateTo The event handler that ultimately handles the events + * @param rateLimitation The minimum number of seconds between events that will be forwarded + * to the delegateTo-handler. + * If less than zero (or NaN), all events will be forwarded. + */ + public static EventHandler create(EventHandler delegateTo, double rateLimitation) { + if (rateLimitation < 0.0 || Double.isNaN(rateLimitation)) { + return delegateTo; + } + return new RateLimitingEventHandler(delegateTo, rateLimitation); + } + + private RateLimitingEventHandler(EventHandler delegateTo, double rateLimitation) { + clock = BlazeClock.instance(); + outputHandler = delegateTo; + this.intervalMillis = rateLimitation * 1000; + } + + @Override + public void handle(Event event) { + switch (event.getKind()) { + case PROGRESS: + case START: + case FINISH: + long currentTime = clock.currentTimeMillis(); + if (lastEventMillis + intervalMillis <= currentTime) { + lastEventMillis = currentTime; + outputHandler.handle(event); + } + break; + default: + outputHandler.handle(event); + break; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java new file mode 100644 index 0000000000..b8d5d45c83 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComponent.java @@ -0,0 +1,26 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.actions.Action; + +/** + * This class records the critical path for the graph of actions executed. + */ +public class SimpleCriticalPathComponent + extends AbstractCriticalPathComponent<SimpleCriticalPathComponent> { + + public SimpleCriticalPathComponent(Action action, long startTime) { super(action, startTime); } +} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java new file mode 100644 index 0000000000..65a9c95247 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/SimpleCriticalPathComputer.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.util.Clock; + +/** + * Computes the critical path during a build. + */ +public class SimpleCriticalPathComputer + extends CriticalPathComputer<SimpleCriticalPathComponent, + AggregatedCriticalPath<SimpleCriticalPathComponent>> { + + public SimpleCriticalPathComputer(Clock clock) { + super(clock); + } + + @Override + public SimpleCriticalPathComponent createComponent(Action action, long startTimeMillis) { + return new SimpleCriticalPathComponent(action, startTimeMillis); + } + + /** + * Return the critical path stats for the current command execution. + * + * <p>This method allow us to calculate lazily the aggregate statistics of the critical path, + * avoiding the memory and cpu penalty for doing it for all the actions executed. + */ + @Override + public AggregatedCriticalPath<SimpleCriticalPathComponent> aggregate() { + ImmutableList.Builder<SimpleCriticalPathComponent> components = ImmutableList.builder(); + SimpleCriticalPathComponent maxCriticalPath = getMaxCriticalPath(); + if (maxCriticalPath == null) { + return new AggregatedCriticalPath<>(0, components.build()); + } + SimpleCriticalPathComponent child = maxCriticalPath; + while (child != null) { + components.add(child); + child = child.getChild(); + } + return new AggregatedCriticalPath<>(maxCriticalPath.getAggregatedWallTime(), + components.build()); + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java new file mode 100644 index 0000000000..0134f55f62 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TerminalTestResultNotifier.java @@ -0,0 +1,220 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.rules.test.TestLogHelper; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestSummaryFormat; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Prints the test results to a terminal. + */ +public class TerminalTestResultNotifier implements TestResultNotifier { + private static class TestResultStats { + int numberOfTargets; + int passCount; + int failedToBuildCount; + int failedCount; + int failedRemotelyCount; + int failedLocallyCount; + int noStatusCount; + int numberOfExecutedTargets; + boolean wasUnreportedWrongSize; + } + + /** + * Flags specific to test summary reporting. + */ + public static class TestSummaryOptions extends OptionsBase { + @Option(name = "verbose_test_summary", + defaultValue = "true", + category = "verbosity", + help = "If true, print additional information (timing, number of failed runs, etc) in the" + + " test summary.") + public boolean verboseSummary; + + @Option(name = "test_verbose_timeout_warnings", + defaultValue = "false", + category = "verbosity", + help = "If true, print additional warnings when the actual test execution time does not " + + "match the timeout defined by the test (whether implied or explicit).") + public boolean testVerboseTimeoutWarnings; + } + + private final AnsiTerminalPrinter printer; + private final OptionsProvider options; + private final TestSummaryOptions summaryOptions; + + /** + * @param printer The terminal to print to + */ + public TerminalTestResultNotifier(AnsiTerminalPrinter printer, OptionsProvider options) { + this.printer = printer; + this.options = options; + this.summaryOptions = options.getOptions(TestSummaryOptions.class); + } + + /** + * Prints a test result summary that contains only failed tests. + */ + private void printDetailedTestResultSummary(Set<TestSummary> summaries) { + for (TestSummary entry : summaries) { + if (entry.getStatus() != BlazeTestStatus.PASSED) { + TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, true); + } + } + } + + /** + * Prints a full test result summary. + */ + private void printShortSummary(Set<TestSummary> summaries, boolean showPassingTests) { + for (TestSummary entry : summaries) { + if (entry.getStatus() != BlazeTestStatus.PASSED || showPassingTests) { + TestSummaryPrinter.print(entry, printer, summaryOptions.verboseSummary, false); + } + } + } + + /** + * Returns true iff the --check_tests_up_to_date option is enabled. + */ + private boolean optionCheckTestsUpToDate() { + return options.getOptions(ExecutionOptions.class).testCheckUpToDate; + } + + + /** + * Prints a test summary information for all tests to the terminal. + * + * @param summaries Summary of all targets that were ran + * @param numberOfExecutedTargets the number of targets that were actually ran + */ + @Override + public void notify(Set<TestSummary> summaries, int numberOfExecutedTargets) { + TestResultStats stats = new TestResultStats(); + stats.numberOfTargets = summaries.size(); + stats.numberOfExecutedTargets = numberOfExecutedTargets; + + TestOutputFormat testOutput = options.getOptions(ExecutionOptions.class).testOutput; + + for (TestSummary summary : summaries) { + if (summary.isLocalActionCached() + && TestLogHelper.shouldOutputTestLog(testOutput, + TestResult.isBlazeTestStatusPassed(summary.getStatus()))) { + TestSummaryPrinter.printCachedOutput(summary, testOutput, printer); + } + } + + for (TestSummary summary : summaries) { + if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) { + stats.passCount++; + } else if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) { + stats.failedToBuildCount++; + } else if (summary.ranRemotely()) { + stats.failedRemotelyCount++; + } else { + stats.failedLocallyCount++; + } + + if (summary.getStatus() == BlazeTestStatus.NO_STATUS) { + stats.noStatusCount++; + } + + if (summary.wasUnreportedWrongSize()) { + stats.wasUnreportedWrongSize = true; + } + } + + stats.failedCount = summaries.size() - stats.passCount; + + TestSummaryFormat testSummaryFormat = options.getOptions(ExecutionOptions.class).testSummary; + switch (testSummaryFormat) { + case DETAILED: + printDetailedTestResultSummary(summaries); + break; + + case SHORT: + printShortSummary(summaries, /*printSuccess=*/true); + break; + + case TERSE: + printShortSummary(summaries, /*printSuccess=*/false); + break; + + case NONE: + break; + } + + printStats(stats); + } + + private void addToErrorList(List<String> list, String failureDescription, int count) { + if (count > 0) { + list.add(String.format("%s%d %s %s%s", + AnsiTerminalPrinter.Mode.ERROR, + count, + count == 1 ? "fails" : "fail", + failureDescription, + AnsiTerminalPrinter.Mode.DEFAULT)); + } + } + + private void printStats(TestResultStats stats) { + if (!optionCheckTestsUpToDate()) { + List<String> results = new ArrayList<>(); + if (stats.passCount == 1) { + results.add(stats.passCount + " test passes"); + } else if (stats.passCount > 0) { + results.add(stats.passCount + " tests pass"); + } + addToErrorList(results, "to build", stats.failedToBuildCount); + addToErrorList(results, "locally", stats.failedLocallyCount); + addToErrorList(results, "remotely", stats.failedRemotelyCount); + printer.print(String.format("\nExecuted %d out of %d tests: %s.\n", + stats.numberOfExecutedTargets, + stats.numberOfTargets, + StringUtil.joinEnglishList(results, "and"))); + } else { + int failingUpToDateCount = stats.failedCount - stats.noStatusCount; + printer.print(String.format( + "\nFinished with %d passing and %s%d failing%s tests up to date, %s%d out of date.%s\n", + stats.passCount, + failingUpToDateCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "", + failingUpToDateCount, + AnsiTerminalPrinter.Mode.DEFAULT, + stats.noStatusCount > 0 ? AnsiTerminalPrinter.Mode.ERROR : "", + stats.noStatusCount, + AnsiTerminalPrinter.Mode.DEFAULT)); + } + + if (stats.wasUnreportedWrongSize) { + printer.print("There were tests whose specified size is too big. Use the " + + "--test_verbose_timeout_warnings command line option to see which " + + "ones these are.\n"); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java new file mode 100644 index 0000000000..ed9120b788 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultAnalyzer.java @@ -0,0 +1,349 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.google.common.eventbus.EventBus; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.packages.TestSize; +import com.google.devtools.build.lib.packages.TestTimeout; +import com.google.devtools.build.lib.rules.test.TestProvider; +import com.google.devtools.build.lib.rules.test.TestResult; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Prints results to the terminal, showing the results of each test target. + */ +@ThreadCompatible +public class TestResultAnalyzer { + private final Path execRoot; + private final TestSummaryOptions summaryOptions; + private final ExecutionOptions executionOptions; + private final EventBus eventBus; + + /** + * @param summaryOptions Parsed test summarization options. + * @param executionOptions Parsed build/test execution options. + * @param eventBus For reporting failed to build and cached tests. + */ + public TestResultAnalyzer(Path execRoot, + TestSummaryOptions summaryOptions, + ExecutionOptions executionOptions, + EventBus eventBus) { + this.execRoot = execRoot; + this.summaryOptions = summaryOptions; + this.executionOptions = executionOptions; + this.eventBus = eventBus; + } + + /** + * Prints out the results of the given tests, and returns true if they all passed. + * Posts any targets which weren't already completed by the listener to the EventBus. + * Reports all targets on the console via the given notifier. + * Run at the end of the build, run only once. + * + * @param testTargets The list of targets being run + * @param listener An aggregating listener with intermediate results + * @param notifier A console notifier to echo results to. + * @return true if all the tests passed, else false + */ + public boolean differentialAnalyzeAndReport( + Collection<ConfiguredTarget> testTargets, + AggregatingTestListener listener, + TestResultNotifier notifier) { + + Preconditions.checkNotNull(testTargets); + Preconditions.checkNotNull(listener); + Preconditions.checkNotNull(notifier); + + // The natural ordering of the summaries defines their output order. + Set<TestSummary> summaries = Sets.newTreeSet(); + + int totalRun = 0; // Number of targets running at least one non-cached test. + int passCount = 0; + + for (ConfiguredTarget testTarget : testTargets) { + TestSummary summary = aggregateAndReportSummary(testTarget, listener).build(); + summaries.add(summary); + + // Finished aggregating; build the final console output. + if (summary.actionRan()) { + totalRun++; + } + + if (TestResult.isBlazeTestStatusPassed(summary.getStatus())) { + passCount++; + } + } + + Preconditions.checkState(summaries.size() == testTargets.size()); + + notifier.notify(summaries, totalRun); + return passCount == testTargets.size(); + } + + private static BlazeTestStatus aggregateStatus(BlazeTestStatus status, BlazeTestStatus other) { + return status.ordinal() > other.ordinal() ? status : other; + } + + /** + * Helper for differential analysis which aggregates the TestSummary + * for an individual target, reporting runs on the EventBus if necessary. + */ + private TestSummary.Builder aggregateAndReportSummary( + ConfiguredTarget testTarget, + AggregatingTestListener listener) { + + // If already reported by the listener, no work remains for this target. + TestSummary.Builder summary = listener.getCurrentSummary(testTarget); + Label testLabel = testTarget.getLabel(); + Preconditions.checkNotNull(summary, + "%s did not complete test filtering, but has a test result", testLabel); + if (listener.targetReported(testTarget)) { + return summary; + } + + Collection<Artifact> incompleteRuns = listener.getIncompleteRuns(testTarget); + Map<Artifact, TestResult> statusMap = listener.getStatusMap(); + + // We will get back multiple TestResult instances if test had to be retried several + // times before passing. Sharding and multiple runs of the same test without retries + // will be represented by separate artifacts and will produce exactly one TestResult. + for (Artifact testStatus : TestProvider.getTestStatusArtifacts(testTarget)) { + // When a build is interrupted ( eg. a broken target with --nokeep_going ) runResult could + // be null for an unrelated test because we were not able to even try to execute the test. + // In that case, for tests that were previously passing we return null ( == NO STATUS), + // because checking if the cached test target is up-to-date would require running the + // dependency checker transitively. + TestResult runResult = statusMap.get(testStatus); + boolean isIncompleteRun = incompleteRuns.contains(testStatus); + if (runResult == null) { + summary = markIncomplete(summary); + } else if (isIncompleteRun) { + // Only process results which were not recorded by the listener. + + boolean newlyFetched = !statusMap.containsKey(testStatus); + summary = incrementalAnalyze(summary, runResult); + if (newlyFetched) { + eventBus.post(runResult); + } + Preconditions.checkState( + listener.getIncompleteRuns(testTarget).contains(testStatus) == isIncompleteRun, + "TestListener changed in differential analysis. Ensure it isn't still registered."); + } + } + + // The target was not posted by the listener and must be posted now. + eventBus.post(summary.build()); + return summary; + } + + /** + * Incrementally updates a TestSummary given an existing summary + * and a new TestResult. Only call on built targets. + * + * @param summaryBuilder Existing unbuilt test summary associated with a target. + * @param result New test result to aggregate into the summary. + * @return The updated TestSummary. + */ + public TestSummary.Builder incrementalAnalyze(TestSummary.Builder summaryBuilder, + TestResult result) { + // Cache retrieval should have been performed already. + Preconditions.checkNotNull(result); + Preconditions.checkNotNull(summaryBuilder); + TestSummary existingSummary = Preconditions.checkNotNull(summaryBuilder.peek()); + + TransitiveInfoCollection target = existingSummary.getTarget(); + Preconditions.checkNotNull( + target, "The existing TestSummary must be associated with a target"); + + BlazeTestStatus status = existingSummary.getStatus(); + int numCached = existingSummary.numCached(); + int numLocalActionCached = existingSummary.numLocalActionCached(); + + if (!existingSummary.actionRan() && !result.isCached()) { + // At least one run of the test actually ran uncached. + summaryBuilder.setActionRan(true); + + // Coverage data artifact will be identical for all test results - it is provided by the + // TestRunnerAction and all results in this collection associate with the same action. + PathFragment coverageData = result.getCoverageData(); + if (coverageData != null) { + summaryBuilder.addCoverageFiles( + Collections.singletonList(execRoot.getRelative(coverageData))); + } + } + + if (result.isCached() || result.getData().getRemotelyCached()) { + numCached++; + } + if (result.isCached()) { + numLocalActionCached++; + } + + if (!executionOptions.runsPerTestDetectsFlakes) { + status = aggregateStatus(status, result.getData().getStatus()); + } else { + int shardNumber = result.getShardNum(); + int runsPerTestForLabel = target.getProvider(TestProvider.class).getTestParams().getRuns(); + List<BlazeTestStatus> singleShardStatuses = summaryBuilder.addShardStatus( + shardNumber, result.getData().getStatus()); + if (singleShardStatuses.size() == runsPerTestForLabel) { + BlazeTestStatus shardStatus = BlazeTestStatus.NO_STATUS; + int passes = 0; + for (BlazeTestStatus runStatusForShard : singleShardStatuses) { + shardStatus = aggregateStatus(shardStatus, runStatusForShard); + if (TestResult.isBlazeTestStatusPassed(shardStatus)) { + passes++; + } + } + // Under the RunsPerTestDetectsFlakes option, return flaky if 1 <= p < n shards pass. + // If all results pass or fail, aggregate the passing/failing shardStatus. + if (passes == 0 || passes == runsPerTestForLabel) { + status = aggregateStatus(status, shardStatus); + } else { + status = aggregateStatus(status, BlazeTestStatus.FLAKY); + } + } + } + + List<String> filtered = new ArrayList<>(); + warningLoop: for (String warning : result.getData().getWarningList()) { + for (String ignoredPrefix : Constants.IGNORED_TEST_WARNING_PREFIXES) { + if (warning.startsWith(ignoredPrefix)) { + continue warningLoop; + } + } + + filtered.add(warning); + } + + List<Path> passed = new ArrayList<>(); + if (result.getData().hasPassedLog()) { + passed.add(result.getTestAction().getTestLog().getPath().getRelative( + result.getData().getPassedLog())); + } + + List<Path> failed = new ArrayList<>(); + for (String path : result.getData().getFailedLogsList()) { + failed.add(result.getTestAction().getTestLog().getPath().getRelative(path)); + } + + summaryBuilder + .addTestTimes(result.getData().getTestTimesList()) + .addPassedLogs(passed) + .addFailedLogs(failed) + .addWarnings(filtered) + .collectFailedTests(result.getData().getTestCase()) + .setRanRemotely(result.getData().getIsRemoteStrategy()); + + List<String> warnings = new ArrayList<>(); + if (status == BlazeTestStatus.PASSED) { + if (shouldEmitTestSizeWarningInSummary( + summaryOptions.testVerboseTimeoutWarnings, + warnings, result.getData().getTestProcessTimesList(), target)) { + summaryBuilder.setWasUnreportedWrongSize(true); + } + } + + return summaryBuilder + .setStatus(status) + .setNumCached(numCached) + .setNumLocalActionCached(numLocalActionCached) + .addWarnings(warnings); + } + + private TestSummary.Builder markIncomplete(TestSummary.Builder summaryBuilder) { + // TODO(bazel-team): (2010) Make NotRunTestResult support both tests failed to built and + // tests with no status and post it here. + TestSummary summary = summaryBuilder.peek(); + BlazeTestStatus status = summary.getStatus(); + if (status != BlazeTestStatus.NO_STATUS) { + status = aggregateStatus(status, BlazeTestStatus.INCOMPLETE); + } + + return summaryBuilder.setStatus(status); + } + + TestSummary.Builder markUnbuilt(TestSummary.Builder summary, boolean blazeHalted) { + BlazeTestStatus runStatus = blazeHalted ? BlazeTestStatus.BLAZE_HALTED_BEFORE_TESTING + : (executionOptions.testCheckUpToDate + ? BlazeTestStatus.NO_STATUS + : BlazeTestStatus.FAILED_TO_BUILD); + + return summary.setStatus(runStatus); + } + + /** + * Checks whether the specified test timeout could have been smaller and adds + * a warning message if verbose is true. + * + * <p>Returns true if there was a test with the wrong timeout, but if was not + * reported. + */ + private static boolean shouldEmitTestSizeWarningInSummary(boolean verbose, + List<String> warnings, List<Long> testTimes, TransitiveInfoCollection target) { + + TestTimeout specifiedTimeout = + target.getProvider(TestProvider.class).getTestParams().getTimeout(); + long maxTimeOfShard = 0; + + for (Long shardTime : testTimes) { + if (shardTime != null) { + maxTimeOfShard = Math.max(maxTimeOfShard, shardTime); + } + } + + int maxTimeInSeconds = (int) (maxTimeOfShard / 1000); + + if (!specifiedTimeout.isInRangeFuzzy(maxTimeInSeconds)) { + TestTimeout expectedTimeout = TestTimeout.getSuggestedTestTimeout(maxTimeInSeconds); + TestSize expectedSize = TestSize.getTestSize(expectedTimeout); + if (verbose) { + StringBuilder builder = new StringBuilder(String.format( + "Test execution time (%.1fs excluding execution overhead) outside of " + + "range for %s tests. Consider setting timeout=\"%s\"", + maxTimeOfShard / 1000.0, + specifiedTimeout.prettyPrint(), + expectedTimeout)); + if (expectedSize != null) { + builder.append(" or size=\"").append(expectedSize).append("\""); + } + builder.append(". You need not modify the size if you think it is correct."); + warnings.add(builder.toString()); + return false; + } + return true; + } else { + return false; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java new file mode 100644 index 0000000000..d7dbebb9c9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestResultNotifier.java @@ -0,0 +1,30 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import java.util.Set; + +/** + * Used to notify interested parties of test results. + */ +public interface TestResultNotifier { + + /** + * @param summaries Summary of all targets that were supposed to be tested + * (regardless whether they actually were executed). + * @param numberOfExecutedTargets the number of targets that were actually run. + * Must not exceed summaries.size(). + */ + void notify(Set<TestSummary> summaries, int numberOfExecutedTargets); +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java new file mode 100644 index 0000000000..171f15085f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummary.java @@ -0,0 +1,428 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Test summary entry. Stores summary information for a single test rule. + * Also used to sort summary output by status. + * + * <p>Invariant: + * All TestSummary mutations should be performed through the Builder. + * No direct TestSummary methods (except the constructor) may mutate the object. + */ +@VisibleForTesting // Ideally package-scoped. +public class TestSummary implements Comparable<TestSummary> { + /** + * Builder class responsible for creating and altering TestSummary objects. + */ + public static class Builder { + private TestSummary summary; + private boolean built; + + private Builder() { + summary = new TestSummary(); + built = false; + } + + private void mergeFrom(TestSummary existingSummary) { + // Yuck, manually fill in fields. + summary.shardRunStatuses = ArrayListMultimap.create(existingSummary.shardRunStatuses); + setTarget(existingSummary.target); + setStatus(existingSummary.status); + addCoverageFiles(existingSummary.coverageFiles); + addPassedLogs(existingSummary.passedLogs); + addFailedLogs(existingSummary.failedLogs); + + if (existingSummary.failedTestCasesStatus != null) { + addFailedTestCases(existingSummary.getFailedTestCases(), + existingSummary.getFailedTestCasesStatus()); + } + + addTestTimes(existingSummary.testTimes); + addWarnings(existingSummary.warnings); + setActionRan(existingSummary.actionRan); + setNumCached(existingSummary.numCached); + setRanRemotely(existingSummary.ranRemotely); + setWasUnreportedWrongSize(existingSummary.wasUnreportedWrongSize); + } + + // Implements copy on write logic, allowing reuse of the same builder. + private void checkMutation() { + // If mutating the builder after an object was built, create another copy. + if (built) { + built = false; + TestSummary lastSummary = summary; + summary = new TestSummary(); + mergeFrom(lastSummary); + } + } + + // This used to return a reference to the value on success. + // However, since it can alter the summary member, inlining it in an + // assignment to a property of summary was unsafe. + private void checkMutation(Object value) { + Preconditions.checkNotNull(value); + checkMutation(); + } + + public Builder setTarget(ConfiguredTarget target) { + checkMutation(target); + summary.target = target; + return this; + } + + public Builder setStatus(BlazeTestStatus status) { + checkMutation(status); + summary.status = status; + return this; + } + + public Builder addCoverageFiles(List<Path> coverageFiles) { + checkMutation(coverageFiles); + summary.coverageFiles.addAll(coverageFiles); + return this; + } + + public Builder addPassedLogs(List<Path> passedLogs) { + checkMutation(passedLogs); + summary.passedLogs.addAll(passedLogs); + return this; + } + + public Builder addFailedLogs(List<Path> failedLogs) { + checkMutation(failedLogs); + summary.failedLogs.addAll(failedLogs); + return this; + } + + public Builder collectFailedTests(TestCase testCase) { + if (testCase == null) { + summary.failedTestCasesStatus = FailedTestCasesStatus.NOT_AVAILABLE; + return this; + } + summary.failedTestCasesStatus = FailedTestCasesStatus.FULL; + return collectFailedTestCases(testCase); + } + + private Builder collectFailedTestCases(TestCase testCase) { + if (testCase.getChildCount() > 0) { + // This is a non-leaf result. Traverse its children, but do not add its + // name to the output list. It should not contain any 'failure' or + // 'error' tags, but we want to be lax here, because the syntax of the + // test.xml file is also lax. + for (TestCase child : testCase.getChildList()) { + collectFailedTestCases(child); + } + } else { + // This is a leaf result. If it passed, don't add it. + if (testCase.getStatus() == TestCase.Status.PASSED) { + return this; + } + + String name = testCase.getName(); + String className = testCase.getClassName(); + if (name == null || className == null) { + // A test case detail is not really interesting if we cannot tell which + // one it is. + this.summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL; + return this; + } + + this.summary.failedTestCases.add(testCase); + } + return this; + } + + public Builder addFailedTestCases(List<TestCase> testCases, FailedTestCasesStatus status) { + checkMutation(status); + checkMutation(testCases); + + if (summary.failedTestCasesStatus == null) { + summary.failedTestCasesStatus = status; + } else if (summary.failedTestCasesStatus != status) { + summary.failedTestCasesStatus = FailedTestCasesStatus.PARTIAL; + } + + if (testCases.isEmpty()) { + return this; + } + + // union of summary.failedTestCases, testCases + Map<String, TestCase> allCases = new TreeMap<>(); + if (summary.failedTestCases != null) { + for (TestCase detail : summary.failedTestCases) { + allCases.put(detail.getClassName() + "." + detail.getName(), detail); + } + } + for (TestCase detail : testCases) { + allCases.put(detail.getClassName() + "." + detail.getName(), detail); + } + + summary.failedTestCases = new ArrayList<TestCase>(allCases.values()); + return this; + } + + public Builder addTestTimes(List<Long> testTimes) { + checkMutation(testTimes); + summary.testTimes.addAll(testTimes); + return this; + } + + public Builder addWarnings(List<String> warnings) { + checkMutation(warnings); + summary.warnings.addAll(warnings); + return this; + } + + public Builder setActionRan(boolean actionRan) { + checkMutation(); + summary.actionRan = actionRan; + return this; + } + + public Builder setNumCached(int numCached) { + checkMutation(); + summary.numCached = numCached; + return this; + } + + public Builder setNumLocalActionCached(int numLocalActionCached) { + checkMutation(); + summary.numLocalActionCached = numLocalActionCached; + return this; + } + + public Builder setRanRemotely(boolean ranRemotely) { + checkMutation(); + summary.ranRemotely = ranRemotely; + return this; + } + + public Builder setWasUnreportedWrongSize(boolean wasUnreportedWrongSize) { + checkMutation(); + summary.wasUnreportedWrongSize = wasUnreportedWrongSize; + return this; + } + + /** + * Records a new result for the given shard of the test. + * + * @return an immutable view of the statuses associated with the shard, with the new element. + */ + public List<BlazeTestStatus> addShardStatus(int shardNumber, BlazeTestStatus status) { + Preconditions.checkState(summary.shardRunStatuses.put(shardNumber, status), + "shardRunStatuses must allow duplicate statuses"); + return ImmutableList.copyOf(summary.shardRunStatuses.get(shardNumber)); + } + + /** + * Returns the created TestSummary object. + * Any actions following a build() will create another copy of the same values. + * Since no mutators are provided directly by TestSummary, a copy will not + * be produced if two builds are invoked in a row without calling a setter. + */ + public TestSummary build() { + peek(); + if (!built) { + makeSummaryImmutable(); + // else: it is already immutable. + } + Preconditions.checkState(built, "Built flag was not set"); + return summary; + } + + /** + * Within-package, it is possible to read directly from an + * incompletely-built TestSummary. Used to pass Builders around directly. + */ + TestSummary peek() { + Preconditions.checkNotNull(summary.target, "Target cannot be null"); + Preconditions.checkNotNull(summary.status, "Status cannot be null"); + return summary; + } + + private void makeSummaryImmutable() { + // Once finalized, the list types are immutable. + summary.passedLogs = Collections.unmodifiableList(summary.passedLogs); + summary.failedLogs = Collections.unmodifiableList(summary.failedLogs); + summary.warnings = Collections.unmodifiableList(summary.warnings); + summary.coverageFiles = Collections.unmodifiableList(summary.coverageFiles); + summary.testTimes = Collections.unmodifiableList(summary.testTimes); + + built = true; + } + } + + private ConfiguredTarget target; + private BlazeTestStatus status; + // Currently only populated if --runs_per_test_detects_flakes is enabled. + private Multimap<Integer, BlazeTestStatus> shardRunStatuses = ArrayListMultimap.create(); + private int numCached; + private int numLocalActionCached; + private boolean actionRan; + private boolean ranRemotely; + private boolean wasUnreportedWrongSize; + private List<TestCase> failedTestCases = new ArrayList<>(); + private List<Path> passedLogs = new ArrayList<>(); + private List<Path> failedLogs = new ArrayList<>(); + private List<String> warnings = new ArrayList<>(); + private List<Path> coverageFiles = new ArrayList<>(); + private List<Long> testTimes = new ArrayList<>(); + private FailedTestCasesStatus failedTestCasesStatus = null; + + // Don't allow public instantiation; go through the Builder. + private TestSummary() { + } + + /** + * Creates a new Builder allowing construction of a new TestSummary object. + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Creates a new Builder initialized with a copy of the existing object's values. + */ + public static Builder newBuilderFromExisting(TestSummary existing) { + Builder builder = new Builder(); + builder.mergeFrom(existing); + return builder; + } + + public ConfiguredTarget getTarget() { + return target; + } + + public BlazeTestStatus getStatus() { + return status; + } + + public boolean isCached() { + return numCached > 0; + } + + public boolean isLocalActionCached() { + return numLocalActionCached > 0; + } + + public int numLocalActionCached() { + return numLocalActionCached; + } + + public int numCached() { + return numCached; + } + + private int numUncached() { + return totalRuns() - numCached; + } + + public boolean actionRan() { + return actionRan; + } + + public boolean ranRemotely() { + return ranRemotely; + } + + public boolean wasUnreportedWrongSize() { + return wasUnreportedWrongSize; + } + + public List<TestCase> getFailedTestCases() { + return failedTestCases; + } + + public List<Path> getCoverageFiles() { + return coverageFiles; + } + + public List<Path> getPassedLogs() { + return passedLogs; + } + + public List<Path> getFailedLogs() { + return failedLogs; + } + + public FailedTestCasesStatus getFailedTestCasesStatus() { + return failedTestCasesStatus; + } + + /** + * Returns an immutable view of the warnings associated with this test. + */ + public List<String> getWarnings() { + return Collections.unmodifiableList(warnings); + } + + private static int getSortKey(BlazeTestStatus status) { + return status == BlazeTestStatus.PASSED ? -1 : status.ordinal(); + } + + @Override + public int compareTo(TestSummary that) { + if (this.isCached() != that.isCached()) { + return this.isCached() ? -1 : 1; + } else if ((this.isCached() && that.isCached()) && (this.numUncached() != that.numUncached())) { + return this.numUncached() - that.numUncached(); + } else if (this.status != that.status) { + return getSortKey(this.status) - getSortKey(that.status); + } else { + Artifact thisExecutable = this.target.getProvider(FilesToRunProvider.class).getExecutable(); + Artifact thatExecutable = that.target.getProvider(FilesToRunProvider.class).getExecutable(); + return thisExecutable.getPath().compareTo(thatExecutable.getPath()); + } + } + + public List<Long> getTestTimes() { + // The return result is unmodifiable (UnmodifiableList instance) + return testTimes; + } + + public int getNumCached() { + return numCached; + } + + public int totalRuns() { + return testTimes.size(); + } + + static Mode getStatusMode(BlazeTestStatus status) { + return status == BlazeTestStatus.PASSED + ? Mode.INFO + : (status == BlazeTestStatus.FLAKY ? Mode.WARNING : Mode.ERROR); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java new file mode 100644 index 0000000000..91c1488054 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/TestSummaryPrinter.java @@ -0,0 +1,255 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; +import com.google.devtools.build.lib.rules.test.TestLogHelper; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter.Mode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; +import com.google.devtools.build.lib.view.test.TestStatus.FailedTestCasesStatus; +import com.google.devtools.build.lib.view.test.TestStatus.TestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; + +/** + * Print test statistics in human readable form. + */ +public class TestSummaryPrinter { + + /** + * Print the cached test log to the given printer. + */ + public static void printCachedOutput(TestSummary summary, + TestOutputFormat testOutput, + AnsiTerminalPrinter printer) { + + String testName = summary.getTarget().getLabel().toString(); + List<String> allLogs = new ArrayList<>(); + for (Path path : summary.getFailedLogs()) { + allLogs.add(path.getPathString()); + } + for (Path path : summary.getPassedLogs()) { + allLogs.add(path.getPathString()); + } + printer.printLn("" + TestSummary.getStatusMode(summary.getStatus()) + summary.getStatus() + ": " + + Mode.DEFAULT + testName + " (see " + Joiner.on(' ').join(allLogs) + ")"); + printer.printLn(Mode.INFO + "INFO: " + Mode.DEFAULT + "From Testing " + testName); + + // Whether to output the target at all was checked by the caller. + // Now check whether to output failing shards. + if (TestLogHelper.shouldOutputTestLog(testOutput, false)) { + for (Path path : summary.getFailedLogs()) { + try { + TestLogHelper.writeTestLog(path, testName, printer.getOutputStream()); + } catch (IOException e) { + printer.printLn("==================== Could not read test output for " + testName); + LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e); + } + } + } + + // And passing shards, independently. + if (TestLogHelper.shouldOutputTestLog(testOutput, true)) { + for (Path path : summary.getPassedLogs()) { + try { + TestLogHelper.writeTestLog(path, testName, printer.getOutputStream()); + } catch (Exception e) { + printer.printLn("==================== Could not read test output for " + testName); + LoggingUtil.logToRemote(Level.WARNING, "Error while reading test log", e); + } + } + } + } + + private static String statusString(BlazeTestStatus status) { + return status.toString().replace('_', ' '); + } + + /** + * Prints summary status for a single test. + * @param terminalPrinter The printer to print to + */ + public static void print( + TestSummary summary, + AnsiTerminalPrinter terminalPrinter, + boolean verboseSummary, boolean printFailedTestCases) { + // Skip output for tests that failed to build. + if (summary.getStatus() == BlazeTestStatus.FAILED_TO_BUILD) { + return; + } + String message = getCacheMessage(summary) + statusString(summary.getStatus()); + terminalPrinter.print( + Strings.padEnd(summary.getTarget().getLabel().toString(), 78 - message.length(), ' ') + + " " + TestSummary.getStatusMode(summary.getStatus()) + message + Mode.DEFAULT + + (verboseSummary ? getAttemptSummary(summary) + getTimeSummary(summary) : "") + "\n"); + + if (printFailedTestCases && summary.getStatus() == BlazeTestStatus.FAILED) { + if (summary.getFailedTestCasesStatus() == FailedTestCasesStatus.NOT_AVAILABLE) { + terminalPrinter.print( + Mode.WARNING + " (individual test case information not available) " + + Mode.DEFAULT + "\n"); + } else { + for (TestCase testCase : summary.getFailedTestCases()) { + if (testCase.getStatus() != TestCase.Status.PASSED) { + TestSummaryPrinter.printTestCase(terminalPrinter, testCase); + } + } + + if (summary.getFailedTestCasesStatus() != FailedTestCasesStatus.FULL) { + terminalPrinter.print( + Mode.WARNING + + " (some shards did not report details, list of failed test" + + " cases incomplete)\n" + + Mode.DEFAULT); + } + } + } + + if (printFailedTestCases) { + // In this mode, test output and coverage files would just clutter up + // the output. + return; + } + + for (String warning : summary.getWarnings()) { + terminalPrinter.print(" " + AnsiTerminalPrinter.Mode.WARNING + "WARNING: " + + AnsiTerminalPrinter.Mode.DEFAULT + warning + "\n"); + } + + for (Path path : summary.getFailedLogs()) { + if (path.exists()) { + // Don't use getPrettyPath() here - we want to print the absolute path, + // so that it cut and paste into a different terminal, and we don't + // want to use the blaze-bin etc. symbolic links because they could be changed + // by a subsequent build with different options. + terminalPrinter.print(" " + path.getPathString() + "\n"); + } + } + for (Path path : summary.getCoverageFiles()) { + // Print only non-trivial coverage files. + try { + if (path.exists() && path.getFileSize() > 0) { + terminalPrinter.print(" " + path.getPathString() + "\n"); + } + } catch (IOException e) { + LoggingUtil.logToRemote(Level.WARNING, "Error while reading coverage data file size", + e); + } + } + } + + /** + * Prints the result of an individual test case. It is assumed not to have + * passed, since passed test cases are not reported. + */ + static void printTestCase( + AnsiTerminalPrinter terminalPrinter, TestCase testCase) { + String timeSummary; + if (testCase.hasRunDurationMillis()) { + timeSummary = " (" + + timeInSec(testCase.getRunDurationMillis(), TimeUnit.MILLISECONDS) + + ")"; + } else { + timeSummary = ""; + } + + terminalPrinter.print( + " " + + Mode.ERROR + + Strings.padEnd(testCase.getStatus().toString(), 8, ' ') + + Mode.DEFAULT + + testCase.getClassName() + + "." + + testCase.getName() + + timeSummary + + "\n"); + } + + /** + * Return the given time in seconds, to 1 decimal place, + * i.e. "32.1s". + */ + static String timeInSec(long time, TimeUnit unit) { + double ms = TimeUnit.MILLISECONDS.convert(time, unit); + return String.format("%.1fs", ms / 1000.0); + } + + static String getAttemptSummary(TestSummary summary) { + int attempts = summary.getPassedLogs().size() + summary.getFailedLogs().size(); + if (attempts > 1) { + // Print number of failed runs for failed tests if testing was completed. + if (summary.getStatus() == BlazeTestStatus.FLAKY) { + return ", failed in " + summary.getFailedLogs().size() + " out of " + attempts; + } + if (summary.getStatus() == BlazeTestStatus.TIMEOUT + || summary.getStatus() == BlazeTestStatus.FAILED) { + return " in " + summary.getFailedLogs().size() + " out of " + attempts; + } + } + return ""; + } + + static String getCacheMessage(TestSummary summary) { + if (summary.getNumCached() == 0 || summary.getStatus() == BlazeTestStatus.INCOMPLETE) { + return ""; + } else if (summary.getNumCached() == summary.totalRuns()) { + return "(cached) "; + } else { + return String.format("(%d/%d cached) ", summary.getNumCached(), summary.totalRuns()); + } + } + + static String getTimeSummary(TestSummary summary) { + if (summary.getTestTimes().isEmpty()) { + return ""; + } else if (summary.getTestTimes().size() == 1) { + return " in " + timeInSec(summary.getTestTimes().get(0), TimeUnit.MILLISECONDS); + } else { + // We previously used com.google.math for this, which added about 1 MB of deps to the total + // size. If we re-introduce a dependency on that package, we could revert this change. + long min = summary.getTestTimes().get(0).longValue(), max = min, sum = 0; + double sumOfSquares = 0.0; + for (Long l : summary.getTestTimes()) { + long value = l.longValue(); + min = value < min ? value : min; + max = value > max ? value : max; + sum += value; + sumOfSquares += ((double) value) * (double) value; + } + double mean = ((double) sum) / summary.getTestTimes().size(); + double stddev = Math.sqrt((sumOfSquares - sum * mean) / summary.getTestTimes().size()); + // For sharded tests, we print the max time on the same line as + // the test, and then print more detailed info about the + // distribution of times on the next line. + String maxTime = timeInSec(max, TimeUnit.MILLISECONDS); + return String.format( + " in %s\n Stats over %d runs: max = %s, min = %s, avg = %s, dev = %s", + maxTime, + summary.getTestTimes().size(), + maxTime, + timeInSec(min, TimeUnit.MILLISECONDS), + timeInSec((long) mean, TimeUnit.MILLISECONDS), + timeInSec((long) stddev, TimeUnit.MILLISECONDS)); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java new file mode 100644 index 0000000000..d6f61eb494 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.List; + +/** + * Handles the 'build' command on the Blaze command line, including targets + * named by arguments passed to Blaze. + */ +@Command(name = "build", + builds = true, + options = { BuildRequestOptions.class, + ExecutionOptions.class, + PackageCacheOptions.class, + BuildView.Options.class, + LoadingPhaseRunner.Options.class, + BuildConfiguration.Options.class, + }, + usesConfigurationOptions = true, + shortDescription = "Builds the specified targets.", + allowResidue = true, + help = "resource:build.txt") +public final class BuildCommand implements BlazeCommand { + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) + throws AbruptExitException { + ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "build"); + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + List<String> targets = ProjectFileSupport.getTargets(runtime, options); + + BuildRequest request = BuildRequest.create( + getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), + targets, + runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime()); + return runtime.getBuildTool().processRequest(request, null).getExitCondition(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java new file mode 100644 index 0000000000..0bb5a0e966 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java @@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandUtils; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Collection; +import java.util.List; + +/** + * The 'blaze canonicalize-flags' command. + */ +@Command(name = "canonicalize-flags", + options = { CanonicalizeCommand.Options.class }, + allowResidue = true, + mustRunInWorkspace = false, + shortDescription = "Canonicalizes a list of Blaze options.", + help = "This command canonicalizes a list of Blaze options. Don't forget to prepend '--' " + + "to end option parsing before the flags to canonicalize.\n" + + "%{options}") +public final class CanonicalizeCommand implements BlazeCommand { + + public static class CommandConverter implements Converter<String> { + + @Override + public String convert(String input) throws OptionsParsingException { + if (input.equals("build")) { + return input; + } else if (input.equals("test")) { + return input; + } + throw new OptionsParsingException("Not a valid command: '" + input + "' (should be " + + getTypeDescription() + ")"); + } + + @Override + public String getTypeDescription() { + return "build or test"; + } + } + + public static class Options extends OptionsBase { + + @Option(name = "for_command", + defaultValue = "build", + category = "misc", + converter = CommandConverter.class, + help = "The command for which the options should be canonicalized.") + public String forCommand; + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + BlazeCommand command = runtime.getCommandMap().get( + options.getOptions(Options.class).forCommand); + Collection<Class<? extends OptionsBase>> optionsClasses = + BlazeCommandUtils.getOptions( + command.getClass(), runtime.getBlazeModules(), runtime.getRuleClassProvider()); + try { + List<String> result = OptionsParser.canonicalize(optionsClasses, options.getResidue()); + for (String piece : result) { + runtime.getReporter().getOutErr().printOutLn(piece); + } + } catch (OptionsParsingException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.COMMAND_LINE_ERROR; + } + return ExitCode.SUCCESS; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java new file mode 100644 index 0000000000..3fd300eaaa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java @@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.ProcessUtils; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Implements 'blaze clean'. + */ +@Command(name = "clean", + builds = true, // Does not, but people expect build options to be there + options = { CleanCommand.Options.class }, + help = "resource:clean.txt", + shortDescription = "Removes output files and optionally stops the server.", + // TODO(bazel-team): Remove this - we inherit a huge number of unused options. + inherits = { BuildCommand.class }) +public final class CleanCommand implements BlazeCommand { + + /** + * An interface for special options for the clean command. + */ + public static class Options extends OptionsBase { + @Option(name = "clean_style", + defaultValue = "", + category = "clean", + help = "Can be either 'expunge' or 'expunge_async'.") + public String cleanStyle; + + @Option(name = "expunge", + defaultValue = "false", + category = "clean", + expansion = "--clean_style=expunge", + help = "If specified, clean will remove the entire working tree for this Blaze " + + "instance, which includes all Blaze-created temporary and build output " + + "files, and it will stop the Blaze server if it is running.") + public boolean expunge; + + @Option(name = "expunge_async", + defaultValue = "false", + category = "clean", + expansion = "--clean_style=expunge_async", + help = "If specified, clean will asynchronously remove the entire working tree for " + + "this Blaze instance, which includes all Blaze-created temporary and build " + + "output files, and it will stop the Blaze server if it is running. When this " + + "command completes, it will be safe to execute new commands in the same client, " + + "even though the deletion may continue in the background.") + public boolean expunge_async; + } + + private static Logger LOG = Logger.getLogger(CleanCommand.class.getName()); + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + Options cleanOptions = options.getOptions(Options.class); + cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async"); + cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge"); + + if (cleanOptions.expunge == false && cleanOptions.expunge_async == false && + !cleanOptions.cleanStyle.isEmpty()) { + runtime.getReporter().handle(Event.error( + null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'")); + return ExitCode.COMMAND_LINE_ERROR; + } + + String cleanBanner = cleanOptions.expunge_async ? + "Starting clean." : + "Starting clean (this may take a while). " + + "Consider using --expunge_async if the clean takes more than several minutes."; + + runtime.getReporter().handle(Event.info(null/*location*/, cleanBanner)); + try { + String symlinkPrefix = + options.getOptions(BuildRequest.BuildRequestOptions.class).symlinkPrefix; + actuallyClean(runtime, runtime.getOutputBase(), cleanOptions, symlinkPrefix); + return ExitCode.SUCCESS; + } catch (IOException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } catch (CommandException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.RUN_FAILURE; + } catch (ExecException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.RUN_FAILURE; + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("clean interrupted")); + return ExitCode.INTERRUPTED; + } + } + + private void actuallyClean(BlazeRuntime runtime, + Path outputBase, Options cleanOptions, String symlinkPrefix) throws IOException, + ShutdownBlazeServerException, CommandException, ExecException, InterruptedException { + if (runtime.getOutputService() != null) { + runtime.getOutputService().clean(); + } + if (cleanOptions.expunge) { + LOG.info("Expunging..."); + // Delete the big subdirectories with the important content first--this + // will take the most time. Then quickly delete the little locks, logs + // and links right before we exit. Once the lock file is gone there will + // be a small possibility of a server race if a client is waiting, but + // all significant files will be gone by then. + FileSystemUtils.deleteTreesBelow(outputBase); + FileSystemUtils.deleteTree(outputBase); + } else if (cleanOptions.expunge_async) { + LOG.info("Expunging asynchronously..."); + String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid(); + + // Keeping tempOutputBase in the same directory ensures it remains in the + // same file system, and therefore the mv will be atomic and fast. + Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName); + outputBase.renameTo(tempOutputBase); + runtime.getReporter().handle(Event.info( + null, "Output base moved to " + tempOutputBase + " for deletion")); + + // Daemonize the shell and use the double-fork idiom to ensure that the shell + // exits even while the "rm -rf" command continues. + String command = String.format("exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&", + ShellEscaper.escapeString(tempOutputBase.getPathString())); + + LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command)); + + // Doesn't throw iff command exited and was successful. + new CommandBuilder().addArg(command).useShell(true) + .setWorkingDir(tempOutputBase.getParentDirectory()) + .build().execute(); + } else { + LOG.info("Output cleaning..."); + runtime.clearCaches(); + for (String directory : new String[] { + BlazeDirectories.RELATIVE_OUTPUT_PATH, runtime.getWorkspaceName() }) { + Path child = outputBase.getChild(directory); + if (child.exists()) { + LOG.finest("Cleaning " + child); + FileSystemUtils.deleteTreesBelow(child); + } + } + } + // remove convenience links + OutputDirectoryLinksUtils.removeOutputDirectoryLinks( + runtime.getWorkspaceName(), runtime.getWorkspace(), runtime.getReporter(), symlinkPrefix); + // shutdown on expunge cleans + if (cleanOptions.expunge || cleanOptions.expunge_async) { + throw new ShutdownBlazeServerException(0); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java new file mode 100644 index 0000000000..5267e71acc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java @@ -0,0 +1,248 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.docgen.BlazeRuleHelpPrinter; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandUtils; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The 'blaze help' command, which prints all available commands as well as + * specific help pages. + */ +@Command(name = "help", + options = { HelpCommand.Options.class }, + allowResidue = true, + mustRunInWorkspace = false, + shortDescription = "Prints help for commands, or the index.", + help = "resource:help.txt") +public final class HelpCommand implements BlazeCommand { + public static class Options extends OptionsBase { + + @Option(name = "help_verbosity", + category = "help", + defaultValue = "medium", + converter = Converters.HelpVerbosityConverter.class, + help = "Select the verbosity of the help command.") + public OptionsParser.HelpVerbosity helpVerbosity; + + @Option(name = "long", + abbrev = 'l', + defaultValue = "null", + category = "help", + expansion = {"--help_verbosity", "long"}, + help = "Show full description of each option, instead of just its name.") + public Void showLongFormOptions; + + @Option(name = "short", + defaultValue = "null", + category = "help", + expansion = {"--help_verbosity", "short"}, + help = "Show only the names of the options, not their types or meanings.") + public Void showShortFormOptions; + } + + /** + * Returns a map that maps option categories to descriptive help strings for categories that + * are not part of the Bazel core. + */ + private ImmutableMap<String, String> getOptionCategories(BlazeRuntime runtime) { + ImmutableMap.Builder<String, String> optionCategoriesBuilder = ImmutableMap.builder(); + optionCategoriesBuilder + .put("checking", + "Checking options, which control Blaze's error checking and/or warnings") + .put("coverage", + "Options that affect how Blaze generates code coverage information") + .put("experimental", + "Experimental options, which control experimental (and potentially risky) features") + .put("flags", + "Flags options, for passing options to other tools") + .put("help", + "Help options") + .put("host jvm startup", + "Options that affect the startup of the Blaze server's JVM") + .put("misc", + "Miscellaneous options") + .put("package loading", + "Options that specify how to locate packages") + .put("query", + "Options affecting the 'blaze query' dependency query command") + .put("run", + "Options specific to 'blaze run'") + .put("semantics", + "Semantics options, which affect the build commands and/or output file contents") + .put("server startup", + "Startup options, which affect the startup of the Blaze server") + .put("strategy", + "Strategy options, which affect how Blaze will execute the build") + .put("testing", + "Options that affect how Blaze runs tests") + .put("verbosity", + "Verbosity options, which control what Blaze prints") + .put("version", + "Version options, for selecting which version of other tools will be used") + .put("what", + "Output selection options, for determining what to build/test"); + for (BlazeModule module : runtime.getBlazeModules()) { + optionCategoriesBuilder.putAll(module.getOptionCategories()); + } + return optionCategoriesBuilder.build(); + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + OutErr outErr = runtime.getReporter().getOutErr(); + Options helpOptions = options.getOptions(Options.class); + if (options.getResidue().isEmpty()) { + emitBlazeVersionInfo(outErr); + emitGenericHelp(runtime, outErr); + return ExitCode.SUCCESS; + } + if (options.getResidue().size() != 1) { + runtime.getReporter().handle(Event.error("You must specify exactly one command")); + return ExitCode.COMMAND_LINE_ERROR; + } + String helpSubject = options.getResidue().get(0); + if (helpSubject.equals("startup_options")) { + emitBlazeVersionInfo(outErr); + emitStartupOptions(outErr, helpOptions.helpVerbosity, runtime, getOptionCategories(runtime)); + return ExitCode.SUCCESS; + } else if (helpSubject.equals("target-syntax")) { + emitBlazeVersionInfo(outErr); + emitTargetSyntaxHelp(outErr, getOptionCategories(runtime)); + return ExitCode.SUCCESS; + } else if (helpSubject.equals("info-keys")) { + emitInfoKeysHelp(runtime, outErr); + return ExitCode.SUCCESS; + } + + BlazeCommand command = runtime.getCommandMap().get(helpSubject); + if (command == null) { + ConfiguredRuleClassProvider provider = runtime.getRuleClassProvider(); + RuleClass ruleClass = provider.getRuleClassMap().get(helpSubject); + if (ruleClass != null && ruleClass.isDocumented()) { + // There is a rule with a corresponding name + outErr.printOut(BlazeRuleHelpPrinter.getRuleDoc(helpSubject, provider)); + return ExitCode.SUCCESS; + } else { + runtime.getReporter().handle(Event.error( + null, "'" + helpSubject + "' is neither a command nor a build rule")); + return ExitCode.COMMAND_LINE_ERROR; + } + } + emitBlazeVersionInfo(outErr); + outErr.printOut(BlazeCommandUtils.getUsage( + command.getClass(), + getOptionCategories(runtime), + helpOptions.helpVerbosity, + runtime.getBlazeModules(), + runtime.getRuleClassProvider())); + return ExitCode.SUCCESS; + } + + private void emitBlazeVersionInfo(OutErr outErr) { + String releaseInfo = BlazeVersionInfo.instance().getReleaseName(); + String line = "[Blaze " + releaseInfo + "]"; + outErr.printOut(String.format("%80s\n", line)); + } + + @SuppressWarnings("unchecked") // varargs generic array creation + private void emitStartupOptions(OutErr outErr, OptionsParser.HelpVerbosity helpVerbosity, + BlazeRuntime runtime, ImmutableMap<String, String> optionCategories) { + outErr.printOut( + BlazeCommandUtils.expandHelpTopic("startup_options", + "resource:startup_options.txt", + getClass(), + BlazeCommandUtils.getStartupOptions(runtime.getBlazeModules()), + optionCategories, + helpVerbosity)); + } + + private void emitTargetSyntaxHelp(OutErr outErr, ImmutableMap<String, String> optionCategories) { + outErr.printOut(BlazeCommandUtils.expandHelpTopic("target-syntax", + "resource:target-syntax.txt", + getClass(), + ImmutableList.<Class<? extends OptionsBase>>of(), + optionCategories, + OptionsParser.HelpVerbosity.MEDIUM)); + } + + private void emitInfoKeysHelp(BlazeRuntime runtime, OutErr outErr) { + for (InfoKey key : InfoKey.values()) { + outErr.printOut(String.format("%-23s %s\n", key.getName(), key.getDescription())); + } + + for (BlazeModule.InfoItem item : InfoCommand.getInfoItemMap(runtime, + OptionsParser.newOptionsParser( + ImmutableList.<Class<? extends OptionsBase>>of())).values()) { + outErr.printOut(String.format("%-23s %s\n", item.getName(), item.getDescription())); + } + } + + private void emitGenericHelp(BlazeRuntime runtime, OutErr outErr) { + outErr.printOut("Usage: blaze <command> <options> ...\n\n"); + + outErr.printOut("Available commands:\n"); + + Map<String, BlazeCommand> commandsByName = runtime.getCommandMap(); + List<String> namesInOrder = new ArrayList<>(commandsByName.keySet()); + Collections.sort(namesInOrder); + + for (String name : namesInOrder) { + BlazeCommand command = commandsByName.get(name); + Command annotation = command.getClass().getAnnotation(Command.class); + if (annotation.hidden()) { + continue; + } + + String shortDescription = annotation.shortDescription(); + outErr.printOut(String.format(" %-19s %s\n", name, shortDescription)); + } + + outErr.printOut("\n"); + outErr.printOut("Getting more help:\n"); + outErr.printOut(" blaze help <command>\n"); + outErr.printOut(" Prints help and options for <command>.\n"); + outErr.printOut(" blaze help startup_options\n"); + outErr.printOut(" Options for the JVM hosting Blaze.\n"); + outErr.printOut(" blaze help target-syntax\n"); + outErr.printOut(" Explains the syntax for specifying targets.\n"); + outErr.printOut(" blaze help info-keys\n"); + outErr.printOut(" Displays a list of keys used by the info command.\n"); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java new file mode 100644 index 0000000000..31aaeb1cef --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java @@ -0,0 +1,448 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Supplier; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.ProtoUtils; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.AllowedRuleClassInfo; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.AttributeDefinition; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.BuildLanguage; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.RuleDefinition; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Implementation of 'blaze info'. + */ +@Command(name = "info", + // TODO(bazel-team): this is not really a build command, but needs access to the + // configuration options to do its job + builds = true, + allowResidue = true, + binaryStdOut = true, + help = "resource:info.txt", + shortDescription = "Displays runtime info about the blaze server.", + options = { InfoCommand.Options.class }, + // We have InfoCommand inherit from {@link BuildCommand} because we want all + // configuration defaults specified in ~/.blazerc for {@code build} to apply to + // {@code info} too, even though it doesn't actually do a build. + // + // (Ideally there would be a way to make {@code info} inherit just the bare + // minimum of relevant options from {@code build}, i.e. those that affect the + // values it prints. But there's no such mechanism.) + inherits = { BuildCommand.class }) +public class InfoCommand implements BlazeCommand { + + public static class Options extends OptionsBase { + @Option(name = "show_make_env", + defaultValue = "false", + category = "misc", + help = "Include the \"Make\" environment in the output.") + public boolean showMakeEnvironment; + } + + /** + * Unchecked variant of ExitCausingException. Below, we need to throw from the Supplier interface, + * which does not allow checked exceptions. + */ + public static class ExitCausingRuntimeException extends RuntimeException { + + private final ExitCode exitCode; + + public ExitCausingRuntimeException(String message, ExitCode exitCode) { + super(message); + this.exitCode = exitCode; + } + + public ExitCausingRuntimeException(ExitCode exitCode) { + this.exitCode = exitCode; + } + + public ExitCode getExitCode() { + return exitCode; + } + } + + private static class HardwiredInfoItem implements BlazeModule.InfoItem { + private final InfoKey key; + private final BlazeRuntime runtime; + private final OptionsProvider commandOptions; + + private HardwiredInfoItem(InfoKey key, BlazeRuntime runtime, OptionsProvider commandOptions) { + this.key = key; + this.runtime = runtime; + this.commandOptions = commandOptions; + } + + @Override + public String getName() { + return key.getName(); + } + + @Override + public String getDescription() { + return key.getDescription(); + } + + @Override + public boolean isHidden() { + return key.isHidden(); + } + + @Override + public byte[] get(Supplier<BuildConfiguration> configurationSupplier) { + return print(getInfoItem(runtime, key, configurationSupplier, commandOptions)); + } + } + + private static class MakeInfoItem implements BlazeModule.InfoItem { + private final String name; + private final String value; + + private MakeInfoItem(String name, String value) { + this.name = name; + this.value = value; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "Make environment variable '" + name + "'"; + } + + @Override + public boolean isHidden() { + return false; + } + + @Override + public byte[] get(Supplier<BuildConfiguration> configurationSupplier) { + return print(value); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + @Override + public ExitCode exec(final BlazeRuntime runtime, final OptionsProvider optionsProvider) { + Options infoOptions = optionsProvider.getOptions(Options.class); + + OutErr outErr = runtime.getReporter().getOutErr(); + // Creating a BuildConfiguration is expensive and often unnecessary. Delay the creation until + // it is needed. + Supplier<BuildConfiguration> configurationSupplier = new Supplier<BuildConfiguration>() { + private BuildConfiguration configuration; + @Override + public BuildConfiguration get() { + if (configuration != null) { + return configuration; + } + try { + // In order to be able to answer configuration-specific queries, we need to setup the + // package path. Since info inherits all the build options, all the necessary information + // is available here. + runtime.setupPackageCache( + optionsProvider.getOptions(PackageCacheOptions.class), + runtime.getDefaultsPackageContent(optionsProvider)); + // TODO(bazel-team): What if there are multiple configurations? [multi-config] + configuration = runtime + .getConfigurations(optionsProvider) + .getTargetConfigurations().get(0); + return configuration; + } catch (InvalidConfigurationException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + throw new ExitCausingRuntimeException(ExitCode.COMMAND_LINE_ERROR); + } catch (AbruptExitException e) { + throw new ExitCausingRuntimeException("unknown error: " + e.getMessage(), + e.getExitCode()); + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("interrupted")); + throw new ExitCausingRuntimeException(ExitCode.INTERRUPTED); + } + } + }; + + Map<String, BlazeModule.InfoItem> items = getInfoItemMap(runtime, optionsProvider); + + try { + if (infoOptions.showMakeEnvironment) { + Map<String, String> makeEnv = configurationSupplier.get().getMakeEnvironment(); + for (Map.Entry<String, String> entry : makeEnv.entrySet()) { + BlazeModule.InfoItem item = new MakeInfoItem(entry.getKey(), entry.getValue()); + items.put(item.getName(), item); + } + } + + List<String> residue = optionsProvider.getResidue(); + if (residue.size() > 1) { + runtime.getReporter().handle(Event.error("at most one key may be specified")); + return ExitCode.COMMAND_LINE_ERROR; + } + + String key = residue.size() == 1 ? residue.get(0) : null; + if (key != null) { // print just the value for the specified key: + byte[] value; + if (items.containsKey(key)) { + value = items.get(key).get(configurationSupplier); + } else { + runtime.getReporter().handle(Event.error("unknown key: '" + key + "'")); + return ExitCode.COMMAND_LINE_ERROR; + } + try { + outErr.getOutputStream().write(value); + outErr.getOutputStream().flush(); + } catch (IOException e) { + runtime.getReporter().handle(Event.error("Cannot write info block: " + e.getMessage())); + return ExitCode.ANALYSIS_FAILURE; + } + } else { // print them all + configurationSupplier.get(); // We'll need this later anyway + for (BlazeModule.InfoItem infoItem : items.values()) { + if (infoItem.isHidden()) { + continue; + } + outErr.getOutputStream().write( + (infoItem.getName() + ": ").getBytes(StandardCharsets.UTF_8)); + outErr.getOutputStream().write(infoItem.get(configurationSupplier)); + } + } + } catch (AbruptExitException e) { + return e.getExitCode(); + } catch (ExitCausingRuntimeException e) { + return e.getExitCode(); + } catch (IOException e) { + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + return ExitCode.SUCCESS; + } + + /** + * Compute and return the info for the given key. Only keys that are not hidden are supported + * here. + */ + private static Object getInfoItem(BlazeRuntime runtime, InfoKey key, + Supplier<BuildConfiguration> configurationSupplier, OptionsProvider options) { + switch (key) { + // directories + case WORKSPACE : return runtime.getWorkspace(); + case INSTALL_BASE : return runtime.getDirectories().getInstallBase(); + case OUTPUT_BASE : return runtime.getOutputBase(); + case EXECUTION_ROOT : return runtime.getExecRoot(); + case OUTPUT_PATH : return runtime.getDirectories().getOutputPath(); + // These are the only (non-hidden) info items that require a configuration, because the + // corresponding paths contain the short name. Maybe we should recommend using the symlinks + // or make them hidden by default? + case BLAZE_BIN : return configurationSupplier.get().getBinDirectory().getPath(); + case BLAZE_GENFILES : return configurationSupplier.get().getGenfilesDirectory().getPath(); + case BLAZE_TESTLOGS : return configurationSupplier.get().getTestLogsDirectory().getPath(); + + // logs + case COMMAND_LOG : return BlazeCommandDispatcher.getCommandLogPath(runtime.getOutputBase()); + case MESSAGE_LOG : + // NB: Duplicated in EventLogModule + return runtime.getOutputBase().getRelative("message.log"); + + // misc + case RELEASE : return BlazeVersionInfo.instance().getReleaseName(); + case SERVER_PID : return OsUtils.getpid(); + case PACKAGE_PATH : return getPackagePath(options); + + // memory statistics + case GC_COUNT : + case GC_TIME : + // The documentation is not very clear on what it means to have more than + // one GC MXBean, so we just sum them up. + int gcCount = 0; + int gcTime = 0; + for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcCount += gcBean.getCollectionCount(); + gcTime += gcBean.getCollectionTime(); + } + if (key == InfoKey.GC_COUNT) { + return gcCount + ""; + } else { + return gcTime + "ms"; + } + + case MAX_HEAP_SIZE : + return StringUtilities.prettyPrintBytes(getMemoryUsage().getMax()); + case USED_HEAP_SIZE : + case COMMITTED_HEAP_SIZE : + return StringUtilities.prettyPrintBytes(key == InfoKey.USED_HEAP_SIZE ? + getMemoryUsage().getUsed() : getMemoryUsage().getCommitted()); + + case USED_HEAP_SIZE_AFTER_GC : + // Note that this info value is not printed by default, but only when explicitly requested. + System.gc(); + return StringUtilities.prettyPrintBytes(getMemoryUsage().getUsed()); + + case DEFAULTS_PACKAGE: + return runtime.getDefaultsPackageContent(); + + case BUILD_LANGUAGE: + return getBuildLanguageDefinition(runtime.getRuleClassProvider()); + + case DEFAULT_PACKAGE_PATH: + return Joiner.on(":").join(Constants.DEFAULT_PACKAGE_PATH); + + default: + throw new IllegalArgumentException("missing implementation for " + key); + } + } + + private static MemoryUsage getMemoryUsage() { + MemoryMXBean memBean = ManagementFactory.getMemoryMXBean(); + return memBean.getHeapMemoryUsage(); + } + + /** + * Get the package_path variable for the given set of options. + */ + private static String getPackagePath(OptionsProvider options) { + PackageCacheOptions packageCacheOptions = + options.getOptions(PackageCacheOptions.class); + return Joiner.on(":").join(packageCacheOptions.packagePath); + } + + private static AllowedRuleClassInfo getAllowedRuleClasses( + Collection<RuleClass> ruleClasses, Attribute attr) { + AllowedRuleClassInfo.Builder info = AllowedRuleClassInfo.newBuilder(); + info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.ANY); + + if (attr.isStrictLabelCheckingEnabled()) { + if (attr.getAllowedRuleClassesPredicate() != Predicates.<RuleClass>alwaysTrue()) { + info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.SPECIFIED); + Predicate<RuleClass> filter = attr.getAllowedRuleClassesPredicate(); + for (RuleClass otherClass : Iterables.filter( + ruleClasses, filter)) { + if (otherClass.isDocumented()) { + info.addAllowedRuleClass(otherClass.getName()); + } + } + } + } + + return info.build(); + } + + /** + * Returns a byte array containing a proto-buffer describing the build language. + */ + private static byte[] getBuildLanguageDefinition(RuleClassProvider provider) { + BuildLanguage.Builder resultPb = BuildLanguage.newBuilder(); + Collection<RuleClass> ruleClasses = provider.getRuleClassMap().values(); + for (RuleClass ruleClass : ruleClasses) { + if (!ruleClass.isDocumented()) { + continue; + } + + RuleDefinition.Builder rulePb = RuleDefinition.newBuilder(); + rulePb.setName(ruleClass.getName()); + for (Attribute attr : ruleClass.getAttributes()) { + if (!attr.isDocumented()) { + continue; + } + + AttributeDefinition.Builder attrPb = AttributeDefinition.newBuilder(); + attrPb.setName(attr.getName()); + // The protocol compiler, in its infinite wisdom, generates the field as one of the + // integer type and the getTypeEnum() method is missing. WTF? + attrPb.setType(ProtoUtils.getDiscriminatorFromType(attr.getType())); + attrPb.setMandatory(attr.isMandatory()); + + if (Type.isLabelType(attr.getType())) { + attrPb.setAllowedRuleClasses(getAllowedRuleClasses(ruleClasses, attr)); + } + + rulePb.addAttribute(attrPb); + } + + resultPb.addRule(rulePb); + } + + return resultPb.build().toByteArray(); + } + + private static byte[] print(Object value) { + if (value instanceof byte[]) { + return (byte[]) value; + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(outputStream); + writer.print(value.toString() + "\n"); + writer.flush(); + return outputStream.toByteArray(); + } + + static Map<String, BlazeModule.InfoItem> getInfoItemMap( + BlazeRuntime runtime, OptionsProvider commandOptions) { + Map<String, BlazeModule.InfoItem> result = new TreeMap<>(); // order by key + for (BlazeModule module : runtime.getBlazeModules()) { + for (BlazeModule.InfoItem item : module.getInfoItems()) { + result.put(item.getName(), item); + } + } + + for (InfoKey key : InfoKey.values()) { + BlazeModule.InfoItem item = new HardwiredInfoItem(key, runtime, commandOptions); + result.put(item.getName(), item); + } + + return result; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java new file mode 100644 index 0000000000..d2e7bc0718 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java @@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + + +/** + * An enumeration of all the valid info keys, excepting the make environment + * variables. + */ +public enum InfoKey { + // directories + WORKSPACE("workspace", "The working directory of the server."), + INSTALL_BASE("install_base", "The installation base directory."), + OUTPUT_BASE("output_base", + "A directory for shared Blaze state as well as tool and strategy specific subdirectories."), + EXECUTION_ROOT("execution_root", + "A directory that makes all input and output files visible to the build."), + OUTPUT_PATH("output_path", "Output directory"), + BLAZE_BIN("blaze-bin", "Configuration dependent directory for binaries."), + BLAZE_GENFILES("blaze-genfiles", "Configuration dependent directory for generated files."), + BLAZE_TESTLOGS("blaze-testlogs", "Configuration dependent directory for logs from a test run."), + + // logs + COMMAND_LOG("command_log", "Location of the log containg the output from the build commands."), + MESSAGE_LOG("message_log" , + "Location of a log containing machine readable message in LogMessage protobuf format."), + + // misc + RELEASE("release", "Blaze release identifier"), + SERVER_PID("server_pid", "Blaze process id"), + PACKAGE_PATH("package_path", "The search path for resolving package labels."), + + // memory statistics + USED_HEAP_SIZE("used-heap-size", "The amount of used memory in bytes. Note that this is not a " + + "good indicator of the actual memory use, as it includes any remaining inaccessible " + + "memory."), + USED_HEAP_SIZE_AFTER_GC("used-heap-size-after-gc", + "The amount of used memory in bytes after a call to System.gc().", true), + COMMITTED_HEAP_SIZE("committed-heap-size", + "The amount of memory in bytes that is committed for the Java virtual machine to use"), + MAX_HEAP_SIZE("max-heap-size", + "The maximum amount of memory in bytes that can be used for memory management."), + GC_COUNT("gc-count", "Number of garbage collection runs."), + GC_TIME("gc-time", "The approximate accumulated time spend on garbage collection."), + + // These are deprecated, they still work, when explicitly requested, but are not shown by default + + // These keys print multi-line messages and thus don't play well with grep. We don't print them + // unless explicitly requested + DEFAULTS_PACKAGE("defaults-package", "Default packages used as implicit dependencies", true), + BUILD_LANGUAGE("build-language", "A protobuffer with the build language structure", true), + DEFAULT_PACKAGE_PATH("default-package-path", "The default package path", true); + + private final String name; + private final String description; + private final boolean hidden; + + private InfoKey(String name, String description) { + this(name, description, false); + } + + private InfoKey(String name, String description, boolean hidden) { + this.name = name; + this.description = description; + this.hidden = hidden; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public boolean isHidden() { + return hidden; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java new file mode 100644 index 0000000000..7b91dc7fa3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java @@ -0,0 +1,771 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.google.common.collect.TreeMultimap; +import com.google.devtools.build.lib.actions.MiddlemanAction; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.profiler.ProfileInfo; +import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry; +import com.google.devtools.build.lib.profiler.ProfileInfo.InfoListener; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.chart.AggregatingChartCreator; +import com.google.devtools.build.lib.profiler.chart.Chart; +import com.google.devtools.build.lib.profiler.chart.ChartCreator; +import com.google.devtools.build.lib.profiler.chart.DetailedChartCreator; +import com.google.devtools.build.lib.profiler.chart.HtmlChartVisitor; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.util.TimeUtilities; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +/** + * Command line wrapper for analyzing Blaze build profiles. + */ +@Command(name = "analyze-profile", + options = { ProfileCommand.ProfileOptions.class }, + shortDescription = "Analyzes build profile data.", + help = "resource:analyze-profile.txt", + allowResidue = true, + mustRunInWorkspace = false) +public final class ProfileCommand implements BlazeCommand { + + private final String TWO_COLUMN_FORMAT = "%-37s %10s\n"; + private final String THREE_COLUMN_FORMAT = "%-28s %10s %8s\n"; + + public static class DumpConverter extends Converters.StringSetConverter { + public DumpConverter() { + super("text", "raw", "text-unsorted", "raw-unsorted"); + } + } + + public static class ProfileOptions extends OptionsBase { + @Option(name = "dump", + abbrev='d', + converter = DumpConverter.class, + defaultValue = "null", + help = "output full profile data dump either in human-readable 'text' format or" + + " script-friendly 'raw' format, either sorted or unsorted.") + public String dumpMode; + + @Option(name = "html", + defaultValue = "false", + help = "If present, an HTML file visualizing the tasks of the profiled build is created. " + + "The name of the html file is the name of the profile file plus '.html'.") + public boolean html; + + @Option(name = "html_pixels_per_second", + defaultValue = "50", + help = "Defines the scale of the time axis of the task diagram. The unit is " + + "pixels per second. Default is 50 pixels per second. ") + public int htmlPixelsPerSecond; + + @Option(name = "html_details", + defaultValue = "false", + help = "If --html_details is present, the task diagram contains all tasks of the profile. " + + "If --nohtml_details is present, an aggregated diagram is generated. The default is " + + "to generate an aggregated diagram.") + public boolean htmlDetails; + + @Option(name = "vfs_stats", + defaultValue = "false", + help = "If present, include VFS path statistics.") + public boolean vfsStats; + + @Option(name = "vfs_stats_limit", + defaultValue = "-1", + help = "Maximum number of VFS path statistics to print.") + public int vfsStatsLimit; + } + + private Function<String, String> currentPathMapping = Functions.<String>identity(); + + private InfoListener getInfoListener(final BlazeRuntime runtime) { + return new InfoListener() { + private final EventHandler reporter = runtime.getReporter(); + + @Override + public void info(String text) { + reporter.handle(Event.info(text)); + } + + @Override + public void warn(String text) { + reporter.handle(Event.warn(text)); + } + }; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(final BlazeRuntime runtime, OptionsProvider options) { + ProfileOptions opts = + options.getOptions(ProfileOptions.class); + + if (!opts.vfsStats) { + opts.vfsStatsLimit = 0; + } + + currentPathMapping = new Function<String, String>() { + @Override + public String apply(String input) { + if (runtime.getWorkspaceName().isEmpty()) { + return input; + } else { + return input.substring(input.lastIndexOf("/" + runtime.getWorkspaceName()) + 1); + } + } + }; + + PrintStream out = new PrintStream(runtime.getReporter().getOutErr().getOutputStream()); + try { + runtime.getReporter().handle(Event.warn( + null, "This information is intended for consumption by Blaze developers" + + " only, and may change at any time. Script against it at your own risk")); + + for (String name : options.getResidue()) { + Path profileFile = runtime.getWorkingDirectory().getRelative(name); + try { + ProfileInfo info = ProfileInfo.loadProfileVerbosely( + profileFile, getInfoListener(runtime)); + if (opts.dumpMode != null) { + dumpProfile(runtime, info, out, opts.dumpMode); + } else if (opts.html) { + createHtml(runtime, info, profileFile, opts); + } else { + createText(runtime, info, out, opts); + } + } catch (IOException e) { + runtime.getReporter().handle(Event.error( + null, "Failed to process file " + name + ": " + e.getMessage())); + } + } + } finally { + out.flush(); + } + return ExitCode.SUCCESS; + } + + private void createText(BlazeRuntime runtime, ProfileInfo info, PrintStream out, + ProfileOptions opts) { + List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts); + + for (ProfilePhaseStatistics stat : statistics) { + String title = stat.getTitle(); + + if (!title.equals("")) { + out.println("\n=== " + title.toUpperCase() + " ===\n"); + } + out.print(stat.getStatistics()); + } + } + + private void createHtml(BlazeRuntime runtime, ProfileInfo info, Path profileFile, + ProfileOptions opts) + throws IOException { + Path htmlFile = + profileFile.getParentDirectory().getChild(profileFile.getBaseName() + ".html"); + List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts); + + runtime.getReporter().handle(Event.info("Creating HTML output in " + htmlFile)); + + ChartCreator chartCreator = + opts.htmlDetails ? new DetailedChartCreator(info, statistics) + : new AggregatingChartCreator(info, statistics); + Chart chart = chartCreator.create(); + OutputStream out = new BufferedOutputStream(htmlFile.getOutputStream()); + try { + chart.accept(new HtmlChartVisitor(new PrintStream(out), opts.htmlPixelsPerSecond)); + } finally { + try { + out.close(); + } catch (IOException e) { + // Ignore + } + } + } + + private List<ProfilePhaseStatistics> getStatistics( + BlazeRuntime runtime, ProfileInfo info, ProfileOptions opts) { + try { + ProfileInfo.aggregateProfile(info, getInfoListener(runtime)); + runtime.getReporter().handle(Event.info("Analyzing relationships")); + + info.analyzeRelationships(); + + List<ProfilePhaseStatistics> statistics = new ArrayList<>(); + + // Print phase durations and total execution time + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + long duration = 0; + for (ProfilePhase phase : ProfilePhase.values()) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + duration += info.getPhaseDuration(phaseTask); + } + } + for (ProfilePhase phase : ProfilePhase.values()) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + long phaseDuration = info.getPhaseDuration(phaseTask); + out.printf(THREE_COLUMN_FORMAT, "Total " + phase.nick + " phase time", + TimeUtilities.prettyTime(phaseDuration), prettyPercentage(phaseDuration, duration)); + } + } + out.printf(THREE_COLUMN_FORMAT, "Total run time", TimeUtilities.prettyTime(duration), + "100.00%"); + statistics.add(new ProfilePhaseStatistics("Phase Summary Information", + new String(byteOutput.toByteArray(), "UTF-8"))); + + // Print details of major phases + if (duration > 0) { + statistics.add(formatInitPhaseStatistics(info, opts)); + statistics.add(formatLoadingPhaseStatistics(info, opts)); + statistics.add(formatAnalysisPhaseStatistics(info, opts)); + ProfilePhaseStatistics stat = formatExecutionPhaseStatistics(info, opts); + if (stat != null) { + statistics.add(stat); + } + } + + return statistics; + } catch (UnsupportedEncodingException e) { + throw new AssertionError("Should not happen since, UTF8 is available on all JVMs"); + } + } + + private void dumpProfile( + BlazeRuntime runtime, ProfileInfo info, PrintStream out, String dumpMode) { + if (!dumpMode.contains("unsorted")) { + ProfileInfo.aggregateProfile(info, getInfoListener(runtime)); + } + if (dumpMode.contains("raw")) { + for (ProfileInfo.Task task : info.allTasksById) { + dumpRaw(task, out); + } + } else if (dumpMode.contains("unsorted")) { + for (ProfileInfo.Task task : info.allTasksById) { + dumpTask(task, out, 0); + } + } else { + for (ProfileInfo.Task task : info.rootTasksById) { + dumpTask(task, out, 0); + } + } + } + + private void dumpTask(ProfileInfo.Task task, PrintStream out, int indent) { + StringBuilder builder = new StringBuilder(String.format( + "\n%s %s\nThread: %-6d Id: %-6d Parent: %d\nStart time: %-12s Duration: %s", + task.type, task.getDescription(), task.threadId, task.id, task.parentId, + TimeUtilities.prettyTime(task.startTime), TimeUtilities.prettyTime(task.duration))); + if (task.hasStats()) { + builder.append("\n"); + ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray(); + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr attr = stats[type.ordinal()]; + if (attr != null) { + builder.append(type.toString().toLowerCase()).append("=("). + append(attr.count).append(", "). + append(TimeUtilities.prettyTime(attr.totalTime)).append(") "); + } + } + } + out.println(StringUtil.indent(builder.toString(), indent)); + for (ProfileInfo.Task subtask : task.subtasks) { + dumpTask(subtask, out, indent + 1); + } + } + + private void dumpRaw(ProfileInfo.Task task, PrintStream out) { + StringBuilder aggregateString = new StringBuilder(); + ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray(); + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr attr = stats[type.ordinal()]; + if (attr != null) { + aggregateString.append(type.toString().toLowerCase()).append(","). + append(attr.count).append(",").append(attr.totalTime).append(" "); + } + } + out.println( + task.threadId + "|" + task.id + "|" + task.parentId + "|" + + task.startTime + "|" + task.duration + "|" + + aggregateString.toString().trim() + "|" + + task.type + "|" + task.getDescription()); + } + + /** + * Converts relative duration to the percentage string + * @return formatted percentage string or "N/A" if result is undefined. + */ + private static String prettyPercentage(long duration, long total) { + if (total == 0) { + // Return "not available" string if total is 0 and result is undefined. + return "N/A"; + } + return String.format("%5.2f%%", duration*100.0/total); + } + + private void printCriticalPath(String title, PrintStream out, CriticalPathEntry path) { + out.println(String.format("\n%s (%s):", title, + TimeUtilities.prettyTime(path.cumulativeDuration))); + + boolean lightCriticalPath = isLightCriticalPath(path); + out.println(lightCriticalPath ? + String.format("%6s %11s %8s %s", "Id", "Time", "Percentage", "Description") + : String.format("%6s %11s %8s %8s %s", "Id", "Time", "Share", "Critical", "Description")); + + long totalPathTime = path.cumulativeDuration; + int middlemanCount = 0; + long middlemanDuration = 0L; + long middlemanCritTime = 0L; + + for (; path != null ; path = path.next) { + if (path.task.id < 0) { + // Ignore fake actions. + continue; + } else if (path.task.getDescription().startsWith(MiddlemanAction.MIDDLEMAN_MNEMONIC + " ") + || path.task.getDescription().startsWith("TargetCompletionMiddleman")) { + // Aggregate middleman actions. + middlemanCount++; + middlemanDuration += path.duration; + middlemanCritTime += path.getCriticalTime(); + } else { + String desc = path.task.getDescription().replace(':', ' '); + if (lightCriticalPath) { + out.println(String.format("%6d %11s %8s %s", path.task.id, + TimeUtilities.prettyTime(path.duration), + prettyPercentage(path.duration, totalPathTime), + desc)); + } else { + out.println(String.format("%6d %11s %8s %8s %s", path.task.id, + TimeUtilities.prettyTime(path.duration), + prettyPercentage(path.duration, totalPathTime), + prettyPercentage(path.getCriticalTime(), totalPathTime), desc)); + } + } + } + if (middlemanCount > 0) { + if (lightCriticalPath) { + out.println(String.format(" %11s %8s [%d middleman actions]", + TimeUtilities.prettyTime(middlemanDuration), + prettyPercentage(middlemanDuration, totalPathTime), + middlemanCount)); + } else { + out.println(String.format(" %11s %8s %8s [%d middleman actions]", + TimeUtilities.prettyTime(middlemanDuration), + prettyPercentage(middlemanDuration, totalPathTime), + prettyPercentage(middlemanCritTime, totalPathTime), middlemanCount)); + } + } + } + + private boolean isLightCriticalPath(CriticalPathEntry path) { + return path.task.type == ProfilerTask.CRITICAL_PATH_COMPONENT; + } + + private void printShortPhaseAnalysis(ProfileInfo info, PrintStream out, ProfilePhase phase) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + long phaseDuration = info.getPhaseDuration(phaseTask); + out.printf(TWO_COLUMN_FORMAT, "Total " + phase.nick + " phase time", + TimeUtilities.prettyTime(phaseDuration)); + printTimeDistributionByType(info, out, phaseTask); + } + } + + private void printTimeDistributionByType(ProfileInfo info, PrintStream out, + ProfileInfo.Task phaseTask) { + List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask); + long phaseDuration = info.getPhaseDuration(phaseTask); + long totalDuration = phaseDuration; + for (ProfileInfo.Task task : taskList) { + // Tasks on the phaseTask thread already accounted for in the phaseDuration. + if (task.threadId != phaseTask.threadId) { + totalDuration += task.duration; + } + } + boolean headerNeeded = true; + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr stats = info.getStatsForType(type, taskList); + if (stats.count > 0 && stats.totalTime > 0) { + if (headerNeeded) { + out.println("\nTotal time (across all threads) spent on:"); + out.println(String.format("%18s %8s %8s %11s", "Type", "Total", "Count", "Average")); + headerNeeded = false; + } + out.println(String.format("%18s %8s %8d %11s", type.toString(), + prettyPercentage(stats.totalTime, totalDuration), stats.count, + TimeUtilities.prettyTime(stats.totalTime / stats.count))); + } + } + } + + static class Stat implements Comparable<Stat> { + public long duration; + public long frequency; + + @Override + public int compareTo(Stat o) { + return this.duration == o.duration ? Long.compare(this.frequency, o.frequency) + : Long.compare(this.duration, o.duration); + } + } + + /** + * Print the time spent on VFS operations on each path. Output is grouped by operation and sorted + * by descending duration. If multiple of the same VFS operation were logged for the same path, + * print the total duration. + * + * @param info profiling data. + * @param out output stream. + * @param phase build phase. + * @param limit maximum number of statistics to print, or -1 for no limit. + */ + private void printVfsStatistics(ProfileInfo info, PrintStream out, + ProfilePhase phase, int limit) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask == null) { + return; + } + + if (limit == 0) { + return; + } + + // Group into VFS operations and build maps from path to duration. + + List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask); + EnumMap<ProfilerTask, Map<String, Stat>> stats = Maps.newEnumMap(ProfilerTask.class); + + collectVfsEntries(stats, taskList); + + if (!stats.isEmpty()) { + out.printf("\nVFS path statistics:\n"); + out.printf("%15s %10s %10s %s\n", "Type", "Frequency", "Duration", "Path"); + } + + // Reverse the maps to get maps from duration to path. We use a TreeMultimap to sort by duration + // and because durations are not unique. + + for (ProfilerTask type : stats.keySet()) { + Map<String, Stat> statsForType = stats.get(type); + TreeMultimap<Stat, String> sortedStats = + TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural()); + + for (Map.Entry<String, Stat> stat : statsForType.entrySet()) { + sortedStats.put(stat.getValue(), stat.getKey()); + } + + int numPrinted = 0; + for (Map.Entry<Stat, String> stat : sortedStats.entries()) { + if (limit != -1 && numPrinted++ == limit) { + out.printf("... %d more ...\n", sortedStats.size() - limit); + break; + } + out.printf("%15s %10d %10s %s\n", + type.name(), stat.getKey().frequency, TimeUtilities.prettyTime(stat.getKey().duration), + stat.getValue()); + } + } + } + + private void collectVfsEntries(EnumMap<ProfilerTask, Map<String, Stat>> stats, + List<ProfileInfo.Task> taskList) { + for (ProfileInfo.Task task : taskList) { + collectVfsEntries(stats, Arrays.asList(task.subtasks)); + if (!task.type.name().startsWith("VFS_")) { + continue; + } + + Map<String, Stat> statsForType = stats.get(task.type); + if (statsForType == null) { + statsForType = Maps.newHashMap(); + stats.put(task.type, statsForType); + } + + String path = currentPathMapping.apply(task.getDescription()); + + Stat stat = statsForType.get(path); + if (stat == null) { + stat = new Stat(); + } + + stat.duration += task.duration; + stat.frequency++; + statsForType.put(path, stat); + } + } + + /** + * Returns set of profiler tasks to be filtered from critical path. + * Also always filters out ACTION_LOCK and WAIT tasks to simulate + * unlimited resource critical path (see comments inside formatExecutionPhaseStatistics() + * method). + */ + private EnumSet<ProfilerTask> getTypeFilter(ProfilerTask... tasks) { + EnumSet<ProfilerTask> filter = EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT); + for (ProfilerTask task : tasks) { + filter.add(task); + } + return filter; + } + + private ProfilePhaseStatistics formatInitPhaseStatistics(ProfileInfo info, ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Init", ProfilePhase.INIT); + } + + private ProfilePhaseStatistics formatLoadingPhaseStatistics(ProfileInfo info, ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Loading", ProfilePhase.LOAD); + } + + private ProfilePhaseStatistics formatAnalysisPhaseStatistics(ProfileInfo info, + ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Analysis", ProfilePhase.ANALYZE); + } + + private ProfilePhaseStatistics formatSimplePhaseStatistics(ProfileInfo info, + ProfileOptions opts, + String name, + ProfilePhase phase) + throws UnsupportedEncodingException { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + + printShortPhaseAnalysis(info, out, phase); + printVfsStatistics(info, out, phase, opts.vfsStatsLimit); + return new ProfilePhaseStatistics(name + " Phase Information", + new String(byteOutput.toByteArray(), "UTF-8")); + } + + private ProfilePhaseStatistics formatExecutionPhaseStatistics(ProfileInfo info, + ProfileOptions opts) + throws UnsupportedEncodingException { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + + ProfileInfo.Task prepPhase = info.getPhaseTask(ProfilePhase.PREPARE); + ProfileInfo.Task execPhase = info.getPhaseTask(ProfilePhase.EXECUTE); + ProfileInfo.Task finishPhase = info.getPhaseTask(ProfilePhase.FINISH); + if (execPhase == null) { + return null; + } + + List<ProfileInfo.Task> execTasks = info.getTasksForPhase(execPhase); + long graphTime = info.getStatsForType(ProfilerTask.ACTION_GRAPH, execTasks).totalTime; + long execTime = info.getPhaseDuration(execPhase) - graphTime; + + if (prepPhase != null) { + out.printf(TWO_COLUMN_FORMAT, "Total preparation time", + TimeUtilities.prettyTime(info.getPhaseDuration(prepPhase))); + } + out.printf(TWO_COLUMN_FORMAT, "Total execution phase time", + TimeUtilities.prettyTime(info.getPhaseDuration(execPhase))); + if (finishPhase != null) { + out.printf(TWO_COLUMN_FORMAT, "Total time finalizing build", + TimeUtilities.prettyTime(info.getPhaseDuration(finishPhase))); + } + out.println(""); + out.printf(TWO_COLUMN_FORMAT, "Action dependency map creation", + TimeUtilities.prettyTime(graphTime)); + out.printf(TWO_COLUMN_FORMAT, "Actual execution time", + TimeUtilities.prettyTime(execTime)); + + EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class); + CriticalPathEntry totalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, totalPath); + + typeFilter = getTypeFilter(); + CriticalPathEntry optimalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, optimalPath); + + if (totalPath != null) { + printCriticalPathTimingBreakdown(info, totalPath, optimalPath, execTime, out); + } else { + out.println("\nCritical path not available because no action graph was generated."); + } + + printTimeDistributionByType(info, out, execPhase); + + if (totalPath != null) { + printCriticalPath("Critical path", out, totalPath); + // In light critical path we do not record scheduling delay data so it does not make sense + // to differentiate it. + if (!isLightCriticalPath(totalPath)) { + printCriticalPath("Critical path excluding scheduling delays", out, optimalPath); + } + } + + if (info.getMissingActionsCount() > 0) { + out.println("\n" + info.getMissingActionsCount() + " action(s) are present in the" + + " action graph but missing instrumentation data. Most likely profile file" + + " has been created for the failed or aborted build."); + } + + printVfsStatistics(info, out, ProfilePhase.EXECUTE, opts.vfsStatsLimit); + + return new ProfilePhaseStatistics("Execution Phase Information", + new String(byteOutput.toByteArray(), "UTF-8")); + } + + void printCriticalPathTimingBreakdown(ProfileInfo info, CriticalPathEntry totalPath, + CriticalPathEntry optimalPath, long execTime, PrintStream out) { + Preconditions.checkNotNull(totalPath); + Preconditions.checkNotNull(optimalPath); + // TODO(bazel-team): Print remote vs build stats recorded by CriticalPathStats + if (isLightCriticalPath(totalPath)) { + return; + } + out.println(totalPath.task.type); + // Worker thread pool scheduling delays for the actual critical path. + long workerWaitTime = 0; + long mainThreadWaitTime = 0; + for (ProfileInfo.CriticalPathEntry entry = totalPath; entry != null; entry = entry.next) { + workerWaitTime += info.getActionWaitTime(entry.task); + mainThreadWaitTime += info.getActionQueueTime(entry.task); + } + out.printf(TWO_COLUMN_FORMAT, "Worker thread scheduling delays", + TimeUtilities.prettyTime(workerWaitTime)); + out.printf(TWO_COLUMN_FORMAT, "Main thread scheduling delays", + TimeUtilities.prettyTime(mainThreadWaitTime)); + + out.println("\nCritical path time:"); + // Actual critical path. + long totalTime = totalPath.cumulativeDuration; + out.printf("%-37s %10s (%s of execution time)\n", "Actual time", + TimeUtilities.prettyTime(totalTime), + prettyPercentage(totalTime, execTime)); + // Unlimited resource critical path. Essentially, we assume that if we + // remove all scheduling delays caused by resource semaphore contention, + // each action execution time would not change (even though load now would + // be substantially higher - so this assumption might be incorrect but it is + // still useful for modeling). Given those assumptions we calculate critical + // path excluding scheduling delays. + long optimalTime = optimalPath.cumulativeDuration; + out.printf("%-37s %10s (%s of execution time)\n", "Time excluding scheduling delays", + TimeUtilities.prettyTime(optimalTime), + prettyPercentage(optimalTime, execTime)); + + // Artificial critical path if we ignore all the time spent in all tasks, + // except time directly attributed to the ACTION tasks. + out.println("\nTime related to:"); + + EnumSet<ProfilerTask> typeFilter = EnumSet.allOf(ProfilerTask.class); + ProfileInfo.CriticalPathEntry path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the builder overhead", + prettyPercentage(path.cumulativeDuration, totalTime)); + + typeFilter = getTypeFilter(); + for (ProfilerTask task : ProfilerTask.values()) { + if (task.name().startsWith("VFS_")) { + typeFilter.add(task); + } + } + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the VFS calls", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.ACTION_CHECK); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the dependency checking", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.ACTION_EXECUTE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the execution setup", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.SPAWN, ProfilerTask.LOCAL_EXECUTION); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "local execution", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.SCANNER); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the include scanner", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION, ProfilerTask.PROCESS_TIME, + ProfilerTask.LOCAL_PARSE, ProfilerTask.UPLOAD_TIME, + ProfilerTask.REMOTE_QUEUE, ProfilerTask.REMOTE_SETUP, ProfilerTask.FETCH); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "Remote execution (cumulative)", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter( ProfilerTask.UPLOAD_TIME, ProfilerTask.REMOTE_SETUP); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " file uploads", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.FETCH); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " file fetching", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.PROCESS_TIME); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " process time", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_QUEUE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " remote queueing", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.LOCAL_PARSE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " remote execution parse", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " other remote activities", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java new file mode 100644 index 0000000000..2e5faf6ca8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java @@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.CommonCommandOptions; +import com.google.devtools.build.lib.runtime.ProjectFile; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.List; + +/** + * Provides support for implementations for {@link BlazeCommand} to work with {@link ProjectFile}. + */ +public final class ProjectFileSupport { + static final String PROJECT_FILE_PREFIX = "+"; + + private ProjectFileSupport() {} + + /** + * Reads any project files specified on the command line and updates the options parser + * accordingly. If project files cannot be read or if they contain unparsable options, or if they + * are not enabled, then it throws an exception instead. + */ + public static void handleProjectFiles(BlazeRuntime runtime, OptionsParser optionsParser, + String command) throws AbruptExitException { + List<String> targets = optionsParser.getResidue(); + ProjectFile.Provider projectFileProvider = runtime.getProjectFileProvider(); + if (projectFileProvider != null && targets.size() > 0 + && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) { + if (targets.size() > 1) { + throw new AbruptExitException("Cannot handle more than one +<file> argument yet", + ExitCode.COMMAND_LINE_ERROR); + } + if (!optionsParser.getOptions(CommonCommandOptions.class).allowProjectFiles) { + throw new AbruptExitException("project file support is not enabled", + ExitCode.COMMAND_LINE_ERROR); + } + // TODO(bazel-team): This is currently treated as a path relative to the workspace - if the + // cwd is a subdirectory of the workspace, that will be surprising, and we should interpret it + // relative to the cwd instead. + PathFragment projectFilePath = new PathFragment(targets.get(0).substring(1)); + List<Path> packagePath = PathPackageLocator.create( + optionsParser.getOptions(PackageCacheOptions.class).packagePath, runtime.getReporter(), + runtime.getWorkspace(), runtime.getWorkingDirectory()).getPathEntries(); + ProjectFile projectFile = projectFileProvider.getProjectFile(packagePath, projectFilePath); + runtime.getReporter().handle(Event.info("Using " + projectFile.getName())); + + try { + optionsParser.parse( + OptionPriority.RC_FILE, projectFile.getName(), projectFile.getCommandLineFor(command)); + } catch (OptionsParsingException e) { + throw new AbruptExitException(e.getMessage(), ExitCode.COMMAND_LINE_ERROR); + } + } + } + + /** + * Returns a list of targets from the options residue. If a project file is supplied as the first + * argument, it will be ignored, on the assumption that handleProjectFiles() has been called to + * process it. + */ + public static List<String> getTargets(BlazeRuntime runtime, OptionsProvider options) { + List<String> targets = options.getResidue(); + if (runtime.getProjectFileProvider() != null && targets.size() > 0 + && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) { + return targets.subList(1, targets.size()); + } + return targets; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java new file mode 100644 index 0000000000..c5120cb74b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java @@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.query2.BlazeQueryEnvironment; +import com.google.devtools.build.lib.query2.SkyframeQueryEnvironment; +import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter; +import com.google.devtools.build.lib.query2.output.QueryOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.channels.ClosedByInterruptException; +import java.util.Set; + +/** + * Command line wrapper for executing a query with blaze. + */ +@Command(name = "query", + options = { PackageCacheOptions.class, + QueryOptions.class }, + help = "resource:query.txt", + shortDescription = "Executes a dependency graph query.", + allowResidue = true, + binaryStdOut = true, + canRunInOutputDirectory = true) +public final class QueryCommand implements BlazeCommand { + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + /** + * Exit codes: + * 0 on successful evaluation. + * 1 if query evaluation did not complete. + * 2 if query parsing failed. + * 3 if errors were reported but evaluation produced a partial result + * (only when --keep_going is in effect.) + */ + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + QueryOptions queryOptions = options.getOptions(QueryOptions.class); + + try { + runtime.setupPackageCache( + options.getOptions(PackageCacheOptions.class), + runtime.getDefaultsPackageContent()); + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("query interrupted")); + return ExitCode.INTERRUPTED; + } catch (AbruptExitException e) { + runtime.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage())); + return e.getExitCode(); + } + + if (options.getResidue().isEmpty()) { + runtime.getReporter().handle(Event.error( + "missing query expression. Type 'blaze help query' for syntax and help")); + return ExitCode.COMMAND_LINE_ERROR; + } + + Iterable<OutputFormatter> formatters = runtime.getQueryOutputFormatters(); + OutputFormatter formatter = + OutputFormatter.getFormatter(formatters, queryOptions.outputFormat); + if (formatter == null) { + runtime.getReporter().handle(Event.error( + String.format("Invalid output format '%s'. Valid values are: %s", + queryOptions.outputFormat, OutputFormatter.formatterNames(formatters)))); + return ExitCode.COMMAND_LINE_ERROR; + } + + String query = Joiner.on(' ').join(options.getResidue()); + + Set<Setting> settings = queryOptions.toSettings(); + BlazeQueryEnvironment env = newQueryEnvironment( + runtime, + queryOptions.keepGoing, + queryOptions.loadingPhaseThreads, + settings); + + // 1. Parse query: + QueryExpression expr; + try { + expr = QueryExpression.parse(query, env); + } catch (QueryException e) { + runtime.getReporter().handle(Event.error( + null, "Error while parsing '" + query + "': " + e.getMessage())); + return ExitCode.COMMAND_LINE_ERROR; + } + + // 2. Evaluate expression: + BlazeQueryEvalResult<Target> result; + try { + result = env.evaluateQuery(expr); + } catch (QueryException e) { + // Keep consistent with reportBuildFileError() + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.ANALYSIS_FAILURE; + } + + // 3. Output results: + OutputFormatter.UnorderedFormatter unorderedFormatter = null; + if (!queryOptions.orderResults && formatter instanceof UnorderedFormatter) { + unorderedFormatter = (UnorderedFormatter) formatter; + } + + PrintStream output = new PrintStream(runtime.getReporter().getOutErr().getOutputStream()); + try { + if (unorderedFormatter != null) { + unorderedFormatter.outputUnordered(queryOptions, result.getResultSet(), output); + } else { + formatter.output(queryOptions, result.getResultGraph(), output); + } + } catch (ClosedByInterruptException e) { + runtime.getReporter().handle(Event.error("query interrupted")); + return ExitCode.INTERRUPTED; + } catch (IOException e) { + runtime.getReporter().handle(Event.error("I/O error: " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } finally { + output.flush(); + } + if (result.getResultSet().isEmpty()) { + runtime.getReporter().handle(Event.info("Empty results")); + } + + return result.getSuccess() ? ExitCode.SUCCESS : ExitCode.PARTIAL_ANALYSIS_FAILURE; + } + + @VisibleForTesting // for com.google.devtools.deps.gquery.test.QueryResultTestUtil + public static BlazeQueryEnvironment newQueryEnvironment(BlazeRuntime runtime, + boolean keepGoing, int loadingPhaseThreads, Set<Setting> settings) { + ImmutableList.Builder<QueryFunction> functions = ImmutableList.builder(); + for (BlazeModule module : runtime.getBlazeModules()) { + functions.addAll(module.getQueryFunctions()); + } + return new SkyframeQueryEnvironment( + runtime.getPackageManager().newTransitiveLoader(), + runtime.getPackageManager(), + runtime.getTargetPatternEvaluator(), + keepGoing, loadingPhaseThreads, runtime.getReporter(), settings, functions.build()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java new file mode 100644 index 0000000000..b128d372ab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java @@ -0,0 +1,519 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.BuildResult; +import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.buildtool.TargetValidator; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.SymlinkTreeHelper; +import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.shell.AbnormalTerminationException; +import com.google.devtools.build.lib.shell.BadExitStatusException; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.CommandDescriptionForm; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Builds and run a target with the given command line arguments. + */ +@Command(name = "run", + builds = true, + options = { RunCommand.RunOptions.class }, + inherits = { BuildCommand.class }, + shortDescription = "Runs the specified target.", + help = "resource:run.txt", + allowResidue = true, + binaryStdOut = true, + binaryStdErr = true) +public class RunCommand implements BlazeCommand { + + public static class RunOptions extends OptionsBase { + @Option(name = "script_path", + category = "run", + defaultValue = "null", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, write a shell script to the given file which invokes the " + + "target. If this option is set, the target is not run from Blaze. " + + "Use 'blaze run --script_path=foo //foo && foo' to invoke target '//foo' " + + "This differs from 'blaze run //foo' in that the Blaze lock is released " + + "and the executable is connected to the terminal's stdin.") + public PathFragment scriptPath; + } + + @VisibleForTesting + public static final String SINGLE_TARGET_MESSAGE = "Blaze can only run a single target. " + + "Do not use wildcards that match more than one target"; + @VisibleForTesting + public static final String NO_TARGET_MESSAGE = "No targets found to run"; + + private static final String PROCESS_WRAPPER = "process-wrapper"; + + // Value of --run_under as of the most recent command invocation. + private RunUnder currentRunUnder; + + private static final FileType RUNFILES_MANIFEST = FileType.of(".runfiles_manifest"); + + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + protected BuildResult processRequest(final BlazeRuntime runtime, BuildRequest request) { + return runtime.getBuildTool().processRequest(request, new TargetValidator() { + @Override + public void validateTargets(Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException { + RunCommand.this.validateTargets(runtime.getReporter(), targets, keepGoing); + } + }); + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + RunOptions runOptions = options.getOptions(RunOptions.class); + // This list should look like: ["//executable:target", "arg1", "arg2"] + List<String> targetAndArgs = options.getResidue(); + + // The user must at the least specify an executable target. + if (targetAndArgs.isEmpty()) { + runtime.getReporter().handle(Event.error("Must specify a target to run")); + return ExitCode.COMMAND_LINE_ERROR; + } + String targetString = targetAndArgs.get(0); + List<String> runTargetArgs = targetAndArgs.subList(1, targetAndArgs.size()); + RunUnder runUnder = options.getOptions(BuildConfiguration.Options.class).runUnder; + + OutErr outErr = runtime.getReporter().getOutErr(); + List<String> targets = (runUnder != null) && (runUnder.getLabel() != null) + ? ImmutableList.of(targetString, runUnder.getLabel().toString()) + : ImmutableList.of(targetString); + BuildRequest request = BuildRequest.create( + this.getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), targets, outErr, + runtime.getCommandId(), runtime.getCommandStartTime()); + if (request.getBuildOptions().compileOnly) { + String message = "The '" + getClass().getAnnotation(Command.class).name() + + "' command is incompatible with the --compile_only option"; + runtime.getReporter().handle(Event.error(message)); + return ExitCode.COMMAND_LINE_ERROR; + } + + currentRunUnder = runUnder; + BuildResult result; + try { + result = processRequest(runtime, request); + } finally { + currentRunUnder = null; + } + + if (!result.getSuccess()) { + runtime.getReporter().handle(Event.error("Build failed. Not running target")); + return result.getExitCondition(); + } + + // Make sure that we have exactly 1 built target (excluding --run_under), + // and that it is executable. + // These checks should only fail if keepGoing is true, because we already did + // validation before the build began. See {@link #validateTargets()}. + Collection<ConfiguredTarget> targetsBuilt = result.getSuccessfulTargets(); + ConfiguredTarget targetToRun = null; + ConfiguredTarget runUnderTarget = null; + + if (targetsBuilt != null) { + int maxTargets = runUnder != null && runUnder.getLabel() != null ? 2 : 1; + if (targetsBuilt.size() > maxTargets) { + runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + for (ConfiguredTarget target : targetsBuilt) { + ExitCode targetValidation = fullyValidateTarget(runtime, target); + if (targetValidation != ExitCode.SUCCESS) { + return targetValidation; + } + if (runUnder != null && target.getLabel().equals(runUnder.getLabel())) { + if (runUnderTarget != null) { + runtime.getReporter().handle(Event.error( + null, "Can't identify the run_under target from multiple options?")); + return ExitCode.COMMAND_LINE_ERROR; + } + runUnderTarget = target; + } else if (targetToRun == null) { + targetToRun = target; + } else { + runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + } + } + // Handle target & run_under referring to the same target. + if ((targetToRun == null) && (runUnderTarget != null)) { + targetToRun = runUnderTarget; + } + if (targetToRun == null) { + runtime.getReporter().handle(Event.error(NO_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + + Path executablePath = Preconditions.checkNotNull( + targetToRun.getProvider(FilesToRunProvider.class).getExecutable().getPath()); + BuildConfiguration configuration = targetToRun.getConfiguration(); + if (configuration == null) { + // The target may be an input file, which doesn't have a configuration. In that case, we + // choose any target configuration. + configuration = runtime.getBuildTool().getView().getConfigurationCollection() + .getTargetConfigurations().get(0); + } + Path workingDir; + try { + workingDir = ensureRunfilesBuilt(runtime, targetToRun); + } catch (CommandException e) { + runtime.getReporter().handle(Event.error("Error creating runfiles: " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + + List<String> args = runTargetArgs; + + FilesToRunProvider provider = targetToRun.getProvider(FilesToRunProvider.class); + RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport(); + if (runfilesSupport != null && runfilesSupport.getArgs() != null) { + List<String> targetArgs = runfilesSupport.getArgs(); + if (!targetArgs.isEmpty()) { + args = Lists.newArrayListWithCapacity(targetArgs.size() + runTargetArgs.size()); + args.addAll(targetArgs); + args.addAll(runTargetArgs); + } + } + + // + // We now have a unique executable ready to be run. + // + // We build up two different versions of the command to run: one with an absolute path, which + // we'll actually run, and a prettier one with the long absolute path to the executable + // replaced with a shorter relative path that uses the symlinks in the workspace. + PathFragment prettyExecutablePath = + OutputDirectoryLinksUtils.getPrettyPath(executablePath, + runtime.getWorkspaceName(), runtime.getWorkspace(), + options.getOptions(BuildRequestOptions.class).symlinkPrefix); + List<String> cmdLine = new ArrayList<>(); + if (runOptions.scriptPath == null) { + cmdLine.add(runtime.getDirectories().getExecRoot() + .getRelative(runtime.getBinTools().getExecPath(PROCESS_WRAPPER)).getPathString()); + cmdLine.add("-1"); + cmdLine.add("15"); + cmdLine.add("-"); + cmdLine.add("-"); + } + List<String> prettyCmdLine = new ArrayList<>(); + // Insert the command prefix specified by the "--run_under=<command-prefix>" option + // at the start of the command line. + if (runUnder != null) { + String runUnderValue = runUnder.getValue(); + if (runUnderTarget != null) { + // --run_under specifies a target. Get the corresponding executable. + // This must be an absolute path, because the run_under target is only + // in the runfiles of test targets. + runUnderValue = runUnderTarget + .getProvider(FilesToRunProvider.class).getExecutable().getPath().getPathString(); + // If the run_under command contains any options, make sure to add them + // to the command line as well. + List<String> opts = runUnder.getOptions(); + if (!opts.isEmpty()) { + runUnderValue += " " + ShellEscaper.escapeJoinAll(opts); + } + } + cmdLine.add(configuration.getShExecutable().getPathString()); + cmdLine.add("-c"); + cmdLine.add(runUnderValue + " " + executablePath.getPathString() + " " + + ShellEscaper.escapeJoinAll(args)); + prettyCmdLine.add(configuration.getShExecutable().getPathString()); + prettyCmdLine.add("-c"); + prettyCmdLine.add(runUnderValue + " " + prettyExecutablePath.getPathString() + " " + + ShellEscaper.escapeJoinAll(args)); + } else { + cmdLine.add(executablePath.getPathString()); + cmdLine.addAll(args); + prettyCmdLine.add(prettyExecutablePath.getPathString()); + prettyCmdLine.addAll(args); + } + + // Add a newline between the blaze output and the binary's output. + outErr.printErrLn(""); + + if (runOptions.scriptPath != null) { + String unisolatedCommand = CommandFailureUtils.describeCommand( + CommandDescriptionForm.COMPLETE_UNISOLATED, + cmdLine, null, workingDir.getPathString()); + if (writeScript(runtime, runOptions.scriptPath, unisolatedCommand)) { + return ExitCode.SUCCESS; + } else { + return ExitCode.RUN_FAILURE; + } + } + + runtime.getReporter().handle(Event.info( + null, "Running command line: " + ShellEscaper.escapeJoinAll(prettyCmdLine))); + + com.google.devtools.build.lib.shell.Command command = new CommandBuilder() + .addArgs(cmdLine).setEnv(runtime.getClientEnv()).setWorkingDir(workingDir).build(); + + try { + // The command API is a little strange in that the following statement + // will return normally only if the program exits with exit code 0. + // If it ends with any other code, we have to catch BadExitStatusException. + command.execute(com.google.devtools.build.lib.shell.Command.NO_INPUT, + com.google.devtools.build.lib.shell.Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + true /* interruptible */).getTerminationStatus().getExitCode(); + return ExitCode.SUCCESS; + } catch (BadExitStatusException e) { + String message = "Non-zero return code '" + + e.getResult().getTerminationStatus().getExitCode() + + "' from command: " + e.getMessage(); + runtime.getReporter().handle(Event.error(message)); + return ExitCode.RUN_FAILURE; + } catch (AbnormalTerminationException e) { + // The process was likely terminated by a signal in this case. + return ExitCode.INTERRUPTED; + } catch (CommandException e) { + runtime.getReporter().handle(Event.error("Error running program: " + e.getMessage())); + return ExitCode.RUN_FAILURE; + } + } + + /** + * Ensures that runfiles are built for the specified target. If they already + * are, does nothing, otherwise builds them. + * + * @param target the target to build runfiles for. + * @return the path of the runfiles directory. + * @throws CommandException + */ + private Path ensureRunfilesBuilt(BlazeRuntime runtime, ConfiguredTarget target) + throws CommandException { + FilesToRunProvider provider = target.getProvider(FilesToRunProvider.class); + RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport(); + if (runfilesSupport == null) { + return runtime.getWorkingDirectory(); + } + + Artifact manifest = runfilesSupport.getRunfilesManifest(); + PathFragment runfilesDir = runfilesSupport.getRunfilesDirectoryExecPath(); + Path workingDir = runtime.getExecRoot() + .getRelative(runfilesDir) + .getRelative(runtime.getRunfilesPrefix()); + + // When runfiles are not generated, getManifest() returns the + // .runfiles_manifest file, otherwise it returns the MANIFEST file. This is + // a handy way to check whether runfiles were built or not. + if (!RUNFILES_MANIFEST.matches(manifest.getFilename())) { + // Runfiles already built, nothing to do. + return workingDir; + } + + SymlinkTreeHelper helper = new SymlinkTreeHelper( + manifest.getExecPath(), + runfilesDir, + false); + helper.createSymlinksUsingCommand(runtime.getExecRoot(), target.getConfiguration(), + runtime.getBinTools()); + return workingDir; + } + + private boolean writeScript(BlazeRuntime runtime, PathFragment scriptPathFrag, String cmd) { + final String SH_SHEBANG = "#!/bin/sh"; + Path scriptPath = runtime.getWorkingDirectory().getRelative(scriptPathFrag); + try { + FileSystemUtils.writeContent(scriptPath, StandardCharsets.ISO_8859_1, + SH_SHEBANG + "\n" + cmd + " \"$@\""); + scriptPath.setExecutable(true); + } catch (IOException e) { + runtime.getReporter().handle(Event.error("Error writing run script:" + e.getMessage())); + return false; + } + return true; + } + + // Make sure we are building exactly 1 binary target. + // If keepGoing, we'll build all the targets even if they are non-binary. + private void validateTargets(Reporter reporter, Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException { + Target targetToRun = null; + Target runUnderTarget = null; + + boolean singleTargetWarningWasOutput = false; + int maxTargets = currentRunUnder != null && currentRunUnder.getLabel() != null ? 2 : 1; + if (targets.size() > maxTargets) { + warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing); + singleTargetWarningWasOutput = true; + } + for (Target target : targets) { + String targetError = validateTarget(target); + if (targetError != null) { + warningOrException(reporter, targetError, keepGoing); + } + + if (currentRunUnder != null && target.getLabel().equals(currentRunUnder.getLabel())) { + // It's impossible to have two targets with the same label. + Preconditions.checkState(runUnderTarget == null); + runUnderTarget = target; + } else if (targetToRun == null) { + targetToRun = target; + } else { + if (!singleTargetWarningWasOutput) { + warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing); + } + return; + } + } + // Handle target & run_under referring to the same target. + if ((targetToRun == null) && (runUnderTarget != null)) { + targetToRun = runUnderTarget; + } + if (targetToRun == null) { + warningOrException(reporter, NO_TARGET_MESSAGE, keepGoing); + } + } + + // If keepGoing, print a warning and return the given collection. + // Otherwise, throw InvalidTargetException. + private void warningOrException(Reporter reporter, String message, + boolean keepGoing) throws LoadingFailedException { + if (keepGoing) { + reporter.handle(Event.warn(message + ". Will continue anyway")); + } else { + throw new LoadingFailedException(message); + } + } + + private static String notExecutableError(Target target) { + return "Cannot run target " + target.getLabel() + ": Not executable"; + } + + /** Returns null if the target is a runnable rule, or an appropriate error message otherwise. */ + private static String validateTarget(Target target) { + return isExecutable(target) + ? null + : notExecutableError(target); + } + + /** + * Performs all available validation checks on an individual target. + * + * @param target ConfiguredTarget to validate + * @return ExitCode.SUCCESS if all checks succeeded, otherwise a different error code. + */ + private ExitCode fullyValidateTarget(BlazeRuntime runtime, ConfiguredTarget target) { + String targetError = validateTarget(target.getTarget()); + + if (targetError != null) { + runtime.getReporter().handle(Event.error(targetError)); + return ExitCode.COMMAND_LINE_ERROR; + } + + Artifact executable = target.getProvider(FilesToRunProvider.class).getExecutable(); + if (executable == null) { + runtime.getReporter().handle(Event.error(notExecutableError(target.getTarget()))); + return ExitCode.COMMAND_LINE_ERROR; + } + + // Shouldn't happen: We just validated the target. + Preconditions.checkState(executable != null, + "Could not find executable for target %s", target); + Path executablePath = executable.getPath(); + try { + if (!executablePath.exists() || !executablePath.isExecutable()) { + runtime.getReporter().handle(Event.error( + null, "Non-existent or non-executable " + executablePath)); + return ExitCode.BLAZE_INTERNAL_ERROR; + } + } catch (IOException e) { + runtime.getReporter().handle(Event.error( + "Error checking " + executablePath.getPathString() + ": " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + + return ExitCode.SUCCESS; + } + + /** + * Return true iff {@code target} is a rule that has an executable file. This includes + * *_test rules, *_binary rules, and generated outputs. + */ + private static boolean isExecutable(Target target) { + return isOutputFile(target) || isExecutableNonTestRule(target) + || TargetUtils.isTestRule(target); + } + + /** + * Return true iff {@code target} is a rule that generates an executable file and is user-executed + * code. + */ + private static boolean isExecutableNonTestRule(Target target) { + if (!(target instanceof Rule)) { + return false; + } + Rule rule = ((Rule) target); + if (rule.getRuleClassObject().hasAttr("$is_executable", Type.BOOLEAN)) { + return NonconfigurableAttributeMapper.of(rule).get("$is_executable", Type.BOOLEAN); + } + return false; + } + + private static boolean isOutputFile(Target target) { + return (target instanceof OutputFile); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java new file mode 100644 index 0000000000..fb9ba39f37 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * The 'blaze shutdown' command. + */ +@Command(name = "shutdown", + options = { ShutdownCommand.Options.class }, + allowResidue = false, + mustRunInWorkspace = false, + shortDescription = "Stops the Blaze server.", + help = "This command shuts down the memory resident Blaze server process.\n%{options}") +public final class ShutdownCommand implements BlazeCommand { + + public static class Options extends OptionsBase { + + @Option(name="iff_heap_size_greater_than", + defaultValue = "0", + category = "misc", + help="Iff non-zero, then shutdown will only shut down the " + + "server if the total memory (in MB) consumed by the JVM " + + "exceeds this value.") + public int heapSizeLimit; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + + int limit = options.getOptions(Options.class).heapSizeLimit; + + // Iff limit is non-zero, shut down the server if total memory exceeds the + // limit. totalMemory is the actual heap size that the VM currently uses + // *from the OS perspective*. That is, it's not the size occupied by all + // objects (which is totalMemory() - freeMemory()), and not the -Xmx + // (which is maxMemory()). It's really how much memory this process + // currently consumes, in addition to the JVM code and C heap. + + if (limit == 0 || + Runtime.getRuntime().totalMemory() > limit * 1000L * 1000) { + throw new ShutdownBlazeServerException(0); + } + return ExitCode.SUCCESS; + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java new file mode 100644 index 0000000000..70082ef8dd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java @@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.docgen.SkylarkDocumentationProcessor; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Map; + +/** + * The 'doc_ext' command, which prints the extension API doc. + */ +@Command(name = "doc_ext", +allowResidue = true, +mustRunInWorkspace = false, +shortDescription = "Prints help for commands, or the index.", +help = "resource:skylark.txt") +public final class SkylarkCommand implements BlazeCommand { + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + OutErr outErr = runtime.getReporter().getOutErr(); + if (options.getResidue().isEmpty()) { + printTopLevelAPIDoc(outErr); + return ExitCode.SUCCESS; + } + if (options.getResidue().size() != 1) { + runtime.getReporter().handle(Event.error("Cannot specify more than one parameters")); + return ExitCode.COMMAND_LINE_ERROR; + } + return printAPIDoc(options.getResidue().get(0), outErr, runtime.getReporter()); + } + + private ExitCode printAPIDoc(String param, OutErr outErr, Reporter reporter) { + String params[] = param.split("\\."); + if (params.length > 2) { + reporter.handle(Event.error("Identifier not found: " + param)); + return ExitCode.COMMAND_LINE_ERROR; + } + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + String doc = processor.getCommandLineAPIDoc(params); + if (doc == null) { + reporter.handle(Event.error("Identifier not found: " + param)); + return ExitCode.COMMAND_LINE_ERROR; + } + outErr.printOut(doc); + return ExitCode.SUCCESS; + } + + private void printTopLevelAPIDoc(OutErr outErr) { + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + outErr.printOut("Top level language modules, methods and objects:\n\n"); + for (Map.Entry<String, String> entry : processor.collectTopLevelModules().entrySet()) { + outErr.printOut(entry.getKey() + ": " + entry.getValue()); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java new file mode 100644 index 0000000000..561c54a5b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java @@ -0,0 +1,161 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildResult; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.rules.test.TestStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.runtime.AggregatingTestListener; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions; +import com.google.devtools.build.lib.runtime.TestResultAnalyzer; +import com.google.devtools.build.lib.runtime.TestResultNotifier; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Collection; +import java.util.List; + +/** + * Handles the 'test' command on the Blaze command line. + */ +@Command(name = "test", + builds = true, + inherits = { BuildCommand.class }, + options = { TestSummaryOptions.class }, + shortDescription = "Builds and runs the specified test targets.", + help = "resource:test.txt", + allowResidue = true) +public class TestCommand implements BlazeCommand { + private AnsiTerminalPrinter printer; + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) + throws AbruptExitException { + ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "test"); + + TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput; + + if (testOutput == TestStrategy.TestOutputFormat.STREAMED) { + runtime.getReporter().handle(Event.warn( + "Streamed test output requested so all tests will be run locally, without sharding, " + + "one at a time")); + try { + optionsParser.parse(OptionPriority.SOFTWARE_REQUIREMENT, + "streamed output requires locally run tests, without sharding", + ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive")); + } catch (OptionsParsingException e) { + throw new IllegalStateException("Known options failed to parse", e); + } + } + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + TestResultAnalyzer resultAnalyzer = new TestResultAnalyzer( + runtime.getExecRoot(), + options.getOptions(TestSummaryOptions.class), + options.getOptions(ExecutionOptions.class), + runtime.getEventBus()); + + printer = new AnsiTerminalPrinter(runtime.getReporter().getOutErr().getOutputStream(), + options.getOptions(BlazeCommandEventHandler.Options.class).useColor()); + + // Initialize test handler. + AggregatingTestListener testListener = new AggregatingTestListener( + resultAnalyzer, runtime.getEventBus(), runtime.getReporter()); + + runtime.getEventBus().register(testListener); + return doTest(runtime, options, testListener); + } + + private ExitCode doTest(BlazeRuntime runtime, + OptionsProvider options, + AggregatingTestListener testListener) { + // Run simultaneous build and test. + List<String> targets = ProjectFileSupport.getTargets(runtime, options); + BuildRequest request = BuildRequest.create( + getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), targets, + runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime()); + if (request.getBuildOptions().compileOnly) { + String message = "The '" + getClass().getAnnotation(Command.class).name() + + "' command is incompatible with the --compile_only option"; + runtime.getReporter().handle(Event.error(message)); + return ExitCode.COMMAND_LINE_ERROR; + } + request.setRunTests(); + + BuildResult buildResult = runtime.getBuildTool().processRequest(request, null); + + Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets(); + // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests + if (buildResult.getSuccessfulTargets() == null) { + // This can happen if there were errors in the target parsing or loading phase + // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given + // (original exitcode=SUCCESS). + runtime.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests")); + return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition(); + } + // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks + // more tests + if (testTargets.isEmpty()) { + runtime.getReporter().handle(Event.error( + null, "No test targets were found, yet testing was requested")); + return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition(); + } + + boolean buildSuccess = buildResult.getSuccess(); + boolean testSuccess = analyzeTestResults(testTargets, testListener, options); + + if (testSuccess && !buildSuccess) { + // If all tests run successfully, test summary should include warning if + // there were build errors not associated with the test targets. + printer.printLn(AnsiTerminalPrinter.Mode.ERROR + + "One or more non-test targets failed to build.\n" + + AnsiTerminalPrinter.Mode.DEFAULT); + } + + return buildSuccess ? + (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED) + : buildResult.getExitCondition(); + } + + /** + * Analyzes test results and prints summary information. + * Returns true if and only if all tests were successful. + */ + private boolean analyzeTestResults(Collection<ConfiguredTarget> testTargets, + AggregatingTestListener listener, + OptionsProvider options) { + TestResultNotifier notifier = new TerminalTestResultNotifier(printer, options); + return listener.getAnalyzer().differentialAnalyzeAndReport( + testTargets, listener, notifier); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java new file mode 100644 index 0000000000..0804cf6cee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * The 'blaze version' command, which informs users about the blaze version + * information. + */ +@Command(name = "version", + options = {}, + allowResidue = false, + mustRunInWorkspace = false, + help = "resource:version.txt", + shortDescription = "Prints version information for Blaze.") +public final class VersionCommand implements BlazeCommand { + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + BlazeVersionInfo info = BlazeVersionInfo.instance(); + if (info.getSummary() == null) { + runtime.getReporter().handle(Event.error("Version information not available")); + return ExitCode.COMMAND_LINE_ERROR; + } + runtime.getReporter().getOutErr().printOutLn(info.getSummary()); + return ExitCode.SUCCESS; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt new file mode 100644 index 0000000000..0ef55a86f1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt @@ -0,0 +1,14 @@ + +Usage: blaze %{command} <options> <profile-files> [<profile-file> ...] + +Analyzes build profile data for the given profile data files. + +Analyzes each specified profile data file and prints the results. The +input files must have been produced by the 'blaze build +--profile=file' command. + +By default, a summary of the analysis is printed. For post-processing +with scripts, the --dump=raw option is recommended, causing this +command to dump profile data in easily-parsed format. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt new file mode 100644 index 0000000000..5e8d88ae9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt @@ -0,0 +1,10 @@ + +Usage: blaze %{command} <options> <targets> + +Builds the specified targets, using the options. + +See 'blaze help target-syntax' for details and examples on how to +specify targets to build. + +%{options} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt new file mode 100644 index 0000000000..11541ff354 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt @@ -0,0 +1,8 @@ + +Usage: blaze canonicalize-flags <options> -- <options-to-canonicalize> + +Canonicalizes Blaze flags for the test and build commands. This command is +intended to be used for tools that wish to check if two lists of options have +the same effect at runtime. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt new file mode 100644 index 0000000000..7633888ed6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt @@ -0,0 +1,10 @@ + +Usage: blaze %{command} [<option> ...] + +Removes Blaze-created output, including all object files, and Blaze +metadata. + +If '--expunge' is specified, the entire working tree will be removed +and the server stopped. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt new file mode 100644 index 0000000000..a2040c81ca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt @@ -0,0 +1,7 @@ + +Usage: blaze help [<command>] + +Prints a help page for the given command, or, if no command is +specified, prints the index of available commands. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt new file mode 100644 index 0000000000..9c8b552947 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt @@ -0,0 +1,23 @@ + +Usage: blaze info <options> [key] + +Displays information about the state of the blaze process in the +form of several "key: value" pairs. This includes the locations of +several output directories. Because some of the +values are affected by the options passed to 'blaze build', the +info command accepts the same set of options. + +A single non-option argument may be specified (e.g. "blaze-bin"), in +which case only the value for that key will be printed. + +If --show_make_env is specified, the output includes the set of key/value +pairs in the "Make" environment, accessible within BUILD files. + +The full list of keys and the meaning of their values is documented in +the Blaze User Manual, and can be programmatically obtained with +'blaze help info-keys'. + +See also 'blaze version' for more detailed blaze version +information. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt new file mode 100644 index 0000000000..ce10211cc9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt @@ -0,0 +1,19 @@ + +Usage: blaze %{command} <options> <query-expression> + +Executes a query language expression over a specified subgraph of the +build dependency graph. + +For example, to show all C++ test rules in the strings package, use: + + % blaze query 'kind("cc_.*test", strings:*)' + +or to find all dependencies of chubby lockserver, use: + + % blaze query 'deps(//path/to/package:target)' + +or to find a dependency path between //path/to/package:target and //dependency: + + % blaze query 'somepath(//path/to/package:target, //dependency)' + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt new file mode 100644 index 0000000000..57283d50d0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt @@ -0,0 +1,12 @@ + +Usage: blaze %{command} <options> -- <binary target> <flags to binary> + +Build the specified target and run it with the given arguments. + +'run' accepts any 'build' options, and will inherit any defaults +provided by .blazerc. + +If your script needs stdin or execution not constrained by the Blaze lock, +use 'blaze run --script_path' to write a script and then execute it. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt new file mode 100644 index 0000000000..5414707d0a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt @@ -0,0 +1,14 @@ + +Startup options +=============== + +These options affect how Blaze starts up, or more specifically, how +the virtual machine hosting Blaze starts up, and how the Blaze server +starts up. These options must be specified to the left of the Blaze +command (e.g. 'build'), and they must not contain any space between +option name and value. + +Example: + % blaze --host_jvm_args=-Xmx1400m --output_base=/tmp/foo build //base + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt new file mode 100644 index 0000000000..1fac4981db --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt @@ -0,0 +1,64 @@ + +Target pattern syntax +===================== + +The BUILD file label syntax is used to specify a single target. Target +patterns generalize this syntax to sets of targets, and also support +working-directory-relative forms, recursion, subtraction and filtering. +Examples: + +Specifying a single target: + + //foo/bar:wiz The single target '//foo/bar:wiz'. + foo/bar/wiz Equivalent to the first existing one of these: + //foo/bar:wiz + //foo:bar/wiz + //foo/bar Equivalent to '//foo/bar:bar'. + +Specifying all rules in a package: + + //foo/bar:all Matches all rules in package 'foo/bar'. + +Specifying all rules recursively beneath a package: + + //foo/...:all Matches all rules in all packages beneath directory 'foo'. + //foo/... (ditto) + +Working-directory relative forms: (assume cwd = 'workspace/foo') + + Target patterns which do not begin with '//' are taken relative to + the working directory. Patterns which begin with '//' are always + absolute. + + ...:all Equivalent to '//foo/...:all'. + ... (ditto) + + bar/...:all Equivalent to '//foo/bar/...:all'. + bar/... (ditto) + + bar:wiz Equivalent to '//foo/bar:wiz'. + :foo Equivalent to '//foo:foo'. + + bar:all Equivalent to '//foo/bar:all'. + :all Equivalent to '//foo:all'. + +Summary of target wildcards: + + :all, Match all rules in the specified packages. + :*, :all-targets Match all targets (rules and files) in the specified + packages, including .par and _deploy.jar files. + +Subtractive patterns: + + Target patterns may be preceded by '-', meaning they should be + subtracted from the set of targets accumulated by preceding + patterns. For example: + + % blaze build -- foo/... -foo/contrib/... + + builds everything in 'foo', except 'contrib'. In case a target not + under 'contrib' depends on something under 'contrib' though, in order to + build the former blaze has to build the latter too. As usual, the '--' is + required to prevent '-b' from being interpreted as an option. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt new file mode 100644 index 0000000000..a1f0523e2b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt @@ -0,0 +1,15 @@ + +Usage: blaze %{command} <options> <test-targets> + +Builds the specified targets and runs all test targets among them (test targets +might also need to satisfy provided tag, size or language filters) using +the specified options. + +This command accepts all valid options to 'build', and inherits +defaults for 'build' from your .blazerc. If you don't use .blazerc, +don't forget to pass all your 'build' options to '%{command}' too. + +See 'blaze help target-syntax' for details and examples on how to +specify targets. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt new file mode 100644 index 0000000000..10e1df76cf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt @@ -0,0 +1,3 @@ +Prints the version information that was embedded when blaze was built. + +%{options} |