diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/runtime/commands')
26 files changed, 3253 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java new file mode 100644 index 0000000000..d6f61eb494 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java @@ -0,0 +1,69 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.analysis.BuildView; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.List; + +/** + * Handles the 'build' command on the Blaze command line, including targets + * named by arguments passed to Blaze. + */ +@Command(name = "build", + builds = true, + options = { BuildRequestOptions.class, + ExecutionOptions.class, + PackageCacheOptions.class, + BuildView.Options.class, + LoadingPhaseRunner.Options.class, + BuildConfiguration.Options.class, + }, + usesConfigurationOptions = true, + shortDescription = "Builds the specified targets.", + allowResidue = true, + help = "resource:build.txt") +public final class BuildCommand implements BlazeCommand { + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) + throws AbruptExitException { + ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "build"); + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + List<String> targets = ProjectFileSupport.getTargets(runtime, options); + + BuildRequest request = BuildRequest.create( + getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), + targets, + runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime()); + return runtime.getBuildTool().processRequest(request, null).getExitCondition(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java new file mode 100644 index 0000000000..0bb5a0e966 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CanonicalizeCommand.java @@ -0,0 +1,95 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandUtils; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Collection; +import java.util.List; + +/** + * The 'blaze canonicalize-flags' command. + */ +@Command(name = "canonicalize-flags", + options = { CanonicalizeCommand.Options.class }, + allowResidue = true, + mustRunInWorkspace = false, + shortDescription = "Canonicalizes a list of Blaze options.", + help = "This command canonicalizes a list of Blaze options. Don't forget to prepend '--' " + + "to end option parsing before the flags to canonicalize.\n" + + "%{options}") +public final class CanonicalizeCommand implements BlazeCommand { + + public static class CommandConverter implements Converter<String> { + + @Override + public String convert(String input) throws OptionsParsingException { + if (input.equals("build")) { + return input; + } else if (input.equals("test")) { + return input; + } + throw new OptionsParsingException("Not a valid command: '" + input + "' (should be " + + getTypeDescription() + ")"); + } + + @Override + public String getTypeDescription() { + return "build or test"; + } + } + + public static class Options extends OptionsBase { + + @Option(name = "for_command", + defaultValue = "build", + category = "misc", + converter = CommandConverter.class, + help = "The command for which the options should be canonicalized.") + public String forCommand; + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + BlazeCommand command = runtime.getCommandMap().get( + options.getOptions(Options.class).forCommand); + Collection<Class<? extends OptionsBase>> optionsClasses = + BlazeCommandUtils.getOptions( + command.getClass(), runtime.getBlazeModules(), runtime.getRuleClassProvider()); + try { + List<String> result = OptionsParser.canonicalize(optionsClasses, options.getResidue()); + for (String piece : result) { + runtime.getReporter().getOutErr().printOutLn(piece); + } + } catch (OptionsParsingException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.COMMAND_LINE_ERROR; + } + return ExitCode.SUCCESS; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java new file mode 100644 index 0000000000..3fd300eaaa --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java @@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.ProcessUtils; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Implements 'blaze clean'. + */ +@Command(name = "clean", + builds = true, // Does not, but people expect build options to be there + options = { CleanCommand.Options.class }, + help = "resource:clean.txt", + shortDescription = "Removes output files and optionally stops the server.", + // TODO(bazel-team): Remove this - we inherit a huge number of unused options. + inherits = { BuildCommand.class }) +public final class CleanCommand implements BlazeCommand { + + /** + * An interface for special options for the clean command. + */ + public static class Options extends OptionsBase { + @Option(name = "clean_style", + defaultValue = "", + category = "clean", + help = "Can be either 'expunge' or 'expunge_async'.") + public String cleanStyle; + + @Option(name = "expunge", + defaultValue = "false", + category = "clean", + expansion = "--clean_style=expunge", + help = "If specified, clean will remove the entire working tree for this Blaze " + + "instance, which includes all Blaze-created temporary and build output " + + "files, and it will stop the Blaze server if it is running.") + public boolean expunge; + + @Option(name = "expunge_async", + defaultValue = "false", + category = "clean", + expansion = "--clean_style=expunge_async", + help = "If specified, clean will asynchronously remove the entire working tree for " + + "this Blaze instance, which includes all Blaze-created temporary and build " + + "output files, and it will stop the Blaze server if it is running. When this " + + "command completes, it will be safe to execute new commands in the same client, " + + "even though the deletion may continue in the background.") + public boolean expunge_async; + } + + private static Logger LOG = Logger.getLogger(CleanCommand.class.getName()); + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + Options cleanOptions = options.getOptions(Options.class); + cleanOptions.expunge_async = cleanOptions.cleanStyle.equals("expunge_async"); + cleanOptions.expunge = cleanOptions.cleanStyle.equals("expunge"); + + if (cleanOptions.expunge == false && cleanOptions.expunge_async == false && + !cleanOptions.cleanStyle.isEmpty()) { + runtime.getReporter().handle(Event.error( + null, "Invalid clean_style value '" + cleanOptions.cleanStyle + "'")); + return ExitCode.COMMAND_LINE_ERROR; + } + + String cleanBanner = cleanOptions.expunge_async ? + "Starting clean." : + "Starting clean (this may take a while). " + + "Consider using --expunge_async if the clean takes more than several minutes."; + + runtime.getReporter().handle(Event.info(null/*location*/, cleanBanner)); + try { + String symlinkPrefix = + options.getOptions(BuildRequest.BuildRequestOptions.class).symlinkPrefix; + actuallyClean(runtime, runtime.getOutputBase(), cleanOptions, symlinkPrefix); + return ExitCode.SUCCESS; + } catch (IOException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } catch (CommandException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.RUN_FAILURE; + } catch (ExecException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.RUN_FAILURE; + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("clean interrupted")); + return ExitCode.INTERRUPTED; + } + } + + private void actuallyClean(BlazeRuntime runtime, + Path outputBase, Options cleanOptions, String symlinkPrefix) throws IOException, + ShutdownBlazeServerException, CommandException, ExecException, InterruptedException { + if (runtime.getOutputService() != null) { + runtime.getOutputService().clean(); + } + if (cleanOptions.expunge) { + LOG.info("Expunging..."); + // Delete the big subdirectories with the important content first--this + // will take the most time. Then quickly delete the little locks, logs + // and links right before we exit. Once the lock file is gone there will + // be a small possibility of a server race if a client is waiting, but + // all significant files will be gone by then. + FileSystemUtils.deleteTreesBelow(outputBase); + FileSystemUtils.deleteTree(outputBase); + } else if (cleanOptions.expunge_async) { + LOG.info("Expunging asynchronously..."); + String tempBaseName = outputBase.getBaseName() + "_tmp_" + ProcessUtils.getpid(); + + // Keeping tempOutputBase in the same directory ensures it remains in the + // same file system, and therefore the mv will be atomic and fast. + Path tempOutputBase = outputBase.getParentDirectory().getChild(tempBaseName); + outputBase.renameTo(tempOutputBase); + runtime.getReporter().handle(Event.info( + null, "Output base moved to " + tempOutputBase + " for deletion")); + + // Daemonize the shell and use the double-fork idiom to ensure that the shell + // exits even while the "rm -rf" command continues. + String command = String.format("exec >&- 2>&- <&- && (/usr/bin/setsid /bin/rm -rf %s &)&", + ShellEscaper.escapeString(tempOutputBase.getPathString())); + + LOG.info("Executing shell commmand " + ShellEscaper.escapeString(command)); + + // Doesn't throw iff command exited and was successful. + new CommandBuilder().addArg(command).useShell(true) + .setWorkingDir(tempOutputBase.getParentDirectory()) + .build().execute(); + } else { + LOG.info("Output cleaning..."); + runtime.clearCaches(); + for (String directory : new String[] { + BlazeDirectories.RELATIVE_OUTPUT_PATH, runtime.getWorkspaceName() }) { + Path child = outputBase.getChild(directory); + if (child.exists()) { + LOG.finest("Cleaning " + child); + FileSystemUtils.deleteTreesBelow(child); + } + } + } + // remove convenience links + OutputDirectoryLinksUtils.removeOutputDirectoryLinks( + runtime.getWorkspaceName(), runtime.getWorkspace(), runtime.getReporter(), symlinkPrefix); + // shutdown on expunge cleans + if (cleanOptions.expunge || cleanOptions.expunge_async) { + throw new ShutdownBlazeServerException(0); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java new file mode 100644 index 0000000000..5267e71acc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java @@ -0,0 +1,248 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.docgen.BlazeRuleHelpPrinter; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandUtils; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The 'blaze help' command, which prints all available commands as well as + * specific help pages. + */ +@Command(name = "help", + options = { HelpCommand.Options.class }, + allowResidue = true, + mustRunInWorkspace = false, + shortDescription = "Prints help for commands, or the index.", + help = "resource:help.txt") +public final class HelpCommand implements BlazeCommand { + public static class Options extends OptionsBase { + + @Option(name = "help_verbosity", + category = "help", + defaultValue = "medium", + converter = Converters.HelpVerbosityConverter.class, + help = "Select the verbosity of the help command.") + public OptionsParser.HelpVerbosity helpVerbosity; + + @Option(name = "long", + abbrev = 'l', + defaultValue = "null", + category = "help", + expansion = {"--help_verbosity", "long"}, + help = "Show full description of each option, instead of just its name.") + public Void showLongFormOptions; + + @Option(name = "short", + defaultValue = "null", + category = "help", + expansion = {"--help_verbosity", "short"}, + help = "Show only the names of the options, not their types or meanings.") + public Void showShortFormOptions; + } + + /** + * Returns a map that maps option categories to descriptive help strings for categories that + * are not part of the Bazel core. + */ + private ImmutableMap<String, String> getOptionCategories(BlazeRuntime runtime) { + ImmutableMap.Builder<String, String> optionCategoriesBuilder = ImmutableMap.builder(); + optionCategoriesBuilder + .put("checking", + "Checking options, which control Blaze's error checking and/or warnings") + .put("coverage", + "Options that affect how Blaze generates code coverage information") + .put("experimental", + "Experimental options, which control experimental (and potentially risky) features") + .put("flags", + "Flags options, for passing options to other tools") + .put("help", + "Help options") + .put("host jvm startup", + "Options that affect the startup of the Blaze server's JVM") + .put("misc", + "Miscellaneous options") + .put("package loading", + "Options that specify how to locate packages") + .put("query", + "Options affecting the 'blaze query' dependency query command") + .put("run", + "Options specific to 'blaze run'") + .put("semantics", + "Semantics options, which affect the build commands and/or output file contents") + .put("server startup", + "Startup options, which affect the startup of the Blaze server") + .put("strategy", + "Strategy options, which affect how Blaze will execute the build") + .put("testing", + "Options that affect how Blaze runs tests") + .put("verbosity", + "Verbosity options, which control what Blaze prints") + .put("version", + "Version options, for selecting which version of other tools will be used") + .put("what", + "Output selection options, for determining what to build/test"); + for (BlazeModule module : runtime.getBlazeModules()) { + optionCategoriesBuilder.putAll(module.getOptionCategories()); + } + return optionCategoriesBuilder.build(); + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + OutErr outErr = runtime.getReporter().getOutErr(); + Options helpOptions = options.getOptions(Options.class); + if (options.getResidue().isEmpty()) { + emitBlazeVersionInfo(outErr); + emitGenericHelp(runtime, outErr); + return ExitCode.SUCCESS; + } + if (options.getResidue().size() != 1) { + runtime.getReporter().handle(Event.error("You must specify exactly one command")); + return ExitCode.COMMAND_LINE_ERROR; + } + String helpSubject = options.getResidue().get(0); + if (helpSubject.equals("startup_options")) { + emitBlazeVersionInfo(outErr); + emitStartupOptions(outErr, helpOptions.helpVerbosity, runtime, getOptionCategories(runtime)); + return ExitCode.SUCCESS; + } else if (helpSubject.equals("target-syntax")) { + emitBlazeVersionInfo(outErr); + emitTargetSyntaxHelp(outErr, getOptionCategories(runtime)); + return ExitCode.SUCCESS; + } else if (helpSubject.equals("info-keys")) { + emitInfoKeysHelp(runtime, outErr); + return ExitCode.SUCCESS; + } + + BlazeCommand command = runtime.getCommandMap().get(helpSubject); + if (command == null) { + ConfiguredRuleClassProvider provider = runtime.getRuleClassProvider(); + RuleClass ruleClass = provider.getRuleClassMap().get(helpSubject); + if (ruleClass != null && ruleClass.isDocumented()) { + // There is a rule with a corresponding name + outErr.printOut(BlazeRuleHelpPrinter.getRuleDoc(helpSubject, provider)); + return ExitCode.SUCCESS; + } else { + runtime.getReporter().handle(Event.error( + null, "'" + helpSubject + "' is neither a command nor a build rule")); + return ExitCode.COMMAND_LINE_ERROR; + } + } + emitBlazeVersionInfo(outErr); + outErr.printOut(BlazeCommandUtils.getUsage( + command.getClass(), + getOptionCategories(runtime), + helpOptions.helpVerbosity, + runtime.getBlazeModules(), + runtime.getRuleClassProvider())); + return ExitCode.SUCCESS; + } + + private void emitBlazeVersionInfo(OutErr outErr) { + String releaseInfo = BlazeVersionInfo.instance().getReleaseName(); + String line = "[Blaze " + releaseInfo + "]"; + outErr.printOut(String.format("%80s\n", line)); + } + + @SuppressWarnings("unchecked") // varargs generic array creation + private void emitStartupOptions(OutErr outErr, OptionsParser.HelpVerbosity helpVerbosity, + BlazeRuntime runtime, ImmutableMap<String, String> optionCategories) { + outErr.printOut( + BlazeCommandUtils.expandHelpTopic("startup_options", + "resource:startup_options.txt", + getClass(), + BlazeCommandUtils.getStartupOptions(runtime.getBlazeModules()), + optionCategories, + helpVerbosity)); + } + + private void emitTargetSyntaxHelp(OutErr outErr, ImmutableMap<String, String> optionCategories) { + outErr.printOut(BlazeCommandUtils.expandHelpTopic("target-syntax", + "resource:target-syntax.txt", + getClass(), + ImmutableList.<Class<? extends OptionsBase>>of(), + optionCategories, + OptionsParser.HelpVerbosity.MEDIUM)); + } + + private void emitInfoKeysHelp(BlazeRuntime runtime, OutErr outErr) { + for (InfoKey key : InfoKey.values()) { + outErr.printOut(String.format("%-23s %s\n", key.getName(), key.getDescription())); + } + + for (BlazeModule.InfoItem item : InfoCommand.getInfoItemMap(runtime, + OptionsParser.newOptionsParser( + ImmutableList.<Class<? extends OptionsBase>>of())).values()) { + outErr.printOut(String.format("%-23s %s\n", item.getName(), item.getDescription())); + } + } + + private void emitGenericHelp(BlazeRuntime runtime, OutErr outErr) { + outErr.printOut("Usage: blaze <command> <options> ...\n\n"); + + outErr.printOut("Available commands:\n"); + + Map<String, BlazeCommand> commandsByName = runtime.getCommandMap(); + List<String> namesInOrder = new ArrayList<>(commandsByName.keySet()); + Collections.sort(namesInOrder); + + for (String name : namesInOrder) { + BlazeCommand command = commandsByName.get(name); + Command annotation = command.getClass().getAnnotation(Command.class); + if (annotation.hidden()) { + continue; + } + + String shortDescription = annotation.shortDescription(); + outErr.printOut(String.format(" %-19s %s\n", name, shortDescription)); + } + + outErr.printOut("\n"); + outErr.printOut("Getting more help:\n"); + outErr.printOut(" blaze help <command>\n"); + outErr.printOut(" Prints help and options for <command>.\n"); + outErr.printOut(" blaze help startup_options\n"); + outErr.printOut(" Options for the JVM hosting Blaze.\n"); + outErr.printOut(" blaze help target-syntax\n"); + outErr.printOut(" Explains the syntax for specifying targets.\n"); + outErr.printOut(" blaze help info-keys\n"); + outErr.printOut(" Displays a list of keys used by the info command.\n"); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java new file mode 100644 index 0000000000..31aaeb1cef --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java @@ -0,0 +1,448 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.base.Supplier; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.Constants; +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.ProtoUtils; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClassProvider; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.AllowedRuleClassInfo; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.AttributeDefinition; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.BuildLanguage; +import com.google.devtools.build.lib.query2.proto.proto2api.Build.RuleDefinition; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.StringUtilities; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +/** + * Implementation of 'blaze info'. + */ +@Command(name = "info", + // TODO(bazel-team): this is not really a build command, but needs access to the + // configuration options to do its job + builds = true, + allowResidue = true, + binaryStdOut = true, + help = "resource:info.txt", + shortDescription = "Displays runtime info about the blaze server.", + options = { InfoCommand.Options.class }, + // We have InfoCommand inherit from {@link BuildCommand} because we want all + // configuration defaults specified in ~/.blazerc for {@code build} to apply to + // {@code info} too, even though it doesn't actually do a build. + // + // (Ideally there would be a way to make {@code info} inherit just the bare + // minimum of relevant options from {@code build}, i.e. those that affect the + // values it prints. But there's no such mechanism.) + inherits = { BuildCommand.class }) +public class InfoCommand implements BlazeCommand { + + public static class Options extends OptionsBase { + @Option(name = "show_make_env", + defaultValue = "false", + category = "misc", + help = "Include the \"Make\" environment in the output.") + public boolean showMakeEnvironment; + } + + /** + * Unchecked variant of ExitCausingException. Below, we need to throw from the Supplier interface, + * which does not allow checked exceptions. + */ + public static class ExitCausingRuntimeException extends RuntimeException { + + private final ExitCode exitCode; + + public ExitCausingRuntimeException(String message, ExitCode exitCode) { + super(message); + this.exitCode = exitCode; + } + + public ExitCausingRuntimeException(ExitCode exitCode) { + this.exitCode = exitCode; + } + + public ExitCode getExitCode() { + return exitCode; + } + } + + private static class HardwiredInfoItem implements BlazeModule.InfoItem { + private final InfoKey key; + private final BlazeRuntime runtime; + private final OptionsProvider commandOptions; + + private HardwiredInfoItem(InfoKey key, BlazeRuntime runtime, OptionsProvider commandOptions) { + this.key = key; + this.runtime = runtime; + this.commandOptions = commandOptions; + } + + @Override + public String getName() { + return key.getName(); + } + + @Override + public String getDescription() { + return key.getDescription(); + } + + @Override + public boolean isHidden() { + return key.isHidden(); + } + + @Override + public byte[] get(Supplier<BuildConfiguration> configurationSupplier) { + return print(getInfoItem(runtime, key, configurationSupplier, commandOptions)); + } + } + + private static class MakeInfoItem implements BlazeModule.InfoItem { + private final String name; + private final String value; + + private MakeInfoItem(String name, String value) { + this.name = name; + this.value = value; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "Make environment variable '" + name + "'"; + } + + @Override + public boolean isHidden() { + return false; + } + + @Override + public byte[] get(Supplier<BuildConfiguration> configurationSupplier) { + return print(value); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + @Override + public ExitCode exec(final BlazeRuntime runtime, final OptionsProvider optionsProvider) { + Options infoOptions = optionsProvider.getOptions(Options.class); + + OutErr outErr = runtime.getReporter().getOutErr(); + // Creating a BuildConfiguration is expensive and often unnecessary. Delay the creation until + // it is needed. + Supplier<BuildConfiguration> configurationSupplier = new Supplier<BuildConfiguration>() { + private BuildConfiguration configuration; + @Override + public BuildConfiguration get() { + if (configuration != null) { + return configuration; + } + try { + // In order to be able to answer configuration-specific queries, we need to setup the + // package path. Since info inherits all the build options, all the necessary information + // is available here. + runtime.setupPackageCache( + optionsProvider.getOptions(PackageCacheOptions.class), + runtime.getDefaultsPackageContent(optionsProvider)); + // TODO(bazel-team): What if there are multiple configurations? [multi-config] + configuration = runtime + .getConfigurations(optionsProvider) + .getTargetConfigurations().get(0); + return configuration; + } catch (InvalidConfigurationException e) { + runtime.getReporter().handle(Event.error(e.getMessage())); + throw new ExitCausingRuntimeException(ExitCode.COMMAND_LINE_ERROR); + } catch (AbruptExitException e) { + throw new ExitCausingRuntimeException("unknown error: " + e.getMessage(), + e.getExitCode()); + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("interrupted")); + throw new ExitCausingRuntimeException(ExitCode.INTERRUPTED); + } + } + }; + + Map<String, BlazeModule.InfoItem> items = getInfoItemMap(runtime, optionsProvider); + + try { + if (infoOptions.showMakeEnvironment) { + Map<String, String> makeEnv = configurationSupplier.get().getMakeEnvironment(); + for (Map.Entry<String, String> entry : makeEnv.entrySet()) { + BlazeModule.InfoItem item = new MakeInfoItem(entry.getKey(), entry.getValue()); + items.put(item.getName(), item); + } + } + + List<String> residue = optionsProvider.getResidue(); + if (residue.size() > 1) { + runtime.getReporter().handle(Event.error("at most one key may be specified")); + return ExitCode.COMMAND_LINE_ERROR; + } + + String key = residue.size() == 1 ? residue.get(0) : null; + if (key != null) { // print just the value for the specified key: + byte[] value; + if (items.containsKey(key)) { + value = items.get(key).get(configurationSupplier); + } else { + runtime.getReporter().handle(Event.error("unknown key: '" + key + "'")); + return ExitCode.COMMAND_LINE_ERROR; + } + try { + outErr.getOutputStream().write(value); + outErr.getOutputStream().flush(); + } catch (IOException e) { + runtime.getReporter().handle(Event.error("Cannot write info block: " + e.getMessage())); + return ExitCode.ANALYSIS_FAILURE; + } + } else { // print them all + configurationSupplier.get(); // We'll need this later anyway + for (BlazeModule.InfoItem infoItem : items.values()) { + if (infoItem.isHidden()) { + continue; + } + outErr.getOutputStream().write( + (infoItem.getName() + ": ").getBytes(StandardCharsets.UTF_8)); + outErr.getOutputStream().write(infoItem.get(configurationSupplier)); + } + } + } catch (AbruptExitException e) { + return e.getExitCode(); + } catch (ExitCausingRuntimeException e) { + return e.getExitCode(); + } catch (IOException e) { + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + return ExitCode.SUCCESS; + } + + /** + * Compute and return the info for the given key. Only keys that are not hidden are supported + * here. + */ + private static Object getInfoItem(BlazeRuntime runtime, InfoKey key, + Supplier<BuildConfiguration> configurationSupplier, OptionsProvider options) { + switch (key) { + // directories + case WORKSPACE : return runtime.getWorkspace(); + case INSTALL_BASE : return runtime.getDirectories().getInstallBase(); + case OUTPUT_BASE : return runtime.getOutputBase(); + case EXECUTION_ROOT : return runtime.getExecRoot(); + case OUTPUT_PATH : return runtime.getDirectories().getOutputPath(); + // These are the only (non-hidden) info items that require a configuration, because the + // corresponding paths contain the short name. Maybe we should recommend using the symlinks + // or make them hidden by default? + case BLAZE_BIN : return configurationSupplier.get().getBinDirectory().getPath(); + case BLAZE_GENFILES : return configurationSupplier.get().getGenfilesDirectory().getPath(); + case BLAZE_TESTLOGS : return configurationSupplier.get().getTestLogsDirectory().getPath(); + + // logs + case COMMAND_LOG : return BlazeCommandDispatcher.getCommandLogPath(runtime.getOutputBase()); + case MESSAGE_LOG : + // NB: Duplicated in EventLogModule + return runtime.getOutputBase().getRelative("message.log"); + + // misc + case RELEASE : return BlazeVersionInfo.instance().getReleaseName(); + case SERVER_PID : return OsUtils.getpid(); + case PACKAGE_PATH : return getPackagePath(options); + + // memory statistics + case GC_COUNT : + case GC_TIME : + // The documentation is not very clear on what it means to have more than + // one GC MXBean, so we just sum them up. + int gcCount = 0; + int gcTime = 0; + for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) { + gcCount += gcBean.getCollectionCount(); + gcTime += gcBean.getCollectionTime(); + } + if (key == InfoKey.GC_COUNT) { + return gcCount + ""; + } else { + return gcTime + "ms"; + } + + case MAX_HEAP_SIZE : + return StringUtilities.prettyPrintBytes(getMemoryUsage().getMax()); + case USED_HEAP_SIZE : + case COMMITTED_HEAP_SIZE : + return StringUtilities.prettyPrintBytes(key == InfoKey.USED_HEAP_SIZE ? + getMemoryUsage().getUsed() : getMemoryUsage().getCommitted()); + + case USED_HEAP_SIZE_AFTER_GC : + // Note that this info value is not printed by default, but only when explicitly requested. + System.gc(); + return StringUtilities.prettyPrintBytes(getMemoryUsage().getUsed()); + + case DEFAULTS_PACKAGE: + return runtime.getDefaultsPackageContent(); + + case BUILD_LANGUAGE: + return getBuildLanguageDefinition(runtime.getRuleClassProvider()); + + case DEFAULT_PACKAGE_PATH: + return Joiner.on(":").join(Constants.DEFAULT_PACKAGE_PATH); + + default: + throw new IllegalArgumentException("missing implementation for " + key); + } + } + + private static MemoryUsage getMemoryUsage() { + MemoryMXBean memBean = ManagementFactory.getMemoryMXBean(); + return memBean.getHeapMemoryUsage(); + } + + /** + * Get the package_path variable for the given set of options. + */ + private static String getPackagePath(OptionsProvider options) { + PackageCacheOptions packageCacheOptions = + options.getOptions(PackageCacheOptions.class); + return Joiner.on(":").join(packageCacheOptions.packagePath); + } + + private static AllowedRuleClassInfo getAllowedRuleClasses( + Collection<RuleClass> ruleClasses, Attribute attr) { + AllowedRuleClassInfo.Builder info = AllowedRuleClassInfo.newBuilder(); + info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.ANY); + + if (attr.isStrictLabelCheckingEnabled()) { + if (attr.getAllowedRuleClassesPredicate() != Predicates.<RuleClass>alwaysTrue()) { + info.setPolicy(AllowedRuleClassInfo.AllowedRuleClasses.SPECIFIED); + Predicate<RuleClass> filter = attr.getAllowedRuleClassesPredicate(); + for (RuleClass otherClass : Iterables.filter( + ruleClasses, filter)) { + if (otherClass.isDocumented()) { + info.addAllowedRuleClass(otherClass.getName()); + } + } + } + } + + return info.build(); + } + + /** + * Returns a byte array containing a proto-buffer describing the build language. + */ + private static byte[] getBuildLanguageDefinition(RuleClassProvider provider) { + BuildLanguage.Builder resultPb = BuildLanguage.newBuilder(); + Collection<RuleClass> ruleClasses = provider.getRuleClassMap().values(); + for (RuleClass ruleClass : ruleClasses) { + if (!ruleClass.isDocumented()) { + continue; + } + + RuleDefinition.Builder rulePb = RuleDefinition.newBuilder(); + rulePb.setName(ruleClass.getName()); + for (Attribute attr : ruleClass.getAttributes()) { + if (!attr.isDocumented()) { + continue; + } + + AttributeDefinition.Builder attrPb = AttributeDefinition.newBuilder(); + attrPb.setName(attr.getName()); + // The protocol compiler, in its infinite wisdom, generates the field as one of the + // integer type and the getTypeEnum() method is missing. WTF? + attrPb.setType(ProtoUtils.getDiscriminatorFromType(attr.getType())); + attrPb.setMandatory(attr.isMandatory()); + + if (Type.isLabelType(attr.getType())) { + attrPb.setAllowedRuleClasses(getAllowedRuleClasses(ruleClasses, attr)); + } + + rulePb.addAttribute(attrPb); + } + + resultPb.addRule(rulePb); + } + + return resultPb.build().toByteArray(); + } + + private static byte[] print(Object value) { + if (value instanceof byte[]) { + return (byte[]) value; + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(outputStream); + writer.print(value.toString() + "\n"); + writer.flush(); + return outputStream.toByteArray(); + } + + static Map<String, BlazeModule.InfoItem> getInfoItemMap( + BlazeRuntime runtime, OptionsProvider commandOptions) { + Map<String, BlazeModule.InfoItem> result = new TreeMap<>(); // order by key + for (BlazeModule module : runtime.getBlazeModules()) { + for (BlazeModule.InfoItem item : module.getInfoItems()) { + result.put(item.getName(), item); + } + } + + for (InfoKey key : InfoKey.values()) { + BlazeModule.InfoItem item = new HardwiredInfoItem(key, runtime, commandOptions); + result.put(item.getName(), item); + } + + return result; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java new file mode 100644 index 0000000000..d2e7bc0718 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoKey.java @@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + + +/** + * An enumeration of all the valid info keys, excepting the make environment + * variables. + */ +public enum InfoKey { + // directories + WORKSPACE("workspace", "The working directory of the server."), + INSTALL_BASE("install_base", "The installation base directory."), + OUTPUT_BASE("output_base", + "A directory for shared Blaze state as well as tool and strategy specific subdirectories."), + EXECUTION_ROOT("execution_root", + "A directory that makes all input and output files visible to the build."), + OUTPUT_PATH("output_path", "Output directory"), + BLAZE_BIN("blaze-bin", "Configuration dependent directory for binaries."), + BLAZE_GENFILES("blaze-genfiles", "Configuration dependent directory for generated files."), + BLAZE_TESTLOGS("blaze-testlogs", "Configuration dependent directory for logs from a test run."), + + // logs + COMMAND_LOG("command_log", "Location of the log containg the output from the build commands."), + MESSAGE_LOG("message_log" , + "Location of a log containing machine readable message in LogMessage protobuf format."), + + // misc + RELEASE("release", "Blaze release identifier"), + SERVER_PID("server_pid", "Blaze process id"), + PACKAGE_PATH("package_path", "The search path for resolving package labels."), + + // memory statistics + USED_HEAP_SIZE("used-heap-size", "The amount of used memory in bytes. Note that this is not a " + + "good indicator of the actual memory use, as it includes any remaining inaccessible " + + "memory."), + USED_HEAP_SIZE_AFTER_GC("used-heap-size-after-gc", + "The amount of used memory in bytes after a call to System.gc().", true), + COMMITTED_HEAP_SIZE("committed-heap-size", + "The amount of memory in bytes that is committed for the Java virtual machine to use"), + MAX_HEAP_SIZE("max-heap-size", + "The maximum amount of memory in bytes that can be used for memory management."), + GC_COUNT("gc-count", "Number of garbage collection runs."), + GC_TIME("gc-time", "The approximate accumulated time spend on garbage collection."), + + // These are deprecated, they still work, when explicitly requested, but are not shown by default + + // These keys print multi-line messages and thus don't play well with grep. We don't print them + // unless explicitly requested + DEFAULTS_PACKAGE("defaults-package", "Default packages used as implicit dependencies", true), + BUILD_LANGUAGE("build-language", "A protobuffer with the build language structure", true), + DEFAULT_PACKAGE_PATH("default-package-path", "The default package path", true); + + private final String name; + private final String description; + private final boolean hidden; + + private InfoKey(String name, String description) { + this(name, description, false); + } + + private InfoKey(String name, String description, boolean hidden) { + this.name = name; + this.description = description; + this.hidden = hidden; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public boolean isHidden() { + return hidden; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java new file mode 100644 index 0000000000..7b91dc7fa3 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProfileCommand.java @@ -0,0 +1,771 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.google.common.collect.TreeMultimap; +import com.google.devtools.build.lib.actions.MiddlemanAction; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.profiler.ProfileInfo; +import com.google.devtools.build.lib.profiler.ProfileInfo.CriticalPathEntry; +import com.google.devtools.build.lib.profiler.ProfileInfo.InfoListener; +import com.google.devtools.build.lib.profiler.ProfilePhase; +import com.google.devtools.build.lib.profiler.ProfilePhaseStatistics; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.chart.AggregatingChartCreator; +import com.google.devtools.build.lib.profiler.chart.Chart; +import com.google.devtools.build.lib.profiler.chart.ChartCreator; +import com.google.devtools.build.lib.profiler.chart.DetailedChartCreator; +import com.google.devtools.build.lib.profiler.chart.HtmlChartVisitor; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.StringUtil; +import com.google.devtools.build.lib.util.TimeUtilities; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +/** + * Command line wrapper for analyzing Blaze build profiles. + */ +@Command(name = "analyze-profile", + options = { ProfileCommand.ProfileOptions.class }, + shortDescription = "Analyzes build profile data.", + help = "resource:analyze-profile.txt", + allowResidue = true, + mustRunInWorkspace = false) +public final class ProfileCommand implements BlazeCommand { + + private final String TWO_COLUMN_FORMAT = "%-37s %10s\n"; + private final String THREE_COLUMN_FORMAT = "%-28s %10s %8s\n"; + + public static class DumpConverter extends Converters.StringSetConverter { + public DumpConverter() { + super("text", "raw", "text-unsorted", "raw-unsorted"); + } + } + + public static class ProfileOptions extends OptionsBase { + @Option(name = "dump", + abbrev='d', + converter = DumpConverter.class, + defaultValue = "null", + help = "output full profile data dump either in human-readable 'text' format or" + + " script-friendly 'raw' format, either sorted or unsorted.") + public String dumpMode; + + @Option(name = "html", + defaultValue = "false", + help = "If present, an HTML file visualizing the tasks of the profiled build is created. " + + "The name of the html file is the name of the profile file plus '.html'.") + public boolean html; + + @Option(name = "html_pixels_per_second", + defaultValue = "50", + help = "Defines the scale of the time axis of the task diagram. The unit is " + + "pixels per second. Default is 50 pixels per second. ") + public int htmlPixelsPerSecond; + + @Option(name = "html_details", + defaultValue = "false", + help = "If --html_details is present, the task diagram contains all tasks of the profile. " + + "If --nohtml_details is present, an aggregated diagram is generated. The default is " + + "to generate an aggregated diagram.") + public boolean htmlDetails; + + @Option(name = "vfs_stats", + defaultValue = "false", + help = "If present, include VFS path statistics.") + public boolean vfsStats; + + @Option(name = "vfs_stats_limit", + defaultValue = "-1", + help = "Maximum number of VFS path statistics to print.") + public int vfsStatsLimit; + } + + private Function<String, String> currentPathMapping = Functions.<String>identity(); + + private InfoListener getInfoListener(final BlazeRuntime runtime) { + return new InfoListener() { + private final EventHandler reporter = runtime.getReporter(); + + @Override + public void info(String text) { + reporter.handle(Event.info(text)); + } + + @Override + public void warn(String text) { + reporter.handle(Event.warn(text)); + } + }; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(final BlazeRuntime runtime, OptionsProvider options) { + ProfileOptions opts = + options.getOptions(ProfileOptions.class); + + if (!opts.vfsStats) { + opts.vfsStatsLimit = 0; + } + + currentPathMapping = new Function<String, String>() { + @Override + public String apply(String input) { + if (runtime.getWorkspaceName().isEmpty()) { + return input; + } else { + return input.substring(input.lastIndexOf("/" + runtime.getWorkspaceName()) + 1); + } + } + }; + + PrintStream out = new PrintStream(runtime.getReporter().getOutErr().getOutputStream()); + try { + runtime.getReporter().handle(Event.warn( + null, "This information is intended for consumption by Blaze developers" + + " only, and may change at any time. Script against it at your own risk")); + + for (String name : options.getResidue()) { + Path profileFile = runtime.getWorkingDirectory().getRelative(name); + try { + ProfileInfo info = ProfileInfo.loadProfileVerbosely( + profileFile, getInfoListener(runtime)); + if (opts.dumpMode != null) { + dumpProfile(runtime, info, out, opts.dumpMode); + } else if (opts.html) { + createHtml(runtime, info, profileFile, opts); + } else { + createText(runtime, info, out, opts); + } + } catch (IOException e) { + runtime.getReporter().handle(Event.error( + null, "Failed to process file " + name + ": " + e.getMessage())); + } + } + } finally { + out.flush(); + } + return ExitCode.SUCCESS; + } + + private void createText(BlazeRuntime runtime, ProfileInfo info, PrintStream out, + ProfileOptions opts) { + List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts); + + for (ProfilePhaseStatistics stat : statistics) { + String title = stat.getTitle(); + + if (!title.equals("")) { + out.println("\n=== " + title.toUpperCase() + " ===\n"); + } + out.print(stat.getStatistics()); + } + } + + private void createHtml(BlazeRuntime runtime, ProfileInfo info, Path profileFile, + ProfileOptions opts) + throws IOException { + Path htmlFile = + profileFile.getParentDirectory().getChild(profileFile.getBaseName() + ".html"); + List<ProfilePhaseStatistics> statistics = getStatistics(runtime, info, opts); + + runtime.getReporter().handle(Event.info("Creating HTML output in " + htmlFile)); + + ChartCreator chartCreator = + opts.htmlDetails ? new DetailedChartCreator(info, statistics) + : new AggregatingChartCreator(info, statistics); + Chart chart = chartCreator.create(); + OutputStream out = new BufferedOutputStream(htmlFile.getOutputStream()); + try { + chart.accept(new HtmlChartVisitor(new PrintStream(out), opts.htmlPixelsPerSecond)); + } finally { + try { + out.close(); + } catch (IOException e) { + // Ignore + } + } + } + + private List<ProfilePhaseStatistics> getStatistics( + BlazeRuntime runtime, ProfileInfo info, ProfileOptions opts) { + try { + ProfileInfo.aggregateProfile(info, getInfoListener(runtime)); + runtime.getReporter().handle(Event.info("Analyzing relationships")); + + info.analyzeRelationships(); + + List<ProfilePhaseStatistics> statistics = new ArrayList<>(); + + // Print phase durations and total execution time + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + long duration = 0; + for (ProfilePhase phase : ProfilePhase.values()) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + duration += info.getPhaseDuration(phaseTask); + } + } + for (ProfilePhase phase : ProfilePhase.values()) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + long phaseDuration = info.getPhaseDuration(phaseTask); + out.printf(THREE_COLUMN_FORMAT, "Total " + phase.nick + " phase time", + TimeUtilities.prettyTime(phaseDuration), prettyPercentage(phaseDuration, duration)); + } + } + out.printf(THREE_COLUMN_FORMAT, "Total run time", TimeUtilities.prettyTime(duration), + "100.00%"); + statistics.add(new ProfilePhaseStatistics("Phase Summary Information", + new String(byteOutput.toByteArray(), "UTF-8"))); + + // Print details of major phases + if (duration > 0) { + statistics.add(formatInitPhaseStatistics(info, opts)); + statistics.add(formatLoadingPhaseStatistics(info, opts)); + statistics.add(formatAnalysisPhaseStatistics(info, opts)); + ProfilePhaseStatistics stat = formatExecutionPhaseStatistics(info, opts); + if (stat != null) { + statistics.add(stat); + } + } + + return statistics; + } catch (UnsupportedEncodingException e) { + throw new AssertionError("Should not happen since, UTF8 is available on all JVMs"); + } + } + + private void dumpProfile( + BlazeRuntime runtime, ProfileInfo info, PrintStream out, String dumpMode) { + if (!dumpMode.contains("unsorted")) { + ProfileInfo.aggregateProfile(info, getInfoListener(runtime)); + } + if (dumpMode.contains("raw")) { + for (ProfileInfo.Task task : info.allTasksById) { + dumpRaw(task, out); + } + } else if (dumpMode.contains("unsorted")) { + for (ProfileInfo.Task task : info.allTasksById) { + dumpTask(task, out, 0); + } + } else { + for (ProfileInfo.Task task : info.rootTasksById) { + dumpTask(task, out, 0); + } + } + } + + private void dumpTask(ProfileInfo.Task task, PrintStream out, int indent) { + StringBuilder builder = new StringBuilder(String.format( + "\n%s %s\nThread: %-6d Id: %-6d Parent: %d\nStart time: %-12s Duration: %s", + task.type, task.getDescription(), task.threadId, task.id, task.parentId, + TimeUtilities.prettyTime(task.startTime), TimeUtilities.prettyTime(task.duration))); + if (task.hasStats()) { + builder.append("\n"); + ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray(); + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr attr = stats[type.ordinal()]; + if (attr != null) { + builder.append(type.toString().toLowerCase()).append("=("). + append(attr.count).append(", "). + append(TimeUtilities.prettyTime(attr.totalTime)).append(") "); + } + } + } + out.println(StringUtil.indent(builder.toString(), indent)); + for (ProfileInfo.Task subtask : task.subtasks) { + dumpTask(subtask, out, indent + 1); + } + } + + private void dumpRaw(ProfileInfo.Task task, PrintStream out) { + StringBuilder aggregateString = new StringBuilder(); + ProfileInfo.AggregateAttr[] stats = task.getStatAttrArray(); + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr attr = stats[type.ordinal()]; + if (attr != null) { + aggregateString.append(type.toString().toLowerCase()).append(","). + append(attr.count).append(",").append(attr.totalTime).append(" "); + } + } + out.println( + task.threadId + "|" + task.id + "|" + task.parentId + "|" + + task.startTime + "|" + task.duration + "|" + + aggregateString.toString().trim() + "|" + + task.type + "|" + task.getDescription()); + } + + /** + * Converts relative duration to the percentage string + * @return formatted percentage string or "N/A" if result is undefined. + */ + private static String prettyPercentage(long duration, long total) { + if (total == 0) { + // Return "not available" string if total is 0 and result is undefined. + return "N/A"; + } + return String.format("%5.2f%%", duration*100.0/total); + } + + private void printCriticalPath(String title, PrintStream out, CriticalPathEntry path) { + out.println(String.format("\n%s (%s):", title, + TimeUtilities.prettyTime(path.cumulativeDuration))); + + boolean lightCriticalPath = isLightCriticalPath(path); + out.println(lightCriticalPath ? + String.format("%6s %11s %8s %s", "Id", "Time", "Percentage", "Description") + : String.format("%6s %11s %8s %8s %s", "Id", "Time", "Share", "Critical", "Description")); + + long totalPathTime = path.cumulativeDuration; + int middlemanCount = 0; + long middlemanDuration = 0L; + long middlemanCritTime = 0L; + + for (; path != null ; path = path.next) { + if (path.task.id < 0) { + // Ignore fake actions. + continue; + } else if (path.task.getDescription().startsWith(MiddlemanAction.MIDDLEMAN_MNEMONIC + " ") + || path.task.getDescription().startsWith("TargetCompletionMiddleman")) { + // Aggregate middleman actions. + middlemanCount++; + middlemanDuration += path.duration; + middlemanCritTime += path.getCriticalTime(); + } else { + String desc = path.task.getDescription().replace(':', ' '); + if (lightCriticalPath) { + out.println(String.format("%6d %11s %8s %s", path.task.id, + TimeUtilities.prettyTime(path.duration), + prettyPercentage(path.duration, totalPathTime), + desc)); + } else { + out.println(String.format("%6d %11s %8s %8s %s", path.task.id, + TimeUtilities.prettyTime(path.duration), + prettyPercentage(path.duration, totalPathTime), + prettyPercentage(path.getCriticalTime(), totalPathTime), desc)); + } + } + } + if (middlemanCount > 0) { + if (lightCriticalPath) { + out.println(String.format(" %11s %8s [%d middleman actions]", + TimeUtilities.prettyTime(middlemanDuration), + prettyPercentage(middlemanDuration, totalPathTime), + middlemanCount)); + } else { + out.println(String.format(" %11s %8s %8s [%d middleman actions]", + TimeUtilities.prettyTime(middlemanDuration), + prettyPercentage(middlemanDuration, totalPathTime), + prettyPercentage(middlemanCritTime, totalPathTime), middlemanCount)); + } + } + } + + private boolean isLightCriticalPath(CriticalPathEntry path) { + return path.task.type == ProfilerTask.CRITICAL_PATH_COMPONENT; + } + + private void printShortPhaseAnalysis(ProfileInfo info, PrintStream out, ProfilePhase phase) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask != null) { + long phaseDuration = info.getPhaseDuration(phaseTask); + out.printf(TWO_COLUMN_FORMAT, "Total " + phase.nick + " phase time", + TimeUtilities.prettyTime(phaseDuration)); + printTimeDistributionByType(info, out, phaseTask); + } + } + + private void printTimeDistributionByType(ProfileInfo info, PrintStream out, + ProfileInfo.Task phaseTask) { + List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask); + long phaseDuration = info.getPhaseDuration(phaseTask); + long totalDuration = phaseDuration; + for (ProfileInfo.Task task : taskList) { + // Tasks on the phaseTask thread already accounted for in the phaseDuration. + if (task.threadId != phaseTask.threadId) { + totalDuration += task.duration; + } + } + boolean headerNeeded = true; + for (ProfilerTask type : ProfilerTask.values()) { + ProfileInfo.AggregateAttr stats = info.getStatsForType(type, taskList); + if (stats.count > 0 && stats.totalTime > 0) { + if (headerNeeded) { + out.println("\nTotal time (across all threads) spent on:"); + out.println(String.format("%18s %8s %8s %11s", "Type", "Total", "Count", "Average")); + headerNeeded = false; + } + out.println(String.format("%18s %8s %8d %11s", type.toString(), + prettyPercentage(stats.totalTime, totalDuration), stats.count, + TimeUtilities.prettyTime(stats.totalTime / stats.count))); + } + } + } + + static class Stat implements Comparable<Stat> { + public long duration; + public long frequency; + + @Override + public int compareTo(Stat o) { + return this.duration == o.duration ? Long.compare(this.frequency, o.frequency) + : Long.compare(this.duration, o.duration); + } + } + + /** + * Print the time spent on VFS operations on each path. Output is grouped by operation and sorted + * by descending duration. If multiple of the same VFS operation were logged for the same path, + * print the total duration. + * + * @param info profiling data. + * @param out output stream. + * @param phase build phase. + * @param limit maximum number of statistics to print, or -1 for no limit. + */ + private void printVfsStatistics(ProfileInfo info, PrintStream out, + ProfilePhase phase, int limit) { + ProfileInfo.Task phaseTask = info.getPhaseTask(phase); + if (phaseTask == null) { + return; + } + + if (limit == 0) { + return; + } + + // Group into VFS operations and build maps from path to duration. + + List<ProfileInfo.Task> taskList = info.getTasksForPhase(phaseTask); + EnumMap<ProfilerTask, Map<String, Stat>> stats = Maps.newEnumMap(ProfilerTask.class); + + collectVfsEntries(stats, taskList); + + if (!stats.isEmpty()) { + out.printf("\nVFS path statistics:\n"); + out.printf("%15s %10s %10s %s\n", "Type", "Frequency", "Duration", "Path"); + } + + // Reverse the maps to get maps from duration to path. We use a TreeMultimap to sort by duration + // and because durations are not unique. + + for (ProfilerTask type : stats.keySet()) { + Map<String, Stat> statsForType = stats.get(type); + TreeMultimap<Stat, String> sortedStats = + TreeMultimap.create(Ordering.natural().reverse(), Ordering.natural()); + + for (Map.Entry<String, Stat> stat : statsForType.entrySet()) { + sortedStats.put(stat.getValue(), stat.getKey()); + } + + int numPrinted = 0; + for (Map.Entry<Stat, String> stat : sortedStats.entries()) { + if (limit != -1 && numPrinted++ == limit) { + out.printf("... %d more ...\n", sortedStats.size() - limit); + break; + } + out.printf("%15s %10d %10s %s\n", + type.name(), stat.getKey().frequency, TimeUtilities.prettyTime(stat.getKey().duration), + stat.getValue()); + } + } + } + + private void collectVfsEntries(EnumMap<ProfilerTask, Map<String, Stat>> stats, + List<ProfileInfo.Task> taskList) { + for (ProfileInfo.Task task : taskList) { + collectVfsEntries(stats, Arrays.asList(task.subtasks)); + if (!task.type.name().startsWith("VFS_")) { + continue; + } + + Map<String, Stat> statsForType = stats.get(task.type); + if (statsForType == null) { + statsForType = Maps.newHashMap(); + stats.put(task.type, statsForType); + } + + String path = currentPathMapping.apply(task.getDescription()); + + Stat stat = statsForType.get(path); + if (stat == null) { + stat = new Stat(); + } + + stat.duration += task.duration; + stat.frequency++; + statsForType.put(path, stat); + } + } + + /** + * Returns set of profiler tasks to be filtered from critical path. + * Also always filters out ACTION_LOCK and WAIT tasks to simulate + * unlimited resource critical path (see comments inside formatExecutionPhaseStatistics() + * method). + */ + private EnumSet<ProfilerTask> getTypeFilter(ProfilerTask... tasks) { + EnumSet<ProfilerTask> filter = EnumSet.of(ProfilerTask.ACTION_LOCK, ProfilerTask.WAIT); + for (ProfilerTask task : tasks) { + filter.add(task); + } + return filter; + } + + private ProfilePhaseStatistics formatInitPhaseStatistics(ProfileInfo info, ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Init", ProfilePhase.INIT); + } + + private ProfilePhaseStatistics formatLoadingPhaseStatistics(ProfileInfo info, ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Loading", ProfilePhase.LOAD); + } + + private ProfilePhaseStatistics formatAnalysisPhaseStatistics(ProfileInfo info, + ProfileOptions opts) + throws UnsupportedEncodingException { + return formatSimplePhaseStatistics(info, opts, "Analysis", ProfilePhase.ANALYZE); + } + + private ProfilePhaseStatistics formatSimplePhaseStatistics(ProfileInfo info, + ProfileOptions opts, + String name, + ProfilePhase phase) + throws UnsupportedEncodingException { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + + printShortPhaseAnalysis(info, out, phase); + printVfsStatistics(info, out, phase, opts.vfsStatsLimit); + return new ProfilePhaseStatistics(name + " Phase Information", + new String(byteOutput.toByteArray(), "UTF-8")); + } + + private ProfilePhaseStatistics formatExecutionPhaseStatistics(ProfileInfo info, + ProfileOptions opts) + throws UnsupportedEncodingException { + ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(byteOutput, false, "UTF-8"); + + ProfileInfo.Task prepPhase = info.getPhaseTask(ProfilePhase.PREPARE); + ProfileInfo.Task execPhase = info.getPhaseTask(ProfilePhase.EXECUTE); + ProfileInfo.Task finishPhase = info.getPhaseTask(ProfilePhase.FINISH); + if (execPhase == null) { + return null; + } + + List<ProfileInfo.Task> execTasks = info.getTasksForPhase(execPhase); + long graphTime = info.getStatsForType(ProfilerTask.ACTION_GRAPH, execTasks).totalTime; + long execTime = info.getPhaseDuration(execPhase) - graphTime; + + if (prepPhase != null) { + out.printf(TWO_COLUMN_FORMAT, "Total preparation time", + TimeUtilities.prettyTime(info.getPhaseDuration(prepPhase))); + } + out.printf(TWO_COLUMN_FORMAT, "Total execution phase time", + TimeUtilities.prettyTime(info.getPhaseDuration(execPhase))); + if (finishPhase != null) { + out.printf(TWO_COLUMN_FORMAT, "Total time finalizing build", + TimeUtilities.prettyTime(info.getPhaseDuration(finishPhase))); + } + out.println(""); + out.printf(TWO_COLUMN_FORMAT, "Action dependency map creation", + TimeUtilities.prettyTime(graphTime)); + out.printf(TWO_COLUMN_FORMAT, "Actual execution time", + TimeUtilities.prettyTime(execTime)); + + EnumSet<ProfilerTask> typeFilter = EnumSet.noneOf(ProfilerTask.class); + CriticalPathEntry totalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, totalPath); + + typeFilter = getTypeFilter(); + CriticalPathEntry optimalPath = info.getCriticalPath(typeFilter); + info.analyzeCriticalPath(typeFilter, optimalPath); + + if (totalPath != null) { + printCriticalPathTimingBreakdown(info, totalPath, optimalPath, execTime, out); + } else { + out.println("\nCritical path not available because no action graph was generated."); + } + + printTimeDistributionByType(info, out, execPhase); + + if (totalPath != null) { + printCriticalPath("Critical path", out, totalPath); + // In light critical path we do not record scheduling delay data so it does not make sense + // to differentiate it. + if (!isLightCriticalPath(totalPath)) { + printCriticalPath("Critical path excluding scheduling delays", out, optimalPath); + } + } + + if (info.getMissingActionsCount() > 0) { + out.println("\n" + info.getMissingActionsCount() + " action(s) are present in the" + + " action graph but missing instrumentation data. Most likely profile file" + + " has been created for the failed or aborted build."); + } + + printVfsStatistics(info, out, ProfilePhase.EXECUTE, opts.vfsStatsLimit); + + return new ProfilePhaseStatistics("Execution Phase Information", + new String(byteOutput.toByteArray(), "UTF-8")); + } + + void printCriticalPathTimingBreakdown(ProfileInfo info, CriticalPathEntry totalPath, + CriticalPathEntry optimalPath, long execTime, PrintStream out) { + Preconditions.checkNotNull(totalPath); + Preconditions.checkNotNull(optimalPath); + // TODO(bazel-team): Print remote vs build stats recorded by CriticalPathStats + if (isLightCriticalPath(totalPath)) { + return; + } + out.println(totalPath.task.type); + // Worker thread pool scheduling delays for the actual critical path. + long workerWaitTime = 0; + long mainThreadWaitTime = 0; + for (ProfileInfo.CriticalPathEntry entry = totalPath; entry != null; entry = entry.next) { + workerWaitTime += info.getActionWaitTime(entry.task); + mainThreadWaitTime += info.getActionQueueTime(entry.task); + } + out.printf(TWO_COLUMN_FORMAT, "Worker thread scheduling delays", + TimeUtilities.prettyTime(workerWaitTime)); + out.printf(TWO_COLUMN_FORMAT, "Main thread scheduling delays", + TimeUtilities.prettyTime(mainThreadWaitTime)); + + out.println("\nCritical path time:"); + // Actual critical path. + long totalTime = totalPath.cumulativeDuration; + out.printf("%-37s %10s (%s of execution time)\n", "Actual time", + TimeUtilities.prettyTime(totalTime), + prettyPercentage(totalTime, execTime)); + // Unlimited resource critical path. Essentially, we assume that if we + // remove all scheduling delays caused by resource semaphore contention, + // each action execution time would not change (even though load now would + // be substantially higher - so this assumption might be incorrect but it is + // still useful for modeling). Given those assumptions we calculate critical + // path excluding scheduling delays. + long optimalTime = optimalPath.cumulativeDuration; + out.printf("%-37s %10s (%s of execution time)\n", "Time excluding scheduling delays", + TimeUtilities.prettyTime(optimalTime), + prettyPercentage(optimalTime, execTime)); + + // Artificial critical path if we ignore all the time spent in all tasks, + // except time directly attributed to the ACTION tasks. + out.println("\nTime related to:"); + + EnumSet<ProfilerTask> typeFilter = EnumSet.allOf(ProfilerTask.class); + ProfileInfo.CriticalPathEntry path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the builder overhead", + prettyPercentage(path.cumulativeDuration, totalTime)); + + typeFilter = getTypeFilter(); + for (ProfilerTask task : ProfilerTask.values()) { + if (task.name().startsWith("VFS_")) { + typeFilter.add(task); + } + } + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the VFS calls", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.ACTION_CHECK); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the dependency checking", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.ACTION_EXECUTE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the execution setup", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.SPAWN, ProfilerTask.LOCAL_EXECUTION); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "local execution", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.SCANNER); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "the include scanner", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION, ProfilerTask.PROCESS_TIME, + ProfilerTask.LOCAL_PARSE, ProfilerTask.UPLOAD_TIME, + ProfilerTask.REMOTE_QUEUE, ProfilerTask.REMOTE_SETUP, ProfilerTask.FETCH); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, "Remote execution (cumulative)", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter( ProfilerTask.UPLOAD_TIME, ProfilerTask.REMOTE_SETUP); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " file uploads", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.FETCH); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " file fetching", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.PROCESS_TIME); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " process time", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_QUEUE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " remote queueing", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.LOCAL_PARSE); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " remote execution parse", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + + typeFilter = getTypeFilter(ProfilerTask.REMOTE_EXECUTION); + path = info.getCriticalPath(typeFilter); + out.printf(TWO_COLUMN_FORMAT, " other remote activities", + prettyPercentage(optimalTime - path.cumulativeDuration, optimalTime)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java new file mode 100644 index 0000000000..2e5faf6ca8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ProjectFileSupport.java @@ -0,0 +1,93 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.CommonCommandOptions; +import com.google.devtools.build.lib.runtime.ProjectFile; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.List; + +/** + * Provides support for implementations for {@link BlazeCommand} to work with {@link ProjectFile}. + */ +public final class ProjectFileSupport { + static final String PROJECT_FILE_PREFIX = "+"; + + private ProjectFileSupport() {} + + /** + * Reads any project files specified on the command line and updates the options parser + * accordingly. If project files cannot be read or if they contain unparsable options, or if they + * are not enabled, then it throws an exception instead. + */ + public static void handleProjectFiles(BlazeRuntime runtime, OptionsParser optionsParser, + String command) throws AbruptExitException { + List<String> targets = optionsParser.getResidue(); + ProjectFile.Provider projectFileProvider = runtime.getProjectFileProvider(); + if (projectFileProvider != null && targets.size() > 0 + && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) { + if (targets.size() > 1) { + throw new AbruptExitException("Cannot handle more than one +<file> argument yet", + ExitCode.COMMAND_LINE_ERROR); + } + if (!optionsParser.getOptions(CommonCommandOptions.class).allowProjectFiles) { + throw new AbruptExitException("project file support is not enabled", + ExitCode.COMMAND_LINE_ERROR); + } + // TODO(bazel-team): This is currently treated as a path relative to the workspace - if the + // cwd is a subdirectory of the workspace, that will be surprising, and we should interpret it + // relative to the cwd instead. + PathFragment projectFilePath = new PathFragment(targets.get(0).substring(1)); + List<Path> packagePath = PathPackageLocator.create( + optionsParser.getOptions(PackageCacheOptions.class).packagePath, runtime.getReporter(), + runtime.getWorkspace(), runtime.getWorkingDirectory()).getPathEntries(); + ProjectFile projectFile = projectFileProvider.getProjectFile(packagePath, projectFilePath); + runtime.getReporter().handle(Event.info("Using " + projectFile.getName())); + + try { + optionsParser.parse( + OptionPriority.RC_FILE, projectFile.getName(), projectFile.getCommandLineFor(command)); + } catch (OptionsParsingException e) { + throw new AbruptExitException(e.getMessage(), ExitCode.COMMAND_LINE_ERROR); + } + } + } + + /** + * Returns a list of targets from the options residue. If a project file is supplied as the first + * argument, it will be ignored, on the assumption that handleProjectFiles() has been called to + * process it. + */ + public static List<String> getTargets(BlazeRuntime runtime, OptionsProvider options) { + List<String> targets = options.getResidue(); + if (runtime.getProjectFileProvider() != null && targets.size() > 0 + && targets.get(0).startsWith(PROJECT_FILE_PREFIX)) { + return targets.subList(1, targets.size()); + } + return targets; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java new file mode 100644 index 0000000000..c5120cb74b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/QueryCommand.java @@ -0,0 +1,173 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; +import com.google.devtools.build.lib.query2.BlazeQueryEnvironment; +import com.google.devtools.build.lib.query2.SkyframeQueryEnvironment; +import com.google.devtools.build.lib.query2.engine.BlazeQueryEvalResult; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.QueryFunction; +import com.google.devtools.build.lib.query2.engine.QueryEnvironment.Setting; +import com.google.devtools.build.lib.query2.engine.QueryException; +import com.google.devtools.build.lib.query2.engine.QueryExpression; +import com.google.devtools.build.lib.query2.output.OutputFormatter; +import com.google.devtools.build.lib.query2.output.OutputFormatter.UnorderedFormatter; +import com.google.devtools.build.lib.query2.output.QueryOptions; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.channels.ClosedByInterruptException; +import java.util.Set; + +/** + * Command line wrapper for executing a query with blaze. + */ +@Command(name = "query", + options = { PackageCacheOptions.class, + QueryOptions.class }, + help = "resource:query.txt", + shortDescription = "Executes a dependency graph query.", + allowResidue = true, + binaryStdOut = true, + canRunInOutputDirectory = true) +public final class QueryCommand implements BlazeCommand { + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + /** + * Exit codes: + * 0 on successful evaluation. + * 1 if query evaluation did not complete. + * 2 if query parsing failed. + * 3 if errors were reported but evaluation produced a partial result + * (only when --keep_going is in effect.) + */ + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + QueryOptions queryOptions = options.getOptions(QueryOptions.class); + + try { + runtime.setupPackageCache( + options.getOptions(PackageCacheOptions.class), + runtime.getDefaultsPackageContent()); + } catch (InterruptedException e) { + runtime.getReporter().handle(Event.error("query interrupted")); + return ExitCode.INTERRUPTED; + } catch (AbruptExitException e) { + runtime.getReporter().handle(Event.error(null, "Unknown error: " + e.getMessage())); + return e.getExitCode(); + } + + if (options.getResidue().isEmpty()) { + runtime.getReporter().handle(Event.error( + "missing query expression. Type 'blaze help query' for syntax and help")); + return ExitCode.COMMAND_LINE_ERROR; + } + + Iterable<OutputFormatter> formatters = runtime.getQueryOutputFormatters(); + OutputFormatter formatter = + OutputFormatter.getFormatter(formatters, queryOptions.outputFormat); + if (formatter == null) { + runtime.getReporter().handle(Event.error( + String.format("Invalid output format '%s'. Valid values are: %s", + queryOptions.outputFormat, OutputFormatter.formatterNames(formatters)))); + return ExitCode.COMMAND_LINE_ERROR; + } + + String query = Joiner.on(' ').join(options.getResidue()); + + Set<Setting> settings = queryOptions.toSettings(); + BlazeQueryEnvironment env = newQueryEnvironment( + runtime, + queryOptions.keepGoing, + queryOptions.loadingPhaseThreads, + settings); + + // 1. Parse query: + QueryExpression expr; + try { + expr = QueryExpression.parse(query, env); + } catch (QueryException e) { + runtime.getReporter().handle(Event.error( + null, "Error while parsing '" + query + "': " + e.getMessage())); + return ExitCode.COMMAND_LINE_ERROR; + } + + // 2. Evaluate expression: + BlazeQueryEvalResult<Target> result; + try { + result = env.evaluateQuery(expr); + } catch (QueryException e) { + // Keep consistent with reportBuildFileError() + runtime.getReporter().handle(Event.error(e.getMessage())); + return ExitCode.ANALYSIS_FAILURE; + } + + // 3. Output results: + OutputFormatter.UnorderedFormatter unorderedFormatter = null; + if (!queryOptions.orderResults && formatter instanceof UnorderedFormatter) { + unorderedFormatter = (UnorderedFormatter) formatter; + } + + PrintStream output = new PrintStream(runtime.getReporter().getOutErr().getOutputStream()); + try { + if (unorderedFormatter != null) { + unorderedFormatter.outputUnordered(queryOptions, result.getResultSet(), output); + } else { + formatter.output(queryOptions, result.getResultGraph(), output); + } + } catch (ClosedByInterruptException e) { + runtime.getReporter().handle(Event.error("query interrupted")); + return ExitCode.INTERRUPTED; + } catch (IOException e) { + runtime.getReporter().handle(Event.error("I/O error: " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } finally { + output.flush(); + } + if (result.getResultSet().isEmpty()) { + runtime.getReporter().handle(Event.info("Empty results")); + } + + return result.getSuccess() ? ExitCode.SUCCESS : ExitCode.PARTIAL_ANALYSIS_FAILURE; + } + + @VisibleForTesting // for com.google.devtools.deps.gquery.test.QueryResultTestUtil + public static BlazeQueryEnvironment newQueryEnvironment(BlazeRuntime runtime, + boolean keepGoing, int loadingPhaseThreads, Set<Setting> settings) { + ImmutableList.Builder<QueryFunction> functions = ImmutableList.builder(); + for (BlazeModule module : runtime.getBlazeModules()) { + functions.addAll(module.getQueryFunctions()); + } + return new SkyframeQueryEnvironment( + runtime.getPackageManager().newTransitiveLoader(), + runtime.getPackageManager(), + runtime.getTargetPatternEvaluator(), + keepGoing, loadingPhaseThreads, runtime.getReporter(), settings, functions.build()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java new file mode 100644 index 0000000000..b128d372ab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java @@ -0,0 +1,519 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FilesToRunProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.RunUnder; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.BuildResult; +import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; +import com.google.devtools.build.lib.buildtool.TargetValidator; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.SymlinkTreeHelper; +import com.google.devtools.build.lib.packages.NonconfigurableAttributeMapper; +import com.google.devtools.build.lib.packages.OutputFile; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.pkgcache.LoadingFailedException; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.shell.AbnormalTerminationException; +import com.google.devtools.build.lib.shell.BadExitStatusException; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.CommandBuilder; +import com.google.devtools.build.lib.util.CommandDescriptionForm; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Builds and run a target with the given command line arguments. + */ +@Command(name = "run", + builds = true, + options = { RunCommand.RunOptions.class }, + inherits = { BuildCommand.class }, + shortDescription = "Runs the specified target.", + help = "resource:run.txt", + allowResidue = true, + binaryStdOut = true, + binaryStdErr = true) +public class RunCommand implements BlazeCommand { + + public static class RunOptions extends OptionsBase { + @Option(name = "script_path", + category = "run", + defaultValue = "null", + converter = OptionsUtils.PathFragmentConverter.class, + help = "If set, write a shell script to the given file which invokes the " + + "target. If this option is set, the target is not run from Blaze. " + + "Use 'blaze run --script_path=foo //foo && foo' to invoke target '//foo' " + + "This differs from 'blaze run //foo' in that the Blaze lock is released " + + "and the executable is connected to the terminal's stdin.") + public PathFragment scriptPath; + } + + @VisibleForTesting + public static final String SINGLE_TARGET_MESSAGE = "Blaze can only run a single target. " + + "Do not use wildcards that match more than one target"; + @VisibleForTesting + public static final String NO_TARGET_MESSAGE = "No targets found to run"; + + private static final String PROCESS_WRAPPER = "process-wrapper"; + + // Value of --run_under as of the most recent command invocation. + private RunUnder currentRunUnder; + + private static final FileType RUNFILES_MANIFEST = FileType.of(".runfiles_manifest"); + + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + protected BuildResult processRequest(final BlazeRuntime runtime, BuildRequest request) { + return runtime.getBuildTool().processRequest(request, new TargetValidator() { + @Override + public void validateTargets(Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException { + RunCommand.this.validateTargets(runtime.getReporter(), targets, keepGoing); + } + }); + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) { } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + RunOptions runOptions = options.getOptions(RunOptions.class); + // This list should look like: ["//executable:target", "arg1", "arg2"] + List<String> targetAndArgs = options.getResidue(); + + // The user must at the least specify an executable target. + if (targetAndArgs.isEmpty()) { + runtime.getReporter().handle(Event.error("Must specify a target to run")); + return ExitCode.COMMAND_LINE_ERROR; + } + String targetString = targetAndArgs.get(0); + List<String> runTargetArgs = targetAndArgs.subList(1, targetAndArgs.size()); + RunUnder runUnder = options.getOptions(BuildConfiguration.Options.class).runUnder; + + OutErr outErr = runtime.getReporter().getOutErr(); + List<String> targets = (runUnder != null) && (runUnder.getLabel() != null) + ? ImmutableList.of(targetString, runUnder.getLabel().toString()) + : ImmutableList.of(targetString); + BuildRequest request = BuildRequest.create( + this.getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), targets, outErr, + runtime.getCommandId(), runtime.getCommandStartTime()); + if (request.getBuildOptions().compileOnly) { + String message = "The '" + getClass().getAnnotation(Command.class).name() + + "' command is incompatible with the --compile_only option"; + runtime.getReporter().handle(Event.error(message)); + return ExitCode.COMMAND_LINE_ERROR; + } + + currentRunUnder = runUnder; + BuildResult result; + try { + result = processRequest(runtime, request); + } finally { + currentRunUnder = null; + } + + if (!result.getSuccess()) { + runtime.getReporter().handle(Event.error("Build failed. Not running target")); + return result.getExitCondition(); + } + + // Make sure that we have exactly 1 built target (excluding --run_under), + // and that it is executable. + // These checks should only fail if keepGoing is true, because we already did + // validation before the build began. See {@link #validateTargets()}. + Collection<ConfiguredTarget> targetsBuilt = result.getSuccessfulTargets(); + ConfiguredTarget targetToRun = null; + ConfiguredTarget runUnderTarget = null; + + if (targetsBuilt != null) { + int maxTargets = runUnder != null && runUnder.getLabel() != null ? 2 : 1; + if (targetsBuilt.size() > maxTargets) { + runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + for (ConfiguredTarget target : targetsBuilt) { + ExitCode targetValidation = fullyValidateTarget(runtime, target); + if (targetValidation != ExitCode.SUCCESS) { + return targetValidation; + } + if (runUnder != null && target.getLabel().equals(runUnder.getLabel())) { + if (runUnderTarget != null) { + runtime.getReporter().handle(Event.error( + null, "Can't identify the run_under target from multiple options?")); + return ExitCode.COMMAND_LINE_ERROR; + } + runUnderTarget = target; + } else if (targetToRun == null) { + targetToRun = target; + } else { + runtime.getReporter().handle(Event.error(SINGLE_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + } + } + // Handle target & run_under referring to the same target. + if ((targetToRun == null) && (runUnderTarget != null)) { + targetToRun = runUnderTarget; + } + if (targetToRun == null) { + runtime.getReporter().handle(Event.error(NO_TARGET_MESSAGE)); + return ExitCode.COMMAND_LINE_ERROR; + } + + Path executablePath = Preconditions.checkNotNull( + targetToRun.getProvider(FilesToRunProvider.class).getExecutable().getPath()); + BuildConfiguration configuration = targetToRun.getConfiguration(); + if (configuration == null) { + // The target may be an input file, which doesn't have a configuration. In that case, we + // choose any target configuration. + configuration = runtime.getBuildTool().getView().getConfigurationCollection() + .getTargetConfigurations().get(0); + } + Path workingDir; + try { + workingDir = ensureRunfilesBuilt(runtime, targetToRun); + } catch (CommandException e) { + runtime.getReporter().handle(Event.error("Error creating runfiles: " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + + List<String> args = runTargetArgs; + + FilesToRunProvider provider = targetToRun.getProvider(FilesToRunProvider.class); + RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport(); + if (runfilesSupport != null && runfilesSupport.getArgs() != null) { + List<String> targetArgs = runfilesSupport.getArgs(); + if (!targetArgs.isEmpty()) { + args = Lists.newArrayListWithCapacity(targetArgs.size() + runTargetArgs.size()); + args.addAll(targetArgs); + args.addAll(runTargetArgs); + } + } + + // + // We now have a unique executable ready to be run. + // + // We build up two different versions of the command to run: one with an absolute path, which + // we'll actually run, and a prettier one with the long absolute path to the executable + // replaced with a shorter relative path that uses the symlinks in the workspace. + PathFragment prettyExecutablePath = + OutputDirectoryLinksUtils.getPrettyPath(executablePath, + runtime.getWorkspaceName(), runtime.getWorkspace(), + options.getOptions(BuildRequestOptions.class).symlinkPrefix); + List<String> cmdLine = new ArrayList<>(); + if (runOptions.scriptPath == null) { + cmdLine.add(runtime.getDirectories().getExecRoot() + .getRelative(runtime.getBinTools().getExecPath(PROCESS_WRAPPER)).getPathString()); + cmdLine.add("-1"); + cmdLine.add("15"); + cmdLine.add("-"); + cmdLine.add("-"); + } + List<String> prettyCmdLine = new ArrayList<>(); + // Insert the command prefix specified by the "--run_under=<command-prefix>" option + // at the start of the command line. + if (runUnder != null) { + String runUnderValue = runUnder.getValue(); + if (runUnderTarget != null) { + // --run_under specifies a target. Get the corresponding executable. + // This must be an absolute path, because the run_under target is only + // in the runfiles of test targets. + runUnderValue = runUnderTarget + .getProvider(FilesToRunProvider.class).getExecutable().getPath().getPathString(); + // If the run_under command contains any options, make sure to add them + // to the command line as well. + List<String> opts = runUnder.getOptions(); + if (!opts.isEmpty()) { + runUnderValue += " " + ShellEscaper.escapeJoinAll(opts); + } + } + cmdLine.add(configuration.getShExecutable().getPathString()); + cmdLine.add("-c"); + cmdLine.add(runUnderValue + " " + executablePath.getPathString() + " " + + ShellEscaper.escapeJoinAll(args)); + prettyCmdLine.add(configuration.getShExecutable().getPathString()); + prettyCmdLine.add("-c"); + prettyCmdLine.add(runUnderValue + " " + prettyExecutablePath.getPathString() + " " + + ShellEscaper.escapeJoinAll(args)); + } else { + cmdLine.add(executablePath.getPathString()); + cmdLine.addAll(args); + prettyCmdLine.add(prettyExecutablePath.getPathString()); + prettyCmdLine.addAll(args); + } + + // Add a newline between the blaze output and the binary's output. + outErr.printErrLn(""); + + if (runOptions.scriptPath != null) { + String unisolatedCommand = CommandFailureUtils.describeCommand( + CommandDescriptionForm.COMPLETE_UNISOLATED, + cmdLine, null, workingDir.getPathString()); + if (writeScript(runtime, runOptions.scriptPath, unisolatedCommand)) { + return ExitCode.SUCCESS; + } else { + return ExitCode.RUN_FAILURE; + } + } + + runtime.getReporter().handle(Event.info( + null, "Running command line: " + ShellEscaper.escapeJoinAll(prettyCmdLine))); + + com.google.devtools.build.lib.shell.Command command = new CommandBuilder() + .addArgs(cmdLine).setEnv(runtime.getClientEnv()).setWorkingDir(workingDir).build(); + + try { + // The command API is a little strange in that the following statement + // will return normally only if the program exits with exit code 0. + // If it ends with any other code, we have to catch BadExitStatusException. + command.execute(com.google.devtools.build.lib.shell.Command.NO_INPUT, + com.google.devtools.build.lib.shell.Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + true /* interruptible */).getTerminationStatus().getExitCode(); + return ExitCode.SUCCESS; + } catch (BadExitStatusException e) { + String message = "Non-zero return code '" + + e.getResult().getTerminationStatus().getExitCode() + + "' from command: " + e.getMessage(); + runtime.getReporter().handle(Event.error(message)); + return ExitCode.RUN_FAILURE; + } catch (AbnormalTerminationException e) { + // The process was likely terminated by a signal in this case. + return ExitCode.INTERRUPTED; + } catch (CommandException e) { + runtime.getReporter().handle(Event.error("Error running program: " + e.getMessage())); + return ExitCode.RUN_FAILURE; + } + } + + /** + * Ensures that runfiles are built for the specified target. If they already + * are, does nothing, otherwise builds them. + * + * @param target the target to build runfiles for. + * @return the path of the runfiles directory. + * @throws CommandException + */ + private Path ensureRunfilesBuilt(BlazeRuntime runtime, ConfiguredTarget target) + throws CommandException { + FilesToRunProvider provider = target.getProvider(FilesToRunProvider.class); + RunfilesSupport runfilesSupport = provider == null ? null : provider.getRunfilesSupport(); + if (runfilesSupport == null) { + return runtime.getWorkingDirectory(); + } + + Artifact manifest = runfilesSupport.getRunfilesManifest(); + PathFragment runfilesDir = runfilesSupport.getRunfilesDirectoryExecPath(); + Path workingDir = runtime.getExecRoot() + .getRelative(runfilesDir) + .getRelative(runtime.getRunfilesPrefix()); + + // When runfiles are not generated, getManifest() returns the + // .runfiles_manifest file, otherwise it returns the MANIFEST file. This is + // a handy way to check whether runfiles were built or not. + if (!RUNFILES_MANIFEST.matches(manifest.getFilename())) { + // Runfiles already built, nothing to do. + return workingDir; + } + + SymlinkTreeHelper helper = new SymlinkTreeHelper( + manifest.getExecPath(), + runfilesDir, + false); + helper.createSymlinksUsingCommand(runtime.getExecRoot(), target.getConfiguration(), + runtime.getBinTools()); + return workingDir; + } + + private boolean writeScript(BlazeRuntime runtime, PathFragment scriptPathFrag, String cmd) { + final String SH_SHEBANG = "#!/bin/sh"; + Path scriptPath = runtime.getWorkingDirectory().getRelative(scriptPathFrag); + try { + FileSystemUtils.writeContent(scriptPath, StandardCharsets.ISO_8859_1, + SH_SHEBANG + "\n" + cmd + " \"$@\""); + scriptPath.setExecutable(true); + } catch (IOException e) { + runtime.getReporter().handle(Event.error("Error writing run script:" + e.getMessage())); + return false; + } + return true; + } + + // Make sure we are building exactly 1 binary target. + // If keepGoing, we'll build all the targets even if they are non-binary. + private void validateTargets(Reporter reporter, Collection<Target> targets, boolean keepGoing) + throws LoadingFailedException { + Target targetToRun = null; + Target runUnderTarget = null; + + boolean singleTargetWarningWasOutput = false; + int maxTargets = currentRunUnder != null && currentRunUnder.getLabel() != null ? 2 : 1; + if (targets.size() > maxTargets) { + warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing); + singleTargetWarningWasOutput = true; + } + for (Target target : targets) { + String targetError = validateTarget(target); + if (targetError != null) { + warningOrException(reporter, targetError, keepGoing); + } + + if (currentRunUnder != null && target.getLabel().equals(currentRunUnder.getLabel())) { + // It's impossible to have two targets with the same label. + Preconditions.checkState(runUnderTarget == null); + runUnderTarget = target; + } else if (targetToRun == null) { + targetToRun = target; + } else { + if (!singleTargetWarningWasOutput) { + warningOrException(reporter, SINGLE_TARGET_MESSAGE, keepGoing); + } + return; + } + } + // Handle target & run_under referring to the same target. + if ((targetToRun == null) && (runUnderTarget != null)) { + targetToRun = runUnderTarget; + } + if (targetToRun == null) { + warningOrException(reporter, NO_TARGET_MESSAGE, keepGoing); + } + } + + // If keepGoing, print a warning and return the given collection. + // Otherwise, throw InvalidTargetException. + private void warningOrException(Reporter reporter, String message, + boolean keepGoing) throws LoadingFailedException { + if (keepGoing) { + reporter.handle(Event.warn(message + ". Will continue anyway")); + } else { + throw new LoadingFailedException(message); + } + } + + private static String notExecutableError(Target target) { + return "Cannot run target " + target.getLabel() + ": Not executable"; + } + + /** Returns null if the target is a runnable rule, or an appropriate error message otherwise. */ + private static String validateTarget(Target target) { + return isExecutable(target) + ? null + : notExecutableError(target); + } + + /** + * Performs all available validation checks on an individual target. + * + * @param target ConfiguredTarget to validate + * @return ExitCode.SUCCESS if all checks succeeded, otherwise a different error code. + */ + private ExitCode fullyValidateTarget(BlazeRuntime runtime, ConfiguredTarget target) { + String targetError = validateTarget(target.getTarget()); + + if (targetError != null) { + runtime.getReporter().handle(Event.error(targetError)); + return ExitCode.COMMAND_LINE_ERROR; + } + + Artifact executable = target.getProvider(FilesToRunProvider.class).getExecutable(); + if (executable == null) { + runtime.getReporter().handle(Event.error(notExecutableError(target.getTarget()))); + return ExitCode.COMMAND_LINE_ERROR; + } + + // Shouldn't happen: We just validated the target. + Preconditions.checkState(executable != null, + "Could not find executable for target %s", target); + Path executablePath = executable.getPath(); + try { + if (!executablePath.exists() || !executablePath.isExecutable()) { + runtime.getReporter().handle(Event.error( + null, "Non-existent or non-executable " + executablePath)); + return ExitCode.BLAZE_INTERNAL_ERROR; + } + } catch (IOException e) { + runtime.getReporter().handle(Event.error( + "Error checking " + executablePath.getPathString() + ": " + e.getMessage())); + return ExitCode.LOCAL_ENVIRONMENTAL_ERROR; + } + + return ExitCode.SUCCESS; + } + + /** + * Return true iff {@code target} is a rule that has an executable file. This includes + * *_test rules, *_binary rules, and generated outputs. + */ + private static boolean isExecutable(Target target) { + return isOutputFile(target) || isExecutableNonTestRule(target) + || TargetUtils.isTestRule(target); + } + + /** + * Return true iff {@code target} is a rule that generates an executable file and is user-executed + * code. + */ + private static boolean isExecutableNonTestRule(Target target) { + if (!(target instanceof Rule)) { + return false; + } + Rule rule = ((Rule) target); + if (rule.getRuleClassObject().hasAttr("$is_executable", Type.BOOLEAN)) { + return NonconfigurableAttributeMapper.of(rule).get("$is_executable", Type.BOOLEAN); + } + return false; + } + + private static boolean isOutputFile(Target target) { + return (target instanceof OutputFile); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java new file mode 100644 index 0000000000..fb9ba39f37 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/ShutdownCommand.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * The 'blaze shutdown' command. + */ +@Command(name = "shutdown", + options = { ShutdownCommand.Options.class }, + allowResidue = false, + mustRunInWorkspace = false, + shortDescription = "Stops the Blaze server.", + help = "This command shuts down the memory resident Blaze server process.\n%{options}") +public final class ShutdownCommand implements BlazeCommand { + + public static class Options extends OptionsBase { + + @Option(name="iff_heap_size_greater_than", + defaultValue = "0", + category = "misc", + help="Iff non-zero, then shutdown will only shut down the " + + "server if the total memory (in MB) consumed by the JVM " + + "exceeds this value.") + public int heapSizeLimit; + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + + int limit = options.getOptions(Options.class).heapSizeLimit; + + // Iff limit is non-zero, shut down the server if total memory exceeds the + // limit. totalMemory is the actual heap size that the VM currently uses + // *from the OS perspective*. That is, it's not the size occupied by all + // objects (which is totalMemory() - freeMemory()), and not the -Xmx + // (which is maxMemory()). It's really how much memory this process + // currently consumes, in addition to the JVM code and C heap. + + if (limit == 0 || + Runtime.getRuntime().totalMemory() > limit * 1000L * 1000) { + throw new ShutdownBlazeServerException(0); + } + return ExitCode.SUCCESS; + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java new file mode 100644 index 0000000000..70082ef8dd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/SkylarkCommand.java @@ -0,0 +1,82 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.docgen.SkylarkDocumentationProcessor; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.OutErr; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Map; + +/** + * The 'doc_ext' command, which prints the extension API doc. + */ +@Command(name = "doc_ext", +allowResidue = true, +mustRunInWorkspace = false, +shortDescription = "Prints help for commands, or the index.", +help = "resource:skylark.txt") +public final class SkylarkCommand implements BlazeCommand { + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) + throws ShutdownBlazeServerException { + OutErr outErr = runtime.getReporter().getOutErr(); + if (options.getResidue().isEmpty()) { + printTopLevelAPIDoc(outErr); + return ExitCode.SUCCESS; + } + if (options.getResidue().size() != 1) { + runtime.getReporter().handle(Event.error("Cannot specify more than one parameters")); + return ExitCode.COMMAND_LINE_ERROR; + } + return printAPIDoc(options.getResidue().get(0), outErr, runtime.getReporter()); + } + + private ExitCode printAPIDoc(String param, OutErr outErr, Reporter reporter) { + String params[] = param.split("\\."); + if (params.length > 2) { + reporter.handle(Event.error("Identifier not found: " + param)); + return ExitCode.COMMAND_LINE_ERROR; + } + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + String doc = processor.getCommandLineAPIDoc(params); + if (doc == null) { + reporter.handle(Event.error("Identifier not found: " + param)); + return ExitCode.COMMAND_LINE_ERROR; + } + outErr.printOut(doc); + return ExitCode.SUCCESS; + } + + private void printTopLevelAPIDoc(OutErr outErr) { + SkylarkDocumentationProcessor processor = new SkylarkDocumentationProcessor(); + outErr.printOut("Top level language modules, methods and objects:\n\n"); + for (Map.Entry<String, String> entry : processor.collectTopLevelModules().entrySet()) { + outErr.printOut(entry.getKey() + ": " + entry.getValue()); + } + } + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java new file mode 100644 index 0000000000..561c54a5b2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/TestCommand.java @@ -0,0 +1,161 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.runtime.commands; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildResult; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.rules.test.TestStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy.TestOutputFormat; +import com.google.devtools.build.lib.runtime.AggregatingTestListener; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier; +import com.google.devtools.build.lib.runtime.TerminalTestResultNotifier.TestSummaryOptions; +import com.google.devtools.build.lib.runtime.TestResultAnalyzer; +import com.google.devtools.build.lib.runtime.TestResultNotifier; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.AnsiTerminalPrinter; +import com.google.devtools.common.options.OptionPriority; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsProvider; + +import java.util.Collection; +import java.util.List; + +/** + * Handles the 'test' command on the Blaze command line. + */ +@Command(name = "test", + builds = true, + inherits = { BuildCommand.class }, + options = { TestSummaryOptions.class }, + shortDescription = "Builds and runs the specified test targets.", + help = "resource:test.txt", + allowResidue = true) +public class TestCommand implements BlazeCommand { + private AnsiTerminalPrinter printer; + + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) + throws AbruptExitException { + ProjectFileSupport.handleProjectFiles(runtime, optionsParser, "test"); + + TestOutputFormat testOutput = optionsParser.getOptions(ExecutionOptions.class).testOutput; + + if (testOutput == TestStrategy.TestOutputFormat.STREAMED) { + runtime.getReporter().handle(Event.warn( + "Streamed test output requested so all tests will be run locally, without sharding, " + + "one at a time")); + try { + optionsParser.parse(OptionPriority.SOFTWARE_REQUIREMENT, + "streamed output requires locally run tests, without sharding", + ImmutableList.of("--test_sharding_strategy=disabled", "--test_strategy=exclusive")); + } catch (OptionsParsingException e) { + throw new IllegalStateException("Known options failed to parse", e); + } + } + } + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + TestResultAnalyzer resultAnalyzer = new TestResultAnalyzer( + runtime.getExecRoot(), + options.getOptions(TestSummaryOptions.class), + options.getOptions(ExecutionOptions.class), + runtime.getEventBus()); + + printer = new AnsiTerminalPrinter(runtime.getReporter().getOutErr().getOutputStream(), + options.getOptions(BlazeCommandEventHandler.Options.class).useColor()); + + // Initialize test handler. + AggregatingTestListener testListener = new AggregatingTestListener( + resultAnalyzer, runtime.getEventBus(), runtime.getReporter()); + + runtime.getEventBus().register(testListener); + return doTest(runtime, options, testListener); + } + + private ExitCode doTest(BlazeRuntime runtime, + OptionsProvider options, + AggregatingTestListener testListener) { + // Run simultaneous build and test. + List<String> targets = ProjectFileSupport.getTargets(runtime, options); + BuildRequest request = BuildRequest.create( + getClass().getAnnotation(Command.class).name(), options, + runtime.getStartupOptionsProvider(), targets, + runtime.getReporter().getOutErr(), runtime.getCommandId(), runtime.getCommandStartTime()); + if (request.getBuildOptions().compileOnly) { + String message = "The '" + getClass().getAnnotation(Command.class).name() + + "' command is incompatible with the --compile_only option"; + runtime.getReporter().handle(Event.error(message)); + return ExitCode.COMMAND_LINE_ERROR; + } + request.setRunTests(); + + BuildResult buildResult = runtime.getBuildTool().processRequest(request, null); + + Collection<ConfiguredTarget> testTargets = buildResult.getTestTargets(); + // TODO(bazel-team): don't handle isEmpty here or fix up a bunch of tests + if (buildResult.getSuccessfulTargets() == null) { + // This can happen if there were errors in the target parsing or loading phase + // (original exitcode=BUILD_FAILURE) or if there weren't but --noanalyze was given + // (original exitcode=SUCCESS). + runtime.getReporter().handle(Event.error("Couldn't start the build. Unable to run tests")); + return buildResult.getSuccess() ? ExitCode.PARSING_FAILURE : buildResult.getExitCondition(); + } + // TODO(bazel-team): the check above shadows NO_TESTS_FOUND, but switching the conditions breaks + // more tests + if (testTargets.isEmpty()) { + runtime.getReporter().handle(Event.error( + null, "No test targets were found, yet testing was requested")); + return buildResult.getSuccess() ? ExitCode.NO_TESTS_FOUND : buildResult.getExitCondition(); + } + + boolean buildSuccess = buildResult.getSuccess(); + boolean testSuccess = analyzeTestResults(testTargets, testListener, options); + + if (testSuccess && !buildSuccess) { + // If all tests run successfully, test summary should include warning if + // there were build errors not associated with the test targets. + printer.printLn(AnsiTerminalPrinter.Mode.ERROR + + "One or more non-test targets failed to build.\n" + + AnsiTerminalPrinter.Mode.DEFAULT); + } + + return buildSuccess ? + (testSuccess ? ExitCode.SUCCESS : ExitCode.TESTS_FAILED) + : buildResult.getExitCondition(); + } + + /** + * Analyzes test results and prints summary information. + * Returns true if and only if all tests were successful. + */ + private boolean analyzeTestResults(Collection<ConfiguredTarget> testTargets, + AggregatingTestListener listener, + OptionsProvider options) { + TestResultNotifier notifier = new TerminalTestResultNotifier(printer, options); + return listener.getAnalyzer().differentialAnalyzeAndReport( + testTargets, listener, notifier); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java new file mode 100644 index 0000000000..0804cf6cee --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/VersionCommand.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime.commands; + +import com.google.devtools.build.lib.analysis.BlazeVersionInfo; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.runtime.BlazeCommand; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsProvider; + +/** + * The 'blaze version' command, which informs users about the blaze version + * information. + */ +@Command(name = "version", + options = {}, + allowResidue = false, + mustRunInWorkspace = false, + help = "resource:version.txt", + shortDescription = "Prints version information for Blaze.") +public final class VersionCommand implements BlazeCommand { + @Override + public void editOptions(BlazeRuntime runtime, OptionsParser optionsParser) {} + + @Override + public ExitCode exec(BlazeRuntime runtime, OptionsProvider options) { + BlazeVersionInfo info = BlazeVersionInfo.instance(); + if (info.getSummary() == null) { + runtime.getReporter().handle(Event.error("Version information not available")); + return ExitCode.COMMAND_LINE_ERROR; + } + runtime.getReporter().getOutErr().printOutLn(info.getSummary()); + return ExitCode.SUCCESS; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt new file mode 100644 index 0000000000..0ef55a86f1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/analyze-profile.txt @@ -0,0 +1,14 @@ + +Usage: blaze %{command} <options> <profile-files> [<profile-file> ...] + +Analyzes build profile data for the given profile data files. + +Analyzes each specified profile data file and prints the results. The +input files must have been produced by the 'blaze build +--profile=file' command. + +By default, a summary of the analysis is printed. For post-processing +with scripts, the --dump=raw option is recommended, causing this +command to dump profile data in easily-parsed format. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt new file mode 100644 index 0000000000..5e8d88ae9c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/build.txt @@ -0,0 +1,10 @@ + +Usage: blaze %{command} <options> <targets> + +Builds the specified targets, using the options. + +See 'blaze help target-syntax' for details and examples on how to +specify targets to build. + +%{options} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt new file mode 100644 index 0000000000..11541ff354 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/canonicalize.txt @@ -0,0 +1,8 @@ + +Usage: blaze canonicalize-flags <options> -- <options-to-canonicalize> + +Canonicalizes Blaze flags for the test and build commands. This command is +intended to be used for tools that wish to check if two lists of options have +the same effect at runtime. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt new file mode 100644 index 0000000000..7633888ed6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/clean.txt @@ -0,0 +1,10 @@ + +Usage: blaze %{command} [<option> ...] + +Removes Blaze-created output, including all object files, and Blaze +metadata. + +If '--expunge' is specified, the entire working tree will be removed +and the server stopped. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt new file mode 100644 index 0000000000..a2040c81ca --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/help.txt @@ -0,0 +1,7 @@ + +Usage: blaze help [<command>] + +Prints a help page for the given command, or, if no command is +specified, prints the index of available commands. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt new file mode 100644 index 0000000000..9c8b552947 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/info.txt @@ -0,0 +1,23 @@ + +Usage: blaze info <options> [key] + +Displays information about the state of the blaze process in the +form of several "key: value" pairs. This includes the locations of +several output directories. Because some of the +values are affected by the options passed to 'blaze build', the +info command accepts the same set of options. + +A single non-option argument may be specified (e.g. "blaze-bin"), in +which case only the value for that key will be printed. + +If --show_make_env is specified, the output includes the set of key/value +pairs in the "Make" environment, accessible within BUILD files. + +The full list of keys and the meaning of their values is documented in +the Blaze User Manual, and can be programmatically obtained with +'blaze help info-keys'. + +See also 'blaze version' for more detailed blaze version +information. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt new file mode 100644 index 0000000000..ce10211cc9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/query.txt @@ -0,0 +1,19 @@ + +Usage: blaze %{command} <options> <query-expression> + +Executes a query language expression over a specified subgraph of the +build dependency graph. + +For example, to show all C++ test rules in the strings package, use: + + % blaze query 'kind("cc_.*test", strings:*)' + +or to find all dependencies of chubby lockserver, use: + + % blaze query 'deps(//path/to/package:target)' + +or to find a dependency path between //path/to/package:target and //dependency: + + % blaze query 'somepath(//path/to/package:target, //dependency)' + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt new file mode 100644 index 0000000000..57283d50d0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/run.txt @@ -0,0 +1,12 @@ + +Usage: blaze %{command} <options> -- <binary target> <flags to binary> + +Build the specified target and run it with the given arguments. + +'run' accepts any 'build' options, and will inherit any defaults +provided by .blazerc. + +If your script needs stdin or execution not constrained by the Blaze lock, +use 'blaze run --script_path' to write a script and then execute it. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt new file mode 100644 index 0000000000..5414707d0a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/startup_options.txt @@ -0,0 +1,14 @@ + +Startup options +=============== + +These options affect how Blaze starts up, or more specifically, how +the virtual machine hosting Blaze starts up, and how the Blaze server +starts up. These options must be specified to the left of the Blaze +command (e.g. 'build'), and they must not contain any space between +option name and value. + +Example: + % blaze --host_jvm_args=-Xmx1400m --output_base=/tmp/foo build //base + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt new file mode 100644 index 0000000000..1fac4981db --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/target-syntax.txt @@ -0,0 +1,64 @@ + +Target pattern syntax +===================== + +The BUILD file label syntax is used to specify a single target. Target +patterns generalize this syntax to sets of targets, and also support +working-directory-relative forms, recursion, subtraction and filtering. +Examples: + +Specifying a single target: + + //foo/bar:wiz The single target '//foo/bar:wiz'. + foo/bar/wiz Equivalent to the first existing one of these: + //foo/bar:wiz + //foo:bar/wiz + //foo/bar Equivalent to '//foo/bar:bar'. + +Specifying all rules in a package: + + //foo/bar:all Matches all rules in package 'foo/bar'. + +Specifying all rules recursively beneath a package: + + //foo/...:all Matches all rules in all packages beneath directory 'foo'. + //foo/... (ditto) + +Working-directory relative forms: (assume cwd = 'workspace/foo') + + Target patterns which do not begin with '//' are taken relative to + the working directory. Patterns which begin with '//' are always + absolute. + + ...:all Equivalent to '//foo/...:all'. + ... (ditto) + + bar/...:all Equivalent to '//foo/bar/...:all'. + bar/... (ditto) + + bar:wiz Equivalent to '//foo/bar:wiz'. + :foo Equivalent to '//foo:foo'. + + bar:all Equivalent to '//foo/bar:all'. + :all Equivalent to '//foo:all'. + +Summary of target wildcards: + + :all, Match all rules in the specified packages. + :*, :all-targets Match all targets (rules and files) in the specified + packages, including .par and _deploy.jar files. + +Subtractive patterns: + + Target patterns may be preceded by '-', meaning they should be + subtracted from the set of targets accumulated by preceding + patterns. For example: + + % blaze build -- foo/... -foo/contrib/... + + builds everything in 'foo', except 'contrib'. In case a target not + under 'contrib' depends on something under 'contrib' though, in order to + build the former blaze has to build the latter too. As usual, the '--' is + required to prevent '-b' from being interpreted as an option. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt new file mode 100644 index 0000000000..a1f0523e2b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/test.txt @@ -0,0 +1,15 @@ + +Usage: blaze %{command} <options> <test-targets> + +Builds the specified targets and runs all test targets among them (test targets +might also need to satisfy provided tag, size or language filters) using +the specified options. + +This command accepts all valid options to 'build', and inherits +defaults for 'build' from your .blazerc. If you don't use .blazerc, +don't forget to pass all your 'build' options to '%{command}' too. + +See 'blaze help target-syntax' for details and examples on how to +specify targets. + +%{options} diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt new file mode 100644 index 0000000000..10e1df76cf --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/version.txt @@ -0,0 +1,3 @@ +Prints the version information that was embedded when blaze was built. + +%{options} |