Add new option categorization and tagging information to HelpCommand's output.

If setting flag --use_new_category_enum, group the options by the new categories in both the command line output and the "everything-as-html" output used for the generated docs at https://bazel.build/versions/master/docs/command-line-reference.html. In the html output, the effect and metadata tags are listed for each option, with links to their descriptions at the bottom of the page. The tags only appear in the terminal output in -l/--long/--help_verbosity=long, and only the names appear.

This is still experimental as the majority of options do not yet use the new categorization system. The new output can be seen in command-line-reference.html format by adding the new flag to the "help everything-as-html" line in //src/main/java/com/google/devtools/build/lib:gen_command-line-reference.

The html output is in the same order as before (by blaze rule, with inherited options not repeated), which means it still has duplicate options, that should ideally be fixed separately. I propose either dropping the high-level grouping and just grouping the options by documentation category, or potentially grouping them by optionsbase in some non-class-naming way, and listing the commands that they apply to, as more and more optionsbases are used by multiple commands without being inherited (for example, all BuildEventServiceOptions are listed 20 times). People probably use ctrl-f to navigate this page anyway. Once we know that we only list each option once, we can actually have links to the options, which will make it possible to have links in the expansion lists.

Issue #3758

RELNOTES: added experimental --use_new_category_enum to the help command to output options grouped by the new type of category.
PiperOrigin-RevId: 170116553

1 files changed, 153 insertions, 51 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
index 8c3821bff6..8b1d9a8f03 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/HelpCommand.java
@@ -43,6 +43,8 @@ import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDefinition;
 import com.google.devtools.common.options.OptionDocumentationCategory;
 import com.google.devtools.common.options.OptionEffectTag;
+import com.google.devtools.common.options.OptionFilterDescriptions;
+import com.google.devtools.common.options.OptionMetadataTag;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsParser;
 import com.google.devtools.common.options.OptionsProvider;
@@ -113,15 +115,25 @@ public final class HelpCommand implements BlazeCommand {
       help = "Show only the names of the options, not their types or meanings."
     )
     public Void showShortFormOptions;
+
+    @Option(
+      name = "use_new_category_enum",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.LOGGING,
+      effectTags = {OptionEffectTag.AFFECTS_OUTPUTS, OptionEffectTag.TERMINAL_OUTPUT},
+      metadataTags = {OptionMetadataTag.EXPERIMENTAL}
+    )
+    public boolean useNewCategoryEnum;
   }
 
   /**
-   * Returns a map that maps option categories to descriptive help strings for categories that
-   * are not part of the Bazel core.
+   * Returns a map that maps option categories to descriptive help strings for categories that are
+   * not part of the Bazel core.
    */
-  private static ImmutableMap<String, String> getOptionCategories(BlazeRuntime runtime) {
+  @Deprecated
+  private static ImmutableMap<String, String> getDeprecatedOptionCategoriesDescriptions(
+      String name) {
     ImmutableMap.Builder<String, String> optionCategoriesBuilder = ImmutableMap.builder();
-    String name = runtime.getProductName();
     optionCategoriesBuilder
         .put("checking", String.format(
              "Checking options, which control %s's error checking and/or warnings", name))
@@ -180,27 +192,40 @@ public final class HelpCommand implements BlazeCommand {
       return ExitCode.COMMAND_LINE_ERROR;
     }
     String helpSubject = options.getResidue().get(0);
-    if (helpSubject.equals("startup_options")) {
-      emitBlazeVersionInfo(outErr, runtime.getProductName());
-      emitStartupOptions(
-          outErr, helpOptions.helpVerbosity, runtime, getOptionCategories(runtime));
-      return ExitCode.SUCCESS;
-    } else if (helpSubject.equals("target-syntax")) {
-      emitBlazeVersionInfo(outErr, runtime.getProductName());
-      emitTargetSyntaxHelp(outErr, getOptionCategories(runtime), runtime.getProductName());
-      return ExitCode.SUCCESS;
-    } else if (helpSubject.equals("info-keys")) {
-      emitInfoKeysHelp(env, outErr);
-      return ExitCode.SUCCESS;
-    } else if (helpSubject.equals("completion")) {
-      emitCompletionHelp(runtime, outErr);
-      return ExitCode.SUCCESS;
-    } else if (helpSubject.equals("flags-as-proto")) {
-      emitFlagsAsProtoHelp(runtime, outErr);
-      return ExitCode.SUCCESS;
-    } else if (helpSubject.equals("everything-as-html")) {
-      new HtmlEmitter(runtime).emit(outErr);
-      return ExitCode.SUCCESS;
+    String productName = runtime.getProductName();
+    // Go through the custom subjects before going through Bazel commands.
+    switch (helpSubject) {
+      case "startup_options":
+        emitBlazeVersionInfo(outErr, runtime.getProductName());
+        emitStartupOptions(
+            outErr,
+            helpOptions.helpVerbosity,
+            runtime,
+            getDeprecatedOptionCategoriesDescriptions(productName),
+            helpOptions.useNewCategoryEnum);
+        return ExitCode.SUCCESS;
+      case "target-syntax":
+        emitBlazeVersionInfo(outErr, runtime.getProductName());
+        emitTargetSyntaxHelp(
+            outErr,
+            getDeprecatedOptionCategoriesDescriptions(productName),
+            productName,
+            helpOptions.useNewCategoryEnum);
+
+        return ExitCode.SUCCESS;
+      case "info-keys":
+        emitInfoKeysHelp(env, outErr);
+        return ExitCode.SUCCESS;
+      case "completion":
+        emitCompletionHelp(runtime, outErr);
+        return ExitCode.SUCCESS;
+      case "flags-as-proto":
+        emitFlagsAsProtoHelp(runtime, outErr);
+        return ExitCode.SUCCESS;
+      case "everything-as-html":
+        new HtmlEmitter(runtime, helpOptions.useNewCategoryEnum).emit(outErr);
+        return ExitCode.SUCCESS;
+      default: // fall out
     }
 
     BlazeCommand command = runtime.getCommandMap().get(helpSubject);
@@ -218,14 +243,17 @@ public final class HelpCommand implements BlazeCommand {
         return ExitCode.COMMAND_LINE_ERROR;
       }
     }
-    emitBlazeVersionInfo(outErr, runtime.getProductName());
-    outErr.printOut(BlazeCommandUtils.getUsage(
-        command.getClass(),
-        getOptionCategories(runtime),
-        helpOptions.helpVerbosity,
-        runtime.getBlazeModules(),
-        runtime.getRuleClassProvider(),
-        runtime.getProductName()));
+    emitBlazeVersionInfo(outErr, productName);
+    outErr.printOut(
+        BlazeCommandUtils.getUsage(
+            command.getClass(),
+            getDeprecatedOptionCategoriesDescriptions(productName),
+            helpOptions.helpVerbosity,
+            runtime.getBlazeModules(),
+            runtime.getRuleClassProvider(),
+            productName,
+            helpOptions.useNewCategoryEnum));
+
     return ExitCode.SUCCESS;
   }
 
@@ -235,16 +263,22 @@ public final class HelpCommand implements BlazeCommand {
     outErr.printOut(String.format("%80s\n", line));
   }
 
-  private void emitStartupOptions(OutErr outErr, OptionsParser.HelpVerbosity helpVerbosity,
-      BlazeRuntime runtime, ImmutableMap<String, String> optionCategories) {
+  private void emitStartupOptions(
+      OutErr outErr,
+      OptionsParser.HelpVerbosity helpVerbosity,
+      BlazeRuntime runtime,
+      ImmutableMap<String, String> optionCategories,
+      boolean useNewCategoryEnum) {
     outErr.printOut(
-        BlazeCommandUtils.expandHelpTopic("startup_options",
+        BlazeCommandUtils.expandHelpTopic(
+            "startup_options",
             "resource:startup_options.txt",
             getClass(),
             BlazeCommandUtils.getStartupOptions(runtime.getBlazeModules()),
             optionCategories,
             helpVerbosity,
-            runtime.getProductName()));
+            runtime.getProductName(),
+            useNewCategoryEnum));
   }
 
   private void emitCompletionHelp(BlazeRuntime runtime, OutErr outErr) {
@@ -345,15 +379,21 @@ public final class HelpCommand implements BlazeCommand {
     return ImmutableSortedMap.copyOf(runtime.getCommandMap());
   }
 
-  private void emitTargetSyntaxHelp(OutErr outErr, ImmutableMap<String, String> optionCategories,
-      String productName) {
-    outErr.printOut(BlazeCommandUtils.expandHelpTopic("target-syntax",
-                                    "resource:target-syntax.txt",
-                                    getClass(),
-                                    ImmutableList.<Class<? extends OptionsBase>>of(),
-                                    optionCategories,
-                                    OptionsParser.HelpVerbosity.MEDIUM,
-                                    productName));
+  private void emitTargetSyntaxHelp(
+      OutErr outErr,
+      ImmutableMap<String, String> optionCategories,
+      String productName,
+      boolean useNewCategoryEnum) {
+    outErr.printOut(
+        BlazeCommandUtils.expandHelpTopic(
+            "target-syntax",
+            "resource:target-syntax.txt",
+            getClass(),
+            ImmutableList.<Class<? extends OptionsBase>>of(),
+            optionCategories,
+            OptionsParser.HelpVerbosity.MEDIUM,
+            productName,
+            useNewCategoryEnum));
   }
 
   private void emitInfoKeysHelp(CommandEnvironment env, OutErr outErr) {
@@ -400,11 +440,19 @@ public final class HelpCommand implements BlazeCommand {
 
   private static final class HtmlEmitter {
     private final BlazeRuntime runtime;
-    private final ImmutableMap<String, String> optionCategories;
+    private final ImmutableMap<String, String> deprecatedOptionCategoryDescriptions;
+    private final boolean useNewCategoriesEnum;
 
-    private HtmlEmitter(BlazeRuntime runtime) {
+    private HtmlEmitter(BlazeRuntime runtime, boolean useNewCategoriesEnum) {
       this.runtime = runtime;
-      this.optionCategories = getOptionCategories(runtime);
+      this.useNewCategoriesEnum = useNewCategoriesEnum;
+      String productName = runtime.getProductName();
+      if (useNewCategoriesEnum) {
+        this.deprecatedOptionCategoryDescriptions = null;
+      } else {
+        this.deprecatedOptionCategoryDescriptions =
+            getDeprecatedOptionCategoriesDescriptions(productName);
+      }
     }
 
     private void emit(OutErr outErr) {
@@ -478,14 +526,67 @@ public final class HelpCommand implements BlazeCommand {
           result.append("\n");
         }
       }
+
+      // Describe the tags once, any mentions above should link to these descriptions.
+      if (useNewCategoriesEnum) {
+        String productName = runtime.getProductName();
+        ImmutableMap<OptionEffectTag, String> effectTagDescriptions =
+            OptionFilterDescriptions.getOptionEffectTagDescription(productName);
+        result.append("<h3>Option Effect Tags</h3>\n");
+        result.append("<table>\n");
+        for (OptionEffectTag tag : OptionEffectTag.values()) {
+          String tagDescription = effectTagDescriptions.get(tag);
+
+          result.append("<tr>\n");
+          result.append(
+              String.format(
+                  "<td id=\"effect_tag_%s\"><code>%s</code></td>\n",
+                  tag, tag.name().toLowerCase()));
+          result.append(String.format("<td>%s</td>\n", HTML_ESCAPER.escape(tagDescription)));
+          result.append("</tr>\n");
+        }
+        result.append("</table>\n");
+
+        ImmutableMap<OptionMetadataTag, String> metadataTagDescriptions =
+            OptionFilterDescriptions.getOptionMetadataTagDescription(productName);
+        result.append("<h3>Option Metadata Tags</h3>\n");
+        result.append("<table>\n");
+        for (OptionMetadataTag tag : OptionMetadataTag.values()) {
+          // skip the tags that are reserved for undocumented flags.
+          if (!tag.equals(OptionMetadataTag.HIDDEN) && !tag.equals(OptionMetadataTag.INTERNAL)) {
+            String tagDescription = metadataTagDescriptions.get(tag);
+
+            result.append("<tr>\n");
+            result.append(
+                String.format(
+                    "<td id=\"metadata_tag_%s\"><code>%s</code></td>\n",
+                    tag, tag.name().toLowerCase()));
+            result.append(String.format("<td>%s</td>\n", HTML_ESCAPER.escape(tagDescription)));
+            result.append("</tr>\n");
+          }
+        }
+        result.append("</table>\n");
+      }
+
       outErr.printOut(result.toString());
     }
 
     private void appendOptionsHtml(
         StringBuilder result, Iterable<Class<? extends OptionsBase>> optionsClasses) {
       OptionsParser parser = OptionsParser.newOptionsParser(optionsClasses);
-      result.append(parser.describeOptionsHtml(optionCategories, HTML_ESCAPER)
-          .replace("%{product}", runtime.getProductName()));
+      String productName = runtime.getProductName();
+      if (useNewCategoriesEnum) {
+        result.append(
+            parser
+                .describeOptionsHtml(HTML_ESCAPER, productName)
+                .replace("%{product}", productName));
+      } else {
+        result.append(
+            parser
+                .describeOptionsHtmlWithDeprecatedCategories(
+                    deprecatedOptionCategoryDescriptions, HTML_ESCAPER)
+                .replace("%{product}", productName));
+      }
     }
 
     private static String capitalize(String s) {
@@ -510,3 +611,4 @@ public final class HelpCommand implements BlazeCommand {
     void visit(String commandName, Command commandAnnotation, OptionsParser parser);
   }
 }
+