enhance fish_indent to normalize keywords

Fish keywords can be quoted and split across lines. Prior to this change
`fish_indent` would retain such odd, obfuscated, formatting. This change
results in all keywords being converted to their canonical form.

This required fixing a bug: the keyword member of parse_node_t wasn't being
populated. This hadn't been noticed prior to now because it wasn't used.

Fixes #2921

5 files changed, 45 insertions, 7 deletions

diff --git a/src/fish_indent.cpp b/src/fish_indent.cpp
index d2e23943..39d296e5 100644
--- a/src/fish_indent.cpp
+++ b/src/fish_indent.cpp
@@ -102,10 +102,10 @@ static void dump_node(indent_t node_indent, const parse_node_t &node, const wcst
         nextc_str[1] = L'c';
         nextc_str[2] = nextc + '@';
     }
-    fwprintf(stderr, L"{off %4d, len %4d, indent %2u, %ls} [%ls|%ls|%ls]\n",
+    fwprintf(stderr, L"{off %4d, len %4d, indent %2u, kw %ls, %ls} [%ls|%ls|%ls]\n",
             node.source_start, node.source_length, node_indent,
-            parser_token_types[node.type], prevc_str, source.substr(node.source_start,
-            node.source_length).c_str(), nextc_str);
+            keyword_description(node.keyword).c_str(), parser_token_types[node.type],
+            prevc_str, source.substr(node.source_start, node.source_length).c_str(), nextc_str);
 }
 
 static void prettify_node_recursive(const wcstring &source, const parse_node_tree_t &tree,
@@ -153,7 +153,13 @@ static void prettify_node_recursive(const wcstring &source, const parse_node_tre
     else if ((node_type >= FIRST_PARSE_TOKEN_TYPE && node_type <= LAST_PARSE_TOKEN_TYPE) ||
             node_type == parse_special_type_parse_error)
     {
-        if (node.has_source())
+        if (node.keyword != parse_keyword_none)
+        {
+            append_whitespace(node_indent, do_indent, *has_new_line, out_result);
+            out_result->append(keyword_description(node.keyword));
+            *has_new_line = false;
+        }
+        else if (node.has_source())
         {
             // Some type representing a particular token.
             if (prev_node_type != parse_token_type_redirection)
diff --git a/src/parse_constants.h b/src/parse_constants.h
index 03dffd28..4668516b 100644
--- a/src/parse_constants.h
+++ b/src/parse_constants.h
@@ -73,6 +73,7 @@ enum parse_token_type_t
     parse_special_type_parse_error,
     parse_special_type_tokenizer_error,
     parse_special_type_comment,
+    LAST_TOKEN_TYPE = parse_special_type_comment,
 
     FIRST_TERMINAL_TYPE = parse_token_type_string,
     LAST_TERMINAL_TYPE = parse_token_type_terminate,
@@ -85,7 +86,11 @@ enum parse_token_type_t
 // Array of strings corresponding to the enums above instantiated in parse_tree.cpp.
 extern const wchar_t * const parser_token_types[];
 
-/* These must be maintained in sorted order (except for none, which isn't a keyword). This enables us to do binary search. */
+// These must be maintained in sorted order (except for none, which isn't a keyword). This enables
+// us to do binary search.
+//
+// IMPORTANT: If the following enum is modified you must update the corresponding keyword_map array
+// in parse_tree.cpp.
 enum parse_keyword_t
 {
     parse_keyword_none,
diff --git a/src/parse_tree.cpp b/src/parse_tree.cpp
index ca6e310b..adb2984f 100644
--- a/src/parse_tree.cpp
+++ b/src/parse_tree.cpp
@@ -1078,9 +1078,11 @@ bool parse_ll_t::top_node_handle_terminal_types(parse_token_t token)
 
         if (matched)
         {
-            // Success. Tell the node that it matched this token, and what its source range is
-            // In the parse phase, we only set source ranges for terminal types. We propagate ranges to parent nodes afterwards.
+            // Success. Tell the node that it matched this token, and what its source range is in
+            // the parse phase, we only set source ranges for terminal types. We propagate ranges to
+            // parent nodes afterwards.
             parse_node_t &node = node_for_top_symbol();
+            node.keyword = token.keyword;
             node.source_start = token.source_start;
             node.source_length = token.source_length;
         }
diff --git a/tests/indent.in b/tests/indent.in
index a1143c67..2d98dd8e 100644
--- a/tests/indent.in
+++ b/tests/indent.in
@@ -89,3 +89,18 @@ echo \nTest redir formatting
 echo -n '
 echo < stdin >>appended yes 2>&1 no > stdout maybe 2>&    4 | cat 2>| cat
 ' | ../test/root/bin/fish_indent
+
+echo \nTest normalization of keywords
+# issue 2921
+echo -n '
+i\
+f true
+    echo yes
+en\
+d
+
+"whil\
+e" true
+    "builtin" yes
+en"d"
+' | ../test/root/bin/fish_indent
diff --git a/tests/indent.out b/tests/indent.out
index 6b38a72f..5a4d9dc0 100644
--- a/tests/indent.out
+++ b/tests/indent.out
@@ -95,3 +95,13 @@ end
 Test redir formatting
 
 echo <stdin >>appended yes 2>&1 no >stdout maybe 2>&4 | cat 2>| cat
+
+Test normalization of keywords
+
+if true
+    echo yes
+end
+
+while true
+    builtin yes
+end