Merge remote-tracking branch 'upstream/master' into bind_mode

Conflicts:
	builtin.cpp
	reader.cpp
	share/functions/fish_default_key_bindings.fish

1 files changed, 74 insertions, 142 deletions

diff --git a/reader.cpp b/reader.cpp
index 34fe0d23..1e57d820 100644
--- a/reader.cpp
+++ b/reader.cpp
@@ -99,6 +99,7 @@ commence.
 #include "path.h"
 #include "parse_util.h"
 #include "parser_keywords.h"
+#include "parse_tree.h"
 
 /**
    Maximum length of prefix string when printing completion
@@ -186,9 +187,6 @@ static volatile unsigned int s_generation_count;
 /* This pthreads generation count is set when an autosuggestion background thread starts up, so it can easily check if the work it is doing is no longer useful. */
 static pthread_key_t generation_count_key;
 
-/* A color is an int */
-typedef int color_t;
-
 static void set_command_line_and_position(const wcstring &new_str, size_t pos);
 
 /**
@@ -292,7 +290,7 @@ public:
        color[i] is the classification (according to the enum in
        highlight.h) of buff[i].
     */
-    std::vector<color_t> colors;
+    std::vector<highlight_spec_t> colors;
 
     /** An array defining the block level at each character. */
     std::vector<int> indents;
@@ -563,7 +561,7 @@ wcstring combine_command_and_autosuggestion(const wcstring &cmdline, const wcstr
 static void reader_repaint()
 {
     // Update the indentation
-    parser_t::principal_parser().test(data->command_line.c_str(), &data->indents[0]);
+    data->indents = parse_util_compute_indents(data->command_line);
 
     // Combine the command and autosuggestion into one string
     wcstring full_line = combine_command_and_autosuggestion(data->command_line, data->autosuggestion);
@@ -572,8 +570,8 @@ static void reader_repaint()
     if (len < 1)
         len = 1;
 
-    std::vector<color_t> colors = data->colors;
-    colors.resize(len, HIGHLIGHT_AUTOSUGGESTION);
+    std::vector<highlight_spec_t> colors = data->colors;
+    colors.resize(len, highlight_spec_autosuggestion);
 
     if(data->sel_active)
     {
@@ -693,7 +691,7 @@ void reader_data_t::command_line_changed()
     size_t len = command_length();
 
     /* When we grow colors, propagate the last color (if any), under the assumption that usually it will be correct. If it is, it avoids a repaint. */
-    color_t last_color = colors.empty() ? color_t() : colors.back();
+    highlight_spec_t last_color = colors.empty() ? highlight_spec_t() : colors.back();
     colors.resize(len, last_color);
 
     indents.resize(len);
@@ -716,117 +714,55 @@ bool reader_expand_abbreviation_in_command(const wcstring &cmdline, size_t curso
     const size_t subcmd_offset = cmdsub_begin - buff;
 
     const wcstring subcmd = wcstring(cmdsub_begin, cmdsub_end - cmdsub_begin);
-    const wchar_t *subcmd_cstr = subcmd.c_str();
-
-    /* Get the token containing the cursor */
-    const wchar_t *subcmd_tok_begin = NULL, *subcmd_tok_end = NULL;
-    assert(cursor_pos >= subcmd_offset);
-    size_t subcmd_cursor_pos = cursor_pos - subcmd_offset;
-    parse_util_token_extent(subcmd_cstr, subcmd_cursor_pos, &subcmd_tok_begin, &subcmd_tok_end, NULL, NULL);
-
-    /* Compute the offset of the token before the cursor within the subcmd */
-    assert(subcmd_tok_begin >= subcmd_cstr);
-    assert(subcmd_tok_end >= subcmd_tok_begin);
-    const size_t subcmd_tok_begin_offset = subcmd_tok_begin - subcmd_cstr;
-    const size_t subcmd_tok_length = subcmd_tok_end - subcmd_tok_begin;
-
-    /* Now parse the subcmd, looking for commands */
-    bool had_cmd = false, previous_token_is_cmd = false;
-    tokenizer_t tok(subcmd_cstr, TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
-    for (; tok_has_next(&tok); tok_next(&tok))
-    {
-        size_t tok_pos = static_cast<size_t>(tok_get_pos(&tok));
-        if (tok_pos > subcmd_tok_begin_offset)
-        {
-            /* We've passed the token we're interested in */
-            break;
-        }
+    const size_t subcmd_cursor_pos = cursor_pos - subcmd_offset;
 
-        int last_type = tok_last_type(&tok);
+    /* Parse this subcmd */
+    parse_node_tree_t parse_tree;
+    parse_tree_from_string(subcmd, parse_flag_continue_after_error | parse_flag_accept_incomplete_tokens, &parse_tree, NULL);
 
-        switch (last_type)
-        {
-            case TOK_STRING:
-            {
-                if (had_cmd)
-                {
-                    /* Parameter to the command. */
-                }
-                else
-                {
-                    const wcstring potential_cmd = tok_last(&tok);
-                    if (parser_keywords_is_subcommand(potential_cmd))
-                    {
-                        if (potential_cmd == L"command" || potential_cmd == L"builtin")
-                        {
-                            /* 'command' and 'builtin' defeat abbreviation expansion. Skip this command. */
-                            had_cmd = true;
-                        }
-                        else
-                        {
-                            /* Other subcommand. Pretend it doesn't exist so that we can expand the following command */
-                            had_cmd = false;
-                        }
-                    }
-                    else
-                    {
-                        /* It's a normal command */
-                        had_cmd = true;
-                        if (tok_pos == subcmd_tok_begin_offset)
-                        {
-                            /* This is the token we care about! */
-                            previous_token_is_cmd = true;
-                        }
-                    }
-                }
-                break;
-            }
+    /* Look for plain statements where the cursor is at the end of the command */
+    const parse_node_t *matching_cmd_node = NULL;
+    const size_t len = parse_tree.size();
+    for (size_t i=0; i < len; i++)
+    {
+        const parse_node_t &node = parse_tree.at(i);
 
-            case TOK_REDIRECT_NOCLOB:
-            case TOK_REDIRECT_OUT:
-            case TOK_REDIRECT_IN:
-            case TOK_REDIRECT_APPEND:
-            case TOK_REDIRECT_FD:
-            {
-                if (!had_cmd)
-                {
-                    break;
-                }
-                tok_next(&tok);
-                break;
-            }
+        /* Only interested in plain statements with source */
+        if (node.type != symbol_plain_statement || ! node.has_source())
+            continue;
 
-            case TOK_PIPE:
-            case TOK_BACKGROUND:
-            case TOK_END:
-            {
-                had_cmd = false;
-                break;
-            }
+        /* Skip decorated statements */
+        if (parse_tree.decoration_for_plain_statement(node) != parse_statement_decoration_none)
+            continue;
 
-            case TOK_COMMENT:
-            case TOK_ERROR:
-            default:
-            {
-                break;
-            }
+        /* Get the command node. Skip it if we can't or it has no source */
+        const parse_node_t *cmd_node = parse_tree.get_child(node, 0, parse_token_type_string);
+        if (cmd_node == NULL || ! cmd_node->has_source())
+            continue;
+
+        /* Now see if its source range contains our cursor, including at the end */
+        if (subcmd_cursor_pos >= cmd_node->source_start && subcmd_cursor_pos <= cmd_node->source_start + cmd_node->source_length)
+        {
+            /* Success! */
+            matching_cmd_node = cmd_node;
+            break;
         }
     }
 
+    /* Now if we found a command node, expand it */
     bool result = false;
-    if (previous_token_is_cmd)
+    if (matching_cmd_node != NULL)
     {
-        /* The token is a command. Try expanding it as an abbreviation. */
-        const wcstring token = wcstring(subcmd, subcmd_tok_begin_offset, subcmd_tok_length);
+        assert(matching_cmd_node->type == parse_token_type_string);
+        const wcstring token = matching_cmd_node->get_source(subcmd);
         wcstring abbreviation;
         if (expand_abbreviation(token, &abbreviation))
         {
             /* There was an abbreviation! Replace the token in the full command. Maintain the relative position of the cursor. */
             if (output != NULL)
             {
-                size_t cmd_tok_begin_offset = subcmd_tok_begin_offset + subcmd_offset;
                 output->assign(cmdline);
-                output->replace(cmd_tok_begin_offset, subcmd_tok_length, abbreviation);
+                output->replace(subcmd_offset + matching_cmd_node->source_start, matching_cmd_node->source_length, abbreviation);
             }
             result = true;
         }
@@ -1551,7 +1487,7 @@ struct autosuggestion_context_t
         {
             const completion_t &comp = completions.at(0);
             size_t cursor = this->cursor_pos;
-            this->autosuggestion = completion_apply_to_command_line(comp.completion.c_str(), comp.flags, this->search_string, &cursor, true /* append only */);
+            this->autosuggestion = completion_apply_to_command_line(comp.completion, comp.flags, this->search_string, &cursor, true /* append only */);
             return 1;
         }
 
@@ -1641,7 +1577,7 @@ static void reader_flash()
 
     for (size_t i=0; i<data->buff_pos; i++)
     {
-        data->colors.at(i) = HIGHLIGHT_SEARCH_MATCH<<16;
+        data->colors.at(i) = highlight_spec_search_match<<16;
     }
 
     reader_repaint();
@@ -2186,11 +2122,9 @@ static void reader_interactive_destroy()
 
 void reader_sanity_check()
 {
-    if (get_is_interactive())
+    /* Note: 'data' is non-null if we are interactive, except in the testing environment */
+    if (get_is_interactive() && data != NULL)
     {
-        if (!data)
-            sanity_lose();
-
         if (!(data->buff_pos <= data->command_length()))
             sanity_lose();
 
@@ -2320,7 +2254,6 @@ static void handle_token_history(int forward, int reset)
             */
             if (data->history_search.go_backwards())
             {
-                wcstring item = data->history_search.current_string();
                 data->token_history_buff = data->history_search.current_string();
             }
             current_pos = data->token_history_buff.size();
@@ -2610,30 +2543,26 @@ void reader_run_command(parser_t &parser, const wcstring &cmd)
 
 int reader_shell_test(const wchar_t *b)
 {
-    int res = parser_t::principal_parser().test(b);
+    assert(b != NULL);
+    wcstring bstr = b;
+
+    /* Append a newline, to act as a statement terminator */
+    bstr.push_back(L'\n');
+
+    parse_error_list_t errors;
+    int res = parse_util_detect_errors(bstr, &errors);
 
     if (res & PARSER_TEST_ERROR)
     {
-        wcstring sb;
-
-        const int tmp[1] = {0};
-        const int tmp2[1] = {0};
-        const wcstring empty;
-
-        s_write(&data->screen,
-                empty,
-                empty,
-                empty,
-                0,
-                tmp,
-                tmp2,
-                0,
-                0,
-                0);
-
-
-        parser_t::principal_parser().test(b, NULL, &sb, L"fish");
-        fwprintf(stderr, L"%ls", sb.c_str());
+        wcstring error_desc;
+        parser_t::principal_parser().get_backtrace(bstr, errors, &error_desc);
+
+        // ensure we end with a newline. Also add an initial newline, because it's likely the user just hit enter and so there's junk on the current line
+        if (! string_suffixes_string(L"\n", error_desc))
+        {
+            error_desc.push_back(L'\n');
+        }
+        fwprintf(stderr, L"\n%ls", error_desc.c_str());
     }
     return res;
 }
@@ -2766,7 +2695,7 @@ public:
     const wcstring string_to_highlight;
 
     /** Color buffer */
-    std::vector<color_t> colors;
+    std::vector<highlight_spec_t> colors;
 
     /** The position to use for bracket matching */
     const size_t match_highlight_pos;
@@ -2821,7 +2750,7 @@ static void highlight_search(void)
             size_t end = match_pos + needle.size();
             for (size_t i=match_pos; i < end; i++)
             {
-                data->colors.at(i) |= (HIGHLIGHT_SEARCH_MATCH<<16);
+                data->colors.at(i) |= (highlight_spec_search_match<<16);
             }
         }
     }
@@ -2883,10 +2812,10 @@ static void reader_super_highlight_me_plenty(size_t match_highlight_pos)
 }
 
 
-int exit_status()
+bool shell_is_exiting()
 {
     if (get_is_interactive())
-        return job_list_is_empty() && data->end_loop;
+        return job_list_is_empty() && data != NULL && data->end_loop;
     else
         return end_loop;
 }
@@ -3137,6 +3066,7 @@ const wchar_t *reader_readline(void)
             is_interactive_read = 1;
             c=input_readch();
             is_interactive_read = was_interactive_read;
+            //fprintf(stderr, "C: %lx\n", (long)c);
 
             if (((!wchar_private(c))) && (c>31) && (c != 127))
             {
@@ -3312,6 +3242,9 @@ const wchar_t *reader_readline(void)
                     const wchar_t *token_begin, *token_end;
                     parse_util_token_extent(cmdsub_begin, data->buff_pos - (cmdsub_begin-buff), &token_begin, &token_end, 0, 0);
 
+                    /* Hack: the token may extend past the end of the command substitution, e.g. in (echo foo) the last token is 'foo)'. Don't let that happen. */
+                    if (token_end > cmdsub_end) token_end = cmdsub_end;
+
                     /* Figure out how many steps to get from the current position to the end of the current token. */
                     size_t end_of_token_offset = token_end - buff;
 
@@ -3466,7 +3399,7 @@ const wchar_t *reader_readline(void)
                     {
                         //history_reset();
                         data->history_search.go_to_end();
-                        reader_set_buffer(data->search_buff.c_str(), data->search_buff.size());
+                        reader_set_buffer(data->search_buff, data->search_buff.size());
                     }
                     else
                     {
@@ -3543,12 +3476,9 @@ const wchar_t *reader_readline(void)
                     case 0:
                     {
                         /* Finished command, execute it. Don't add items that start with a leading space. */
-                        if (! data->command_line.empty() && data->command_line.at(0) != L' ')
+                        if (data->history != NULL && ! data->command_line.empty() && data->command_line.at(0) != L' ')
                         {
-                            if (data->history != NULL)
-                            {
-                                data->history->add_with_file_detection(data->command_line);
-                            }
+                            data->history->add_with_file_detection(data->command_line);
                         }
                         finished=1;
                         update_buff_pos(data->command_length());
@@ -4059,13 +3989,15 @@ static int read_ni(int fd, const io_chain_t &io)
             res = 1;
         }
 
-        wcstring sb;
-        if (! parser.test(str.c_str(), 0, &sb, L"fish"))
+        parse_error_list_t errors;
+        if (! parse_util_detect_errors(str, &errors))
         {
             parser.eval(str, io, TOP);
         }
         else
         {
+            wcstring sb;
+            parser.get_backtrace(str, errors, &sb);
             fwprintf(stderr, L"%ls", sb.c_str());
             res = 1;
         }