Implement new newline-escaping behavior. Backslashes at the end of lines now essentially delete the newline, within normal text or double quotes. Backslashes are retained within single quotes.

Fixes https://github.com/fish-shell/fish-shell/issues/347
Fixes https://github.com/fish-shell/fish-shell/issues/52

7 files changed, 56 insertions, 40 deletions

diff --git a/builtin_commandline.cpp b/builtin_commandline.cpp
index 4b81dc6d..5c9e5aee 100644
--- a/builtin_commandline.cpp
+++ b/builtin_commandline.cpp
@@ -155,8 +155,7 @@ static void write_part(const wchar_t *begin,
 //    fwprintf( stderr, L"Subshell: %ls, end char %lc\n", buff, *end );
         out.clear();
         tokenizer_t tok(buff, TOK_ACCEPT_UNFINISHED);
-        for (; tok_has_next(&tok);
-                tok_next(&tok))
+        for (; tok_has_next(&tok); tok_next(&tok))
         {
             if ((cut_at_cursor) &&
                     (tok_get_pos(&tok)+wcslen(tok_last(&tok)) >= pos))
diff --git a/common.cpp b/common.cpp
index 1e8806ac..dff67592 100644
--- a/common.cpp
+++ b/common.cpp
@@ -1088,8 +1088,6 @@ wcstring escape_string(const wcstring &in, escape_flags_t flags)
 
 wchar_t *unescape(const wchar_t * orig, int flags)
 {
-
-    int mode = 0;
     int out_pos;
     size_t in_pos;
     size_t len;
@@ -1097,8 +1095,8 @@ wchar_t *unescape(const wchar_t * orig, int flags)
     int bracket_count=0;
     wchar_t prev=0;
     wchar_t *in;
-    int unescape_special = flags & UNESCAPE_SPECIAL;
-    int allow_incomplete = flags & UNESCAPE_INCOMPLETE;
+    bool unescape_special = !! (flags & UNESCAPE_SPECIAL);
+    bool allow_incomplete = !! (flags & UNESCAPE_INCOMPLETE);
 
     CHECK(orig, 0);
 
@@ -1107,6 +1105,12 @@ wchar_t *unescape(const wchar_t * orig, int flags)
 
     if (!in)
         DIE_MEM();
+    
+    enum {
+        mode_unquoted,
+        mode_single_quotes,
+        mode_double_quotes
+    } mode = mode_unquoted;
 
     for (in_pos=0, out_pos=0;
             in_pos<len;
@@ -1116,20 +1120,20 @@ wchar_t *unescape(const wchar_t * orig, int flags)
         switch (mode)
         {
 
-                /*
-                  Mode 0 means unquoted string
-                */
-            case 0:
+            /*
+              Mode 0 means unquoted string
+            */
+            case mode_unquoted:
             {
                 if (c == L'\\')
                 {
                     switch (in[++in_pos])
                     {
 
-                            /*
-                              A null character after a backslash is an
-                              error, return null
-                            */
+                        /*
+                          A null character after a backslash is an
+                          error, return null
+                        */
                         case L'\0':
                         {
                             if (!allow_incomplete)
@@ -1317,7 +1321,7 @@ wchar_t *unescape(const wchar_t * orig, int flags)
                             in[out_pos]=L'\t';
                             break;
                         }
-
+                        
                         /*
                           \v means vertical tab
                         */
@@ -1326,6 +1330,11 @@ wchar_t *unescape(const wchar_t * orig, int flags)
                             in[out_pos]=L'\v';
                             break;
                         }
+                        
+                        /* If a backslash is followed by an actual newline, swallow them both */
+                        case L'\n':
+                            out_pos--;
+                            break;
 
                         default:
                         {
@@ -1454,7 +1463,7 @@ wchar_t *unescape(const wchar_t * orig, int flags)
 
                         case L'\'':
                         {
-                            mode = 1;
+                            mode = mode_single_quotes;
                             if (unescape_special)
                                 in[out_pos] = INTERNAL_SEPARATOR;
                             else
@@ -1464,7 +1473,7 @@ wchar_t *unescape(const wchar_t * orig, int flags)
 
                         case L'\"':
                         {
-                            mode = 2;
+                            mode = mode_double_quotes;
                             if (unescape_special)
                                 in[out_pos] = INTERNAL_SEPARATOR;
                             else
@@ -1483,9 +1492,10 @@ wchar_t *unescape(const wchar_t * orig, int flags)
             }
 
             /*
-              Mode 1 means single quoted string, i.e 'foo'
+              Mode 1 means single quoted string, i.e 'foo'.
+              A backslash at the end of a line in a single quoted string does not swallow the backslash or newline.
             */
-            case 1:
+            case mode_single_quotes:
             {
                 if (c == L'\\')
                 {
@@ -1493,13 +1503,12 @@ wchar_t *unescape(const wchar_t * orig, int flags)
                     {
                         case '\\':
                         case L'\'':
-                        case L'\n':
                         {
                             in[out_pos]=in[in_pos];
                             break;
                         }
 
-                        case 0:
+                        case L'\0':
                         {
                             if (!allow_incomplete)
                             {
@@ -1513,6 +1522,7 @@ wchar_t *unescape(const wchar_t * orig, int flags)
                             }
                         }
                         break;
+                        
                         default:
                         {
                             in[out_pos++] = L'\\';
@@ -1527,7 +1537,7 @@ wchar_t *unescape(const wchar_t * orig, int flags)
                         in[out_pos] = INTERNAL_SEPARATOR;
                     else
                         out_pos--;
-                    mode = 0;
+                    mode = mode_unquoted;
                 }
                 else
                 {
@@ -1540,13 +1550,13 @@ wchar_t *unescape(const wchar_t * orig, int flags)
             /*
               Mode 2 means double quoted string, i.e. "foo"
             */
-            case 2:
+            case mode_double_quotes:
             {
                 switch (c)
                 {
                     case '"':
                     {
-                        mode = 0;
+                        mode = mode_unquoted;
                         if (unescape_special)
                             in[out_pos] = INTERNAL_SEPARATOR;
                         else
@@ -1575,11 +1585,16 @@ wchar_t *unescape(const wchar_t * orig, int flags)
                             case '\\':
                             case L'$':
                             case '"':
-                            case '\n':
                             {
                                 in[out_pos]=in[in_pos];
                                 break;
                             }
+                            
+                            case '\n':
+                            {
+                                out_pos--;
+                                break;
+                            }
 
                             default:
                             {
diff --git a/complete.cpp b/complete.cpp
index af70b24d..985107b9 100644
--- a/complete.cpp
+++ b/complete.cpp
@@ -1810,7 +1810,6 @@ void complete(const wcstring &cmd, std::vector<completion_t> &comps, complete_ty
         tokenizer_t tok(buff.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SQUASH_ERRORS);
         while (tok_has_next(&tok) && !end_loop)
         {
-
             switch (tok_last_type(&tok))
             {
 
@@ -1881,7 +1880,6 @@ void complete(const wcstring &cmd, std::vector<completion_t> &comps, complete_ty
                     end_loop=1;
                     break;
                 }
-
             }
 
             if (tok_get_pos(&tok) >= (long)pos)
diff --git a/doc_src/index.hdr.in b/doc_src/index.hdr.in
index b3a671b5..4204a5c7 100644
--- a/doc_src/index.hdr.in
+++ b/doc_src/index.hdr.in
@@ -92,8 +92,9 @@ only backslash escape accepted within single quotes is \\', which
 escapes a single quote and \\\\, which escapes the backslash
 symbol. The only backslash escapes accepted within double quotes are
 \\", which escapes a double quote, \\$, which escapes a dollar
-character, and \\\\, which escapes the backslash symbol. Single quotes
-have no special meaning withing double quotes and vice versa.
+character, \\ followed by a newline, which deletes the backslash
+and the newline, and lastly \\\\, which escapes the backslash symbol.
+Single quotes have no special meaning withing double quotes and vice versa.
 
 Example:
 
@@ -1454,7 +1455,6 @@ g++, javac, java, gcj, lpr, doxygen, whois)
 - Descriptions for variables using 'set -d'.
 - Parse errors should when possible honor IO redirections
 - Support for writing strings like /u/l/b/foo and have them expand to /usr/local/bin/foo - perhaps through tab expansion
-- Right-side prompt
 - Selectable completions in the pager
 - Per process output redirection
 - Reduce the space of the pager by one line to allow the commandline to remain visible.
@@ -1484,8 +1484,6 @@ g++, javac, java, gcj, lpr, doxygen, whois)
 - There have been stray reports of issues with strange values of the PATH variable during startup.
 - bindings in config.fish are overwritten by default key bindings.
 - Adding 'bind -k ...' doesn't overwrite non-keybinding binds of the same sequence.
-- History file does not remove duplicates.
-- History file should apply some kind of maximum history length.
 - Older versions of Doxygen has bugs in the man-page generation which cause the builtin help to render incorrectly. Version 1.2.14 is known to have this problem.
 
 If you think you have found a bug not described here, please send a
diff --git a/tests/test1.in b/tests/test1.in
index 8c214900..159c9cf8 100644
--- a/tests/test1.in
+++ b/tests/test1.in
@@ -15,6 +15,15 @@ echo x-{1}
 echo x-{1,2}
 echo foo-{1,2{3,4}}
 
+# Escpaed newlines
+echo foo\ bar
+echo foo\
+bar
+echo "foo\
+bar"
+echo 'foo\
+bar'
+
 # Simple alias tests
 
 function foo
diff --git a/tests/test1.out b/tests/test1.out
index 3d3fb0d9..52538b19 100644
--- a/tests/test1.out
+++ b/tests/test1.out
@@ -5,6 +5,11 @@
 x-1
 x-1 x-2
 foo-1 foo-23 foo-24
+foo bar
+foobar
+foobar
+foo\
+bar
 Test 2 pass
 Test  pass
 Test 3 pass
diff --git a/tokenizer.cpp b/tokenizer.cpp
index 84702b55..e2e515bf 100644
--- a/tokenizer.cpp
+++ b/tokenizer.cpp
@@ -255,14 +255,6 @@ static void read_string(tokenizer_t *tok)
                         tok->buff--;
                         do_loop = 0;
                     }
-
-
-                }
-                else if (*tok->buff == L'\n' && mode == mode_regular_text)
-                {
-                    tok->buff--;
-                    do_loop = 0;
-                    break;
                 }
 
                 tok->buff++;