Fix for issue where fish_indent would lose blank lines

3 files changed, 17 insertions, 11 deletions

diff --git a/fish_indent.cpp b/fish_indent.cpp
index ddd9189b..195b007c 100644
--- a/fish_indent.cpp
+++ b/fish_indent.cpp
@@ -91,7 +91,7 @@ static int indent(wcstring &out, const wcstring &in, int flags)
     int prev_type = 0;
     int prev_prev_type = 0;
 
-    tokenizer_t tok(in.c_str(), TOK_SHOW_COMMENTS);
+    tokenizer_t tok(in.c_str(), TOK_SHOW_COMMENTS | TOK_SHOW_BLANK_LINES);
     for (; tok_has_next(&tok); tok_next(&tok))
     {
         int type = tok_last_type(&tok);
diff --git a/tokenizer.cpp b/tokenizer.cpp
index 29db04bd..16929a14 100644
--- a/tokenizer.cpp
+++ b/tokenizer.cpp
@@ -101,6 +101,7 @@ tokenizer_t::tokenizer_t(const wchar_t *b, tok_flags_t flags) : buff(NULL), orig
     this->accept_unfinished = !!(flags & TOK_ACCEPT_UNFINISHED);
     this->show_comments = !!(flags & TOK_SHOW_COMMENTS);
     this->squash_errors = !!(flags & TOK_SQUASH_ERRORS);
+    this->show_blank_lines = !!(flags & TOK_SHOW_BLANK_LINES);
 
     this->has_next = (*b != L'\0');
     this->orig_buff = this->buff = b;
@@ -562,7 +563,6 @@ const wchar_t *tok_get_desc(int type)
     return _(tok_desc[type]);
 }
 
-
 void tok_next(tokenizer_t *tok)
 {
 
@@ -628,18 +628,18 @@ void tok_next(tokenizer_t *tok)
             break;
         case 13: // carriage return
         case L'\n':
+        case L';':
+            tok->last_type = TOK_END;
+            tok->buff++;
             // Hack: when we get a newline, swallow as many as we can
             // This compresses multiple subsequent newlines into a single one
-            while (*tok->buff == L'\n' || *tok->buff == 13 || *tok->buff == ' ' || *tok->buff == '\t')
+            if (! tok->show_blank_lines)
             {
-                tok->buff++;
+                while (*tok->buff == L'\n' || *tok->buff == 13 /* CR */ || *tok->buff == ' ' || *tok->buff == '\t')
+                {
+                    tok->buff++;
+                }
             }
-            tok->last_type = TOK_END;
-            break;
-
-        case L';':
-            tok->last_type = TOK_END;
-            tok->buff++;
             break;
         case L'&':
             tok->last_type = TOK_BACKGROUND;
diff --git a/tokenizer.h b/tokenizer.h
index 749337b4..f51906e7 100644
--- a/tokenizer.h
+++ b/tokenizer.h
@@ -61,6 +61,10 @@ enum tokenizer_error
 */
 #define TOK_SQUASH_ERRORS 4
 
+/** Ordinarily, the tokenizer ignores newlines following a newline, or a semicolon.
+    This flag tells the tokenizer to return each of them as a separate END. */
+#define TOK_SHOW_BLANK_LINES 8
+
 typedef unsigned int tok_flags_t;
 
 /**
@@ -84,8 +88,10 @@ struct tokenizer_t
     bool has_next;
     /** Whether incomplete tokens are accepted*/
     bool accept_unfinished;
-    /** Whether commants should be returned*/
+    /** Whether comments should be returned*/
     bool show_comments;
+    /** Whether all blank lines are returned */
+    bool show_blank_lines;
     /** Type of last quote, can be either ' or ".*/
     wchar_t last_quote;
     /** Last error */