First stab at multiline editing

darcs-hash:20061001160258-ac50b-1a760913e64b96e30ff321d7fbe4069ca161cdfe.gz

11 files changed, 885 insertions, 626 deletions

diff --git a/Makefile.in b/Makefile.in
index eb4ee4f3..f4da7427 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -70,7 +70,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o	\
 	expand.o highlight.o history.o kill.o parser.o proc.o reader.o		\
 	sanity.o tokenizer.o wildcard.o wgetopt.o wutil.o input.o	\
 	output.o intern.o env_universal.o env_universal_common.o			\
-	input_common.o event.o signal.o io.o parse_util.o common.o \
+	input_common.o event.o signal.o io.o parse_util.o common.o  screen.o\
 
 # Additional files used by builtin.o
 BUILTIN_FILES := builtin_help.c builtin_set.c builtin_commandline.c	\
diff --git a/output.c b/output.c
index 657abe05..ac431be0 100644
--- a/output.c
+++ b/output.c
@@ -135,6 +135,11 @@ void output_set_writer( int (*writer)(char) )
 	out = writer;
 }
 
+int (*output_get_writer())(char)
+{
+	return out;
+}
+
 
 void set_color( int c, int c2 )
 {
diff --git a/output.h b/output.h
index 8a663cd6..f88f5cde 100644
--- a/output.h
+++ b/output.h
@@ -134,5 +134,9 @@ int writeb( tputs_arg_t b );
 */
 void output_set_writer( int (*writer)(char) );
 
+//typedef int (*func_ptr_t)(char);
+
+int (*output_get_writer())(char) ;
+
 
 #endif
diff --git a/parser.c b/parser.c
index e62c2847..ba230869 100644
--- a/parser.c
+++ b/parser.c
@@ -2996,7 +2996,8 @@ int parser_test( const  wchar_t * buff,
 	int previous_pos=current_tokenizer_pos;
 	static int block_pos[BLOCK_MAX_COUNT];
 	static int block_type[BLOCK_MAX_COUNT];
-
+	int res;
+	
 	/*
 	  Set to 1 if the current command is inside a pipeline
 	*/
@@ -3037,7 +3038,7 @@ int parser_test( const  wchar_t * buff,
 	current_tokenizer = &tok;
 
 	for( tok_init( &tok, buff, 0 );
-		 tok_has_next( &tok );
+		 ;
 		 tok_next( &tok ) )
 	{
 		current_tokenizer_pos = tok_get_pos( &tok );
@@ -3666,6 +3667,8 @@ int parser_test( const  wchar_t * buff,
 			
 		}
 		
+		if( !tok_has_next( &tok ) )
+			break;
 		
 	}
 
@@ -3709,14 +3712,20 @@ int parser_test( const  wchar_t * buff,
 
 	tok_destroy( &tok );
 
-
 	current_tokenizer=previous_tokenizer;
 	current_tokenizer_pos = previous_pos;
-
+	
 	error_code=0;
-
+	
 	halloc_free( context );
-
-	return err | ((count!=0)<<1);
+	
+	res = 0;
+	if( err )
+		res |= PARSER_TEST_ERROR;
+	if( count!= 0 )
+		res |= PARSER_TEST_INCOMPLETE;
+	
+	return res;
+	
 }
 
diff --git a/parser.h b/parser.h
index d563b0fb..be5a3d26 100644
--- a/parser.h
+++ b/parser.h
@@ -12,6 +12,9 @@
 #include "parser.h"
 #include "event.h"
 
+#define PARSER_TEST_ERROR 1
+#define PARSER_TEST_INCOMPLETE 2
+
 /**
    event_block_t represents a block on events of the specified type
 */
diff --git a/reader.c b/reader.c
index f3bca334..61e83d58 100644
--- a/reader.c
+++ b/reader.c
@@ -98,6 +98,7 @@ commence.
 #include "function.h"
 #include "output.h"
 #include "signal.h"
+#include "screen.h"
 
 #include "parse_util.h"
 
@@ -139,15 +140,7 @@ typedef struct reader_data
 	*/
 	wchar_t *buff;
 
-	/**
-	   The output string, may be different than buff if buff can't fit on one line.
-	*/
-	wchar_t *output;
-
-	/**
-	   The number of characters used by the prompt
-	*/
-	int prompt_width;
+	screen_t screen;
 
 	/**
 	   Buffer containing the current search item
@@ -192,18 +185,16 @@ typedef struct reader_data
 	size_t buff_pos;
 
 	/**
-	   The current position of the cursor in output buffer.
-	*/
-	size_t output_pos;
-
-	/**
 	   Name of the current application
 	*/
 	wchar_t *name;
 
-	/** The prompt text */
+	/** The prompt command */
 	wchar_t *prompt;
 
+	/** The output of the last evaluation of the prompt command */
+	string_buffer_t prompt_buff;
+	
 	/**
 	   Color is the syntax highlighting for buff.  The format is that
 	   color[i] is the classification (according to the enum in
@@ -217,11 +208,6 @@ typedef struct reader_data
 	int *new_color;
 
 	/**
-	   Color for the actual output string.
-	*/
-	int *output_color;
-
-	/**
 	   Should the prompt command be reexecuted on the next repaint
 	*/
 	int exec_prompt;
@@ -249,6 +235,7 @@ typedef struct reader_data
 	   When this is true, the reader will exit
 	*/
 	int end_loop;
+
 	/**
 	   If this is true, exit reader even if there are running
 	   jobs. This happens if we press e.g. ^D twice.
@@ -451,15 +438,12 @@ static int check_size()
 							  sizeof(wchar_t)*data->buff_sz);
 		data->search_buff = realloc( data->search_buff,
 									 sizeof(wchar_t)*data->buff_sz);
-		data->output = realloc( data->output,
-								sizeof(wchar_t)*data->buff_sz);
 
 		data->color = realloc( data->color,
 							   sizeof(int)*data->buff_sz);
+
 		data->new_color = realloc( data->new_color,
 								   sizeof(int)*data->buff_sz);
-		data->output_color = realloc( data->output_color,
-									  sizeof(int)*data->buff_sz);
 
 		if( data->buff==0 ||
 			data->search_buff==0 ||
@@ -467,112 +451,12 @@ static int check_size()
 			data->new_color == 0 )
 		{
 			DIE_MEM();
-
 		}
 	}
 	return 1;
 }
 
 /**
-   Check if the screen is not wide enough for the buffer, which means
-   the buffer must be scrolled on input and cursor movement.
-*/
-static int force_repaint()
-{
-	int max_width = common_get_width() - data->prompt_width;
-	int pref_width = my_wcswidth( data->buff ) + (data->buff_pos==data->buff_len);
-	return pref_width >= max_width;
-}
-
-
-/**
-   Calculate what part of the buffer should be visible
-
-   \return returns 1 screen needs repainting, 0 otherwise
-*/
-static int calc_output()
-{
-	int max_width = common_get_width() - data->prompt_width;
-	int pref_width = my_wcswidth( data->buff ) + (data->buff_pos==data->buff_len);
-	if( pref_width <= max_width )
-	{
-		wcscpy( data->output, data->buff );
-		memcpy( data->output_color, data->color, sizeof(int) * data->buff_len );
-		data->output_pos=data->buff_pos;
-
-		return 1;
-	}
-	else
-	{
-		int offset = data->buff_pos;
-		int offset_end = data->buff_pos;
-		int w = 0;
-		wchar_t *pos=data->output;
-		*pos=0;
-
-
-		w = (data->buff_pos==data->buff_len)?1:wcwidth( data->buff[offset] );
-		while( 1 )
-		{
-			int inc=0;
-			int ellipsis_width;
-
-			ellipsis_width = wcwidth(ellipsis_char)*((offset?1:0)+(offset_end<data->buff_len?1:0));
-
-			if( offset > 0 && (ellipsis_width + w + wcwidth( data->buff[offset-1] ) <= max_width ) )
-			{
-				inc=1;
-				offset--;
-				w+= wcwidth( data->buff[offset]);
-			}
-
-			ellipsis_width = wcwidth(ellipsis_char)*((offset?1:0)+(offset_end<data->buff_len?1:0));
-
-			if( offset_end < data->buff_len && (ellipsis_width + w + wcwidth( data->buff[offset_end+1] ) <= max_width ) )
-			{
-				inc = 1;
-				offset_end++;
-				w+= wcwidth( data->buff[offset_end]);
-			}
-
-			if( !inc )
-				break;
-
-		}
-
-		data->output_pos = data->buff_pos - offset + (offset?1:0);
-
-		if( offset )
-		{
-			data->output[0]=ellipsis_char;
-			data->output[1]=0;
-
-		}
-
-		wcsncat( data->output,
-				 data->buff+offset,
-				 offset_end-offset );
-
-		if( offset_end<data->buff_len )
-		{
-			int l = wcslen(data->output);
-
-			data->output[l]=ellipsis_char;
-			data->output[l+1]=0;
-
-		}
-
-		*data->output_color=HIGHLIGHT_NORMAL;
-
-		memcpy( data->output_color+(offset?1:0),
-				data->color+offset,
-				sizeof(int) * (data->buff_len-offset) );
-		return 1;
-	}
-}
-
-
-/**
    Compare two completions, ignoring their description.
 */
 static int fldcmp( wchar_t *a, wchar_t *b )
@@ -624,6 +508,7 @@ static void remove_duplicates( array_list_t *l )
    Translate a highlighting code ()Such as  as returned by the highlight function
    into a color code which is then passed on to set_color.
 */
+
 static void set_color_translated( int c )
 {
 	set_color( highlight_get_color( c & 0xffff ),
@@ -685,162 +570,15 @@ void reader_write_title()
 	set_color( FISH_COLOR_RESET, FISH_COLOR_RESET );
 }
 
-/**
-   Tests if the specified narrow character sequence is present at the
-   specified position of the specified wide character string. All of
-   \c seq must match, but str may be longer than seq.
-*/
-static int try_sequence( char *seq, wchar_t *str )
-{
-	int i;
-
-	for( i=0;; i++ )
-	{
-		if( !seq[i] )
-			return i;
-
-		if( seq[i] != str[i] )
-			return 0;
-	}
-
-	return 0;
-}
-
-/**
-   Calculate the width of the specified prompt. Does some clever magic
-   to detect common escape sequences that may be embeded in a prompt,
-   such as color codes.
-*/
-static int calc_prompt_width( array_list_t *arr )
-{
-	int res = 0;
-	int i, j, k;
-
-	for( i=0; i<al_get_count( arr ); i++ )
-	{
-		wchar_t *next = (wchar_t *)al_get( arr, i );
-
-		for( j=0; next[j]; j++ )
-		{
-			if( next[j] == L'\e' )
-			{
-				/*
-				  This is the start of an escape code. Try to guess it's width.
-				*/
-				int l;
-				int len=0;
-				int found = 0;
-
-				/*
-				  Detect these terminfo color escapes with parameter
-				  value 0..7, all of which don't move the cursor
-				*/
-				char * esc[] =
-					{
-						set_a_foreground,
-						set_a_background,
-						set_foreground,
-						set_background,
-					}
-				;
-
-				/*
-				  Detect these semi-common terminfo escapes without any
-				  parameter values, all of which don't move the cursor
-				*/
-				char *esc2[] =
-					{
-						enter_bold_mode,
-						exit_attribute_mode,
-						enter_underline_mode,
-						exit_underline_mode,
-						enter_standout_mode,
-						exit_standout_mode,
-						flash_screen,
-						enter_subscript_mode,
-						exit_subscript_mode,
-						enter_superscript_mode,
-						exit_superscript_mode,
-						enter_blink_mode,
-						enter_italics_mode,
-						exit_italics_mode,
-						enter_reverse_mode,
-						enter_shadow_mode,
-						exit_shadow_mode,
-						enter_standout_mode,
-						exit_standout_mode,
-						enter_secure_mode
-					}
-				;
-
-				for( l=0; l < (sizeof(esc)/sizeof(char *)) && !found; l++ )
-				{
-					if( !esc[l] )
-						continue;
-
-					for( k=0; k<8; k++ )
-					{
-						len = try_sequence( tparm(esc[l],k), &next[j] );
-						if( len )
-						{
-							j += (len-1);
-							found = 1;
-							break;
-						}
-					}
-				}
-
-				for( l=0; l < (sizeof(esc2)/sizeof(char *)) && !found; l++ )
-				{
-					if( !esc2[l] )
-						continue;
-					/*
-					  Test both padded and unpadded version, just to
-					  be safe. Most versions of tparm don't actually
-					  seem to do anything these days.
-					*/
-					len = maxi( try_sequence( tparm(esc2[l]), &next[j] ),
-								try_sequence( esc2[l], &next[j] ));
-					
-					if( len )
-					{
-						j += (len-1);
-						found = 1;
-					}
-				}
-			}
-			else if( next[j] == L'\t' )
-			{
-				/*
-				  Assume tab stops every 8 characters if undefined
-				*/
-				if( init_tabs <= 0 )
-					init_tabs = 8;
-				
-				res=( (res/init_tabs)+1 )*init_tabs;
-			}
-			else
-			{
-				/*
-				  Ordinary decent character. Just add width.
-				*/
-				res += wcwidth( next[j] );
-			}
-		}
-	}
-	return res;
-}
-
 
 /**
    Write the prompt to screen. If data->exec_prompt is set, the prompt
    command is first evaluated, and the title will be reexecuted as
    well.
 */
-static void write_prompt()
+static void calc_prompt()
 {
 	int i;
-	set_color( FISH_COLOR_NORMAL, FISH_COLOR_NORMAL );
 
 	/*
 	  Check if we need to reexecute the prompt command
@@ -865,21 +603,17 @@ static void write_prompt()
 			proc_pop_interactive();
 		}
 
-		data->prompt_width=calc_prompt_width( &prompt_list );
-
 		data->exec_prompt = 0;
 		reader_write_title();
-	}
 
-	/*
-	  Write out the prompt strings
-	*/
+		sb_clear( &data->prompt_buff );
+		
+		for( i=0; i<al_get_count( &prompt_list); i++ )
+		{
+			sb_append( &data->prompt_buff, (wchar_t *)al_get( &prompt_list, i ) );
+		}
 
-	for( i=0; i<al_get_count( &prompt_list); i++ )
-	{
-		writestr( (wchar_t *)al_get( &prompt_list, i ) );
 	}
-	set_color( FISH_COLOR_RESET, FISH_COLOR_RESET );
 
 }
 
@@ -887,7 +621,7 @@ static void write_prompt()
    Write the whole command line (but not the prompt) to the screen. Do
    not set the cursor correctly afterwards.
 */
-static void write_cmdline()
+/*static void write_cmdline()
 {
 	int i;
 
@@ -897,7 +631,7 @@ static void write_cmdline()
 		writech( data->output[i] );
 	}
 }
-
+*/
 
 void reader_init()
 {
@@ -942,48 +676,21 @@ void reader_exit( int do_exit, int forced )
 
 void repaint( int skip_return )
 {
-	int steps;
-
-	calc_output();
-	set_color( FISH_COLOR_RESET, FISH_COLOR_RESET );
-
-	if( !skip_return )
-		writech('\r');
-
-	writembs(clr_eol);
-	write_prompt();
-	write_cmdline();
-
-/*
-  fwprintf( stderr, L"Width of \'%ls\' (length is %d): ",
-  &data->buff[data->buff_pos],
-  wcslen(&data->buff[data->buff_pos]));
-  fwprintf( stderr, L"%d\n", my_wcswidth(&data->buff[data->buff_pos]));
-*/
+	int flags = 0;
+	
+	if( skip_return )
+		flags |= SCREEN_SKIP_RETURN;
+		
+	calc_prompt();
 
-	steps = my_wcswidth( &data->output[data->output_pos]);
-	if( steps )
-		move_cursor( -steps );
+//	assert( wcslen( (wchar_t *)data->prompt_buff.buff));
 
-	set_color( FISH_COLOR_NORMAL, FISH_COLOR_IGNORE );
+	s_write( &data->screen, (wchar_t *)data->prompt_buff.buff, data->buff, data->color, data->buff_pos, flags );
+	
 	reader_save_status();
 }
 
 /**
-   Make sure color values are correct, and repaint if they are not.
-*/
-static void check_colors()
-{
-	reader_super_highlight_me_plenty( data->new_color, data->buff_pos, 0 );
-	if( memcmp( data->new_color, data->color, sizeof(int)*data->buff_len )!=0 )
-	{
-		memcpy( data->color, data->new_color,  sizeof(int)*data->buff_len );
-
-		repaint( 0 );
-	}
-}
-
-/**
    Stat stdout and stderr and save result.
 
    This should be done before calling a function that may cause output.
@@ -1052,7 +759,6 @@ static void reader_check_status()
 	if( changed )
 	{
 		repaint( 0 );
-		set_color( FISH_COLOR_RESET, FISH_COLOR_RESET );
 	}
 }
 
@@ -1062,7 +768,6 @@ static void reader_check_status()
 */
 static void remove_backward()
 {
-	int wdt;
 
 	if( data->buff_pos <= 0 )
 		return;
@@ -1072,47 +777,17 @@ static void remove_backward()
 		memmove( &data->buff[data->buff_pos-1],
 				 &data->buff[data->buff_pos],
 				 sizeof(wchar_t)*(data->buff_len-data->buff_pos+1) );
-
-		memmove( &data->color[data->buff_pos-1],
-				 &data->color[data->buff_pos],
-				 sizeof(wchar_t)*(data->buff_len-data->buff_pos+1) );
 	}
 	data->buff_pos--;
 	data->buff_len--;
+	data->buff[data->buff_len]=0;
 
-	wdt=wcwidth(data->buff[data->buff_pos]);
-	move_cursor(-wdt);
-	data->buff[data->buff_len]='\0';
-//	wcscpy(data->search_buff,data->buff);
-
-	reader_super_highlight_me_plenty( data->new_color,
+	reader_super_highlight_me_plenty( data->color,
 									  data->buff_pos,
 									  0 );
-	if( (!force_repaint()) && ( memcmp( data->new_color,
-										data->color,
-										sizeof(int)*data->buff_len )==0 ) &&
-		( delete_character != 0) && (wdt==1) )
-	{
-		/*
-		  Only do this if delete mode functions, and only for a column
-		  wide characters, since terminfo seems to break for other
-		  characters. This last check should be removed when terminfo
-		  is fixed.
-		*/
-		if( enter_delete_mode != 0 )
-			writembs(enter_delete_mode);
-		writembs(delete_character);
-		if( exit_delete_mode != 0 )
-			writembs(exit_delete_mode);
-	}
-	else
-	{
-		memcpy( data->color,
-				data->new_color,
-				sizeof(int) * data->buff_len );
 
-		repaint( 0 );
-	}
+	repaint( 0 );
+
 }
 
 /**
@@ -1124,9 +799,7 @@ static void remove_forward()
 	if( data->buff_pos >= data->buff_len )
 		return;
 
-	move_cursor(wcwidth(data->buff[data->buff_pos]));
 	data->buff_pos++;
-
 	remove_backward();
 }
 
@@ -1146,10 +819,6 @@ static int insert_char( int c )
 		memmove( &data->buff[data->buff_pos+1],
 				 &data->buff[data->buff_pos],
 				 sizeof(wchar_t)*(data->buff_len-data->buff_pos) );
-
-		memmove( &data->color[data->buff_pos+1],
-				 &data->color[data->buff_pos],
-				 sizeof(int)*(data->buff_len-data->buff_pos) );
 	}
 	/* Set character */
 	data->buff[data->buff_pos]=c;
@@ -1161,48 +830,12 @@ static int insert_char( int c )
 
 	/* Syntax highlight */
 
-	reader_super_highlight_me_plenty( data->new_color,
+	reader_super_highlight_me_plenty( data->color,
 									  data->buff_pos-1,
 									  0 );
-	data->color[data->buff_pos-1] = data->new_color[data->buff_pos-1];
-
-	/* Check if the coloring has changed */
-	if( (!force_repaint()) && ( memcmp( data->new_color,
-										data->color,
-										sizeof(int)*data->buff_len )==0 ) &&
-		( insert_character ||
-		  ( data->buff_pos == data->buff_len ) ||
-		  enter_insert_mode) )
-	{
-		/*
-		  Colors look ok, so we set the right color and insert a
-		  character
-		*/
-		set_color_translated( data->color[data->buff_pos-1] );
-		if( data->buff_pos < data->buff_len )
-		{
-			if( enter_insert_mode != 0 )
-				writembs(enter_insert_mode);
-			else
-				writembs(insert_character);
-			writech(c);
-			if( insert_padding != 0 )
-				writembs(insert_padding);
-			if( exit_insert_mode != 0 )
-				writembs(exit_insert_mode);
-		}
-		else
-			writech(c);
-		set_color( FISH_COLOR_NORMAL, FISH_COLOR_IGNORE );
-	}
-	else
-	{
-		/* Nope, colors are off, so we repaint the entire command line */
-		memcpy( data->color, data->new_color, sizeof(int) * data->buff_len );
 
-		repaint( 0 );
-	}
-//	wcscpy(data->search_buff,data->buff);
+	repaint( 0 );
+
 	return 1;
 }
 
@@ -1213,42 +846,31 @@ static int insert_char( int c )
 static int insert_str(wchar_t *str)
 {
 	int len = wcslen( str );
-	if( len < 4 )
-	{
-		while( (*str)!=0 )
-			if(!insert_char( *str++ ))
-				return 0;
-	}
-	else
+	int old_len = data->buff_len;
+	
+	data->buff_len += len;
+	check_size();
+	
+	/* Insert space for extra characters at the right position */
+	if( data->buff_pos < old_len )
 	{
-		int old_len = data->buff_len;
-
-		data->buff_len += len;
-		check_size();
-
-		/* Insert space for extra characters at the right position */
-		if( data->buff_pos < old_len )
-		{
-			memmove( &data->buff[data->buff_pos+len],
-					 &data->buff[data->buff_pos],
-					 sizeof(wchar_t)*(data->buff_len-data->buff_pos) );
-		}
-		memmove( &data->buff[data->buff_pos], str, sizeof(wchar_t)*len );
-		data->buff_pos += len;
-		data->buff[data->buff_len]='\0';
-
-		/* Syntax highlight */
-
-		reader_super_highlight_me_plenty( data->new_color,
-										  data->buff_pos-1,
-										  0 );
-		memcpy( data->color, data->new_color, sizeof(int) * data->buff_len );
-
-		/* repaint */
-
-		repaint( 0 );
-
+		memmove( &data->buff[data->buff_pos+len],
+				 &data->buff[data->buff_pos],
+				 sizeof(wchar_t)*(data->buff_len-data->buff_pos) );
 	}
+	memmove( &data->buff[data->buff_pos], str, sizeof(wchar_t)*len );
+	data->buff_pos += len;
+	data->buff[data->buff_len]='\0';
+	
+	/* Syntax highlight */
+	
+	reader_super_highlight_me_plenty( data->color,
+									  data->buff_pos-1,
+									  0 );
+	
+	/* repaint */
+	
+	repaint( 0 );
 	return 1;
 }
 
@@ -1501,7 +1123,10 @@ static void run_pager( wchar_t *prefix, int is_quoted, array_list_t *comp )
 	
 	for( i=0; i<al_get_count( comp); i++ )
 	{
-	    wchar_t *el = escape((wchar_t*)al_get( comp, i ), 0);
+	    wchar_t *el = escape((wchar_t*)al_get( comp, i ), 1);
+
+//		debug( 0, L"Escaped '%ls' to '%ls'", al_get( comp, i ), el );
+		
 		
 		sb_printf( &msg, L"%ls\n", el );
 		free( el );		
@@ -2103,24 +1728,7 @@ static void move_word( int dir, int erase )
 /*		move_cursor(end_buff_pos-data->buff_pos);
   data->buff_pos = end_buff_pos;
 */
-		if( end_buff_pos < data->buff_pos )
-		{
-			while( data->buff_pos != end_buff_pos )
-			{
-				data->buff_pos--;
-				move_cursor( -wcwidth(data->buff[data->buff_pos]));
-			}
-		}
-		else
-		{
-			while( data->buff_pos != end_buff_pos )
-			{
-				move_cursor( wcwidth(data->buff[data->buff_pos]));
-				data->buff_pos++;
-				check_colors();
-			}
-		}
-
+		data->buff_pos = end_buff_pos;
 		repaint( 0 );
 //		check_colors();
 	}
@@ -2204,18 +1812,22 @@ void reader_run_command( const wchar_t *cmd )
 
 static int shell_test( wchar_t *b )
 {
-	if( parser_test( b, 0, 0 ) )
+	int res = parser_test( b, 0, 0 );
+	
+	if( res & PARSER_TEST_ERROR )
 	{
 		string_buffer_t sb;
 		sb_init( &sb );
+
+		int tmp[1];
+		
+		s_write( &data->screen, L"", L"", tmp, 0, 0 );
 		
-		writech( L'\n' );
 		parser_test( b, &sb, L"fish" );
 		fwprintf( stderr, L"%ls", sb.buff );
 		sb_destroy( &sb );
-		return 1;
 	}
-	return 0;
+	return res;
 }
 
 /**
@@ -2233,7 +1845,12 @@ void reader_push( wchar_t *name )
 	reader_data_t *n = calloc( 1, sizeof( reader_data_t ) );
 	n->name = wcsdup( name );
 	n->next = data;
+
 	data=n;
+
+	s_init( &data->screen );
+	sb_init( &data->prompt_buff );
+
 	check_size();
 	data->buff[0]=data->search_buff[0]=0;
 	data->exec_prompt=1;
@@ -2271,8 +1888,9 @@ void reader_pop()
 	free( n->color );
 	free( n->new_color );
 	free( n->search_buff );
-	free( n->output );
-	free( n->output_color );
+	
+	s_destroy( &n->screen );
+	sb_destroy( &n->prompt_buff );
 
 	/*
 	  Clean up after history search
@@ -2355,8 +1973,6 @@ static void reader_super_highlight_me_plenty( int *color, int match_highlight_po
 		}
 	}
 
-	color[data->buff_pos] = 0;
-		
 }
 
 
@@ -2380,7 +1996,7 @@ static int read_i()
 	reader_set_highlight_function( &highlight_shell );
 	reader_set_test_function( &shell_test );
 
-	data->prompt_width=60;
+//	data->prompt_width=60;
 	data->prev_end_loop=0;
 
 	while( (!data->end_loop) && (!sanity_check()) )
@@ -2417,7 +2033,7 @@ static int read_i()
 			if( !reader_exit_forced() && !data->prev_end_loop && has_job )
 			{
 				writestr(_( L"There are stopped jobs\n" ));
-				write_prompt();
+				repaint( 0 );
 				data->end_loop = 0;
 				data->prev_end_loop=1;
 			}
@@ -2480,6 +2096,8 @@ wchar_t *reader_readline()
 
 	al_init( &comp );
 
+	s_reset( &data->screen );
+	
 	data->exec_prompt=1;
 
 	reader_super_highlight_me_plenty( data->color, data->buff_pos, 0 );
@@ -2584,6 +2202,7 @@ wchar_t *reader_readline()
 			case R_NULL:
 			{
 				data->exec_prompt=1;
+				s_reset( &data->screen );
 				repaint( 0 );
 				break;
 			}
@@ -2726,7 +2345,8 @@ wchar_t *reader_readline()
 						reader_replace_current_token( data->search_buff );
 					}
 					*data->search_buff=0;
-					check_colors();
+					repaint(0);
+					//check_colors();
 
 				}
 
@@ -2761,29 +2381,56 @@ wchar_t *reader_readline()
 			/* Newline, evaluate*/
 			case L'\n':
 			{
-				data->buff[data->buff_len]=L'\0';
-
-				if( !data->test_func( data->buff ) )
+				/*
+				  Allow backslash-escaped newlines
+				*/
+				if( data->buff_len && data->buff[data->buff_len-1]==L'\\' )
+				{
+					insert_char( '\n' );
+					break;
+				}
+				
+				switch( data->test_func( data->buff ) )
 				{
 
-					if( wcslen( data->buff ) )
+					case 0:
 					{
+						/*
+						  Finished commend, execute it
+						*/
+						if( wcslen( data->buff ) )
+						{
 //						wcscpy(data->search_buff,L"");
-						history_add( data->buff );
+							history_add( data->buff );
+						}
+						finished=1;
+						data->buff_pos=data->buff_len;
+						repaint(0);
+						writestr( L"\n" );
+						break;
+					}
+					
+					/*
+					  We are incomplete, continue editing
+					*/
+					case PARSER_TEST_INCOMPLETE:
+					{						
+						insert_char( '\n' );
+						break;
+					}
+
+					/*
+					  Result must be some combination including an error. The error message will already be printed, all we need to do is repaint
+					*/
+					default:
+					{
+						s_reset( &data->screen );
+						repaint( 0 );
+						break;
 					}
-					finished=1;
-					data->buff_pos=data->buff_len;
-					check_colors();
-					writestr( L"\n" );
-				}
-				else
-				{
-					writech('\r');
-					writembs(clr_eol);
-					writech('\n');
-					repaint( 0 );
-				}
 
+				}
+				
 				break;
 			}
 
@@ -2858,15 +2505,7 @@ wchar_t *reader_readline()
 				if( data->buff_pos > 0 )
 				{
 					data->buff_pos--;
-					if( !force_repaint() )
-					{
-						move_cursor( -wcwidth(data->buff[data->buff_pos]));
-						check_colors();
-					}
-					else
-					{
-						repaint( 0 );
-					}
+					repaint( 0 );
 				}
 				break;
 			}
@@ -2876,18 +2515,8 @@ wchar_t *reader_readline()
 			{
 				if( data->buff_pos < data->buff_len )
 				{
-					if( !force_repaint() )
-					{
-						move_cursor( wcwidth(data->buff[data->buff_pos]));
-						data->buff_pos++;
-						check_colors();
-					}
-					else
-					{
-						data->buff_pos++;
-
-						repaint( 0 );
-					}
+					data->buff_pos++;					
+					repaint( 0 );
 				}
 				break;
 			}
@@ -2933,6 +2562,7 @@ wchar_t *reader_readline()
 			{
 				if( clear_screen )
 					writembs( clear_screen );
+				s_reset( &data->screen );
 				repaint( 0 );
 				break;
 			}
diff --git a/screen.c b/screen.c
new file mode 100644
index 00000000..fbc35dc3
--- /dev/null
+++ b/screen.c
@@ -0,0 +1,554 @@
+#include "config.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <termios.h>
+#include <sys/types.h>
+
+#ifdef HAVE_SYS_TERMIOS_H
+#include <sys/termios.h>
+#endif
+
+#include <unistd.h>
+#include <wctype.h>
+
+#if HAVE_NCURSES_H
+#include <ncurses.h>
+#else
+#include <curses.h>
+#endif
+
+#if HAVE_TERMIO_H
+#include <termio.h>
+#endif
+
+#if HAVE_TERM_H
+#include <term.h>
+#elif HAVE_NCURSES_TERM_H
+#include <ncurses/term.h>
+#endif
+
+#include <wchar.h>
+
+#include <assert.h>
+
+
+#include "fallback.h"
+#include "common.h"
+#include "util.h"
+#include "wutil.h"
+#include "output.h"
+#include "highlight.h"
+#include "screen.h"
+
+static buffer_t *s_writeb_buffer=0;
+
+/**
+   Tests if the specified narrow character sequence is present at the
+   specified position of the specified wide character string. All of
+   \c seq must match, but str may be longer than seq.
+*/
+static int try_sequence( char *seq, wchar_t *str )
+{
+	int i;
+
+	for( i=0;; i++ )
+	{
+		if( !seq[i] )
+			return i;
+
+		if( seq[i] != str[i] )
+			return 0;
+	}
+
+	return 0;
+}
+
+/**
+   Calculate the width of the specified prompt. Does some clever magic
+   to detect common escape sequences that may be embeded in a prompt,
+   such as color codes.
+*/
+static int calc_prompt_width( wchar_t *prompt )
+{
+	int res = 0;
+	int j, k;
+
+	for( j=0; prompt[j]; j++ )
+	{
+		if( prompt[j] == L'\e' )
+		{
+			/*
+			  This is the start of an escape code. Try to guess it's width.
+			*/
+				int l;
+				int len=0;
+				int found = 0;
+
+				/*
+				  Detect these terminfo color escapes with parameter
+				  value 0..7, all of which don't move the cursor
+				*/
+				char * esc[] =
+					{
+						set_a_foreground,
+						set_a_background,
+						set_foreground,
+						set_background,
+					}
+				;
+
+				/*
+				  Detect these semi-common terminfo escapes without any
+				  parameter values, all of which don't move the cursor
+				*/
+				char *esc2[] =
+					{
+						enter_bold_mode,
+						exit_attribute_mode,
+						enter_underline_mode,
+						exit_underline_mode,
+						enter_standout_mode,
+						exit_standout_mode,
+						flash_screen,
+						enter_subscript_mode,
+						exit_subscript_mode,
+						enter_superscript_mode,
+						exit_superscript_mode,
+						enter_blink_mode,
+						enter_italics_mode,
+						exit_italics_mode,
+						enter_reverse_mode,
+						enter_shadow_mode,
+						exit_shadow_mode,
+						enter_standout_mode,
+						exit_standout_mode,
+						enter_secure_mode
+					}
+				;
+
+				for( l=0; l < (sizeof(esc)/sizeof(char *)) && !found; l++ )
+				{
+					if( !esc[l] )
+						continue;
+
+					for( k=0; k<8; k++ )
+					{
+						len = try_sequence( tparm(esc[l],k), &prompt[j] );
+						if( len )
+						{
+							j += (len-1);
+							found = 1;
+							break;
+						}
+					}
+				}
+
+				for( l=0; l < (sizeof(esc2)/sizeof(char *)) && !found; l++ )
+				{
+					if( !esc2[l] )
+						continue;
+					/*
+					  Test both padded and unpadded version, just to
+					  be safe. Most versions of tparm don't actually
+					  seem to do anything these days.
+					*/
+					len = maxi( try_sequence( tparm(esc2[l]), &prompt[j] ),
+								try_sequence( esc2[l], &prompt[j] ));
+					
+					if( len )
+					{
+						j += (len-1);
+						found = 1;
+					}
+				}
+			}
+			else if( prompt[j] == L'\t' )
+			{
+				/*
+				  Assume tab stops every 8 characters if undefined
+				*/
+				if( init_tabs <= 0 )
+					init_tabs = 8;
+				
+				res=( (res/init_tabs)+1 )*init_tabs;
+			}
+			else
+			{
+				/*
+				  Ordinary decent character. Just add width.
+				*/
+				res += wcwidth( prompt[j] );
+			}
+		}
+	return res;
+}
+
+
+
+static void free_line( void *l )
+{
+	line_t *line = (line_t *)l;
+//	debug( 0, L"Free line at %d", l);
+	al_destroy( &line->text );
+	al_destroy( &line->color );
+	free( line );
+}
+
+
+static void s_reset_arr( array_list_t *l )
+{
+//	debug( 0, L"I have %d lines", al_get_count( l ));
+	
+	al_foreach( l, &free_line );
+	al_truncate( l, 0 );
+}
+
+void s_init( screen_t *s )
+{
+	memset( s, 0, sizeof(screen_t));
+	sb_init( &s->prompt_buff );
+}
+
+
+void s_destroy( screen_t *s )
+{
+	s_reset_arr( &s->screen );
+	al_destroy( &s->screen );
+	s_reset_arr( &s->output );
+	al_destroy( &s->output );
+}
+
+static line_t *s_create_line()
+{
+	line_t *current = malloc( sizeof( line_t ));
+	al_init( &current->text );
+	al_init( &current->color );
+	return current;
+}
+
+/*
+  Appends a character to the end of the line that the output cursor is on
+*/
+static void s_output_append_char( screen_t *s, wchar_t b, int c, int prompt_width )
+{
+	int line_no = s->output_cursor[1];
+	
+	switch( b )
+	{
+		case L'\n':
+		{
+			int i;
+			line_t *current = s_create_line();
+			al_push( &s->output, current );
+			s->output_cursor[1]++;
+			s->output_cursor[0]=0;
+			for( i=0; i < prompt_width; i++ )
+			{
+				s_output_append_char( s, L' ', 0, prompt_width );
+			}
+			break;
+		}
+		
+		case L'\r':
+		{
+			line_t *current;
+			current = (line_t *)al_get( &s->output, line_no );
+			al_truncate( &current->text, 0 );
+			al_truncate( &current->color, 0 );			
+			s->output_cursor[0]=0;
+			break;			
+		}		
+
+		default:
+		{
+			line_t *current;
+			current = (line_t *)al_get( &s->output, line_no );
+
+			if( !current )
+			{
+				current = s_create_line();
+				al_push( &s->output, current );
+			}
+			
+			al_push_long( &current->text, b );
+			al_push_long( &current->color, c );
+			s->output_cursor[0]+= wcwidth(b);
+			break;
+		}
+	}
+	
+}
+
+static int s_writeb( char c )
+{
+	b_append( s_writeb_buffer, &c, 1 );
+	return 0;
+}
+
+
+static void s_move( screen_t *s, buffer_t *b, int new_x, int new_y )
+{
+	int i;
+	int x_steps, y_steps;
+	
+	int (*writer_old)(char) = output_get_writer();
+
+	char *str;
+/*
+	debug( 0, L"move from %d %d to %d %d", 
+		   s->screen_cursor[0], s->screen_cursor[1],  
+		   new_x, new_y );
+*/
+	output_set_writer( &s_writeb );
+	s_writeb_buffer = b;
+	
+	y_steps = new_y - s->screen_cursor[1];
+
+	if( y_steps > 0 && (strcmp( cursor_down, "\n")==0))
+	{	
+		/*
+		  This is very strange - it seems all (most) consoles use a
+		  simple newline as the cursor down escape. This will of
+		  course move the cursor to the beginning of the line as
+		  well. The cursor_up does not have this behaviour...
+		*/
+		s->screen_cursor[0]=0;
+	}
+	
+	if( y_steps < 0 )
+	{
+		str = cursor_up;
+	}
+	else
+	{
+		str = cursor_down;
+		
+	}
+	
+	for( i=0; i<abs(y_steps); i++)
+	{
+		writembs(str);
+	}
+
+
+	x_steps = new_x - s->screen_cursor[0];
+	
+	if( x_steps && new_x == 0 )
+	{
+		char c = '\r';
+		b_append( b, &c, 1 );
+		x_steps = 0;
+//		debug( 0, L"return to first" );		
+	}
+		
+	if( x_steps < 0 ){
+		str = cursor_left;
+	}
+	else
+	{
+		str = cursor_right;
+	}
+	
+	for( i=0; i<abs(x_steps); i++)
+	{
+		writembs(str);
+	}
+
+
+	s->screen_cursor[0] = new_x;
+	s->screen_cursor[1] = new_y;
+	
+	output_set_writer( writer_old );
+	
+}
+
+static void s_set_color( screen_t *s, buffer_t *b, int c )
+{
+	
+	int (*writer_old)(char) = output_get_writer();
+
+	output_set_writer( &s_writeb );
+	s_writeb_buffer = b;
+	
+	set_color( highlight_get_color( c & 0xffff ),
+			   highlight_get_color( (c>>16)&0xffff ) );
+	
+	output_set_writer( writer_old );
+	
+}
+
+static void s_write_char( screen_t *s, buffer_t *b, wchar_t c )
+{
+	int (*writer_old)(char) = output_get_writer();
+
+	output_set_writer( &s_writeb );
+	s_writeb_buffer = b;
+	s->screen_cursor[0]+=wcwidth( c );
+	
+	writech( c );
+	
+	output_set_writer( writer_old );
+}
+
+static void s_write_mbs( buffer_t *b, char *s )
+{
+	int (*writer_old)(char) = output_get_writer();
+
+	output_set_writer( &s_writeb );
+	s_writeb_buffer = b;
+
+	writembs( s );
+	
+	output_set_writer( writer_old );
+}
+
+
+static void s_write_str( buffer_t *b, wchar_t *s )
+{
+	int (*writer_old)(char) = output_get_writer();
+
+	output_set_writer( &s_writeb );
+	s_writeb_buffer = b;
+
+	writestr( s );
+	
+	output_set_writer( writer_old );
+}
+
+
+static void s_update( screen_t *scr, wchar_t *prompt )
+{
+	int i, j;
+	int prompt_width = calc_prompt_width( prompt );
+	
+	buffer_t output;
+	b_init( &output );
+
+	if( wcscmp( prompt, (wchar_t *)scr->prompt_buff.buff ) )
+	{
+		s_move( scr, &output, 0, 0 );
+		s_write_str( &output, prompt );
+		sb_clear( &scr->prompt_buff );
+		sb_append( &scr->prompt_buff, prompt );		
+//		debug( 0, L"YAY %d", prompt_width );
+		scr->screen_cursor[0] = prompt_width;
+	}
+	
+	for( i=0; i< al_get_count( &scr->output ); i++ )
+	{
+		line_t *o_line = (line_t *)al_get( &scr->output, i );
+		line_t *s_line = (line_t *)al_get( &scr->screen, i );
+		
+		if( !s_line )
+		{
+			s_line = s_create_line();
+			al_push( &scr->screen, s_line );
+		}
+		for( j=(i==0?prompt_width:0); j<al_get_count( &o_line->text ); j++ )
+		{
+			wchar_t o = (wchar_t)al_get( &o_line->text, j );
+			int o_c = (int)al_get( &o_line->color, j );
+			if( al_get_count( &s_line->text ) == j )
+			{
+				s_move( scr, &output, j, i );
+				s_set_color( scr, &output, o_c );
+				s_write_char( scr, &output, o );
+				al_set_long( &s_line->text, j, o );
+				al_set_long( &s_line->color, j, o_c );				
+			}
+			else
+			{
+				wchar_t s = (wchar_t)al_get( &s_line->text, j );
+				int s_c = (int)al_get( &s_line->color, j );
+				if( o != s || o_c != s_c )
+				{
+					s_move( scr, &output, j, i );
+					s_set_color( scr, &output, o_c );
+					s_write_char( scr, &output, o );
+					al_set_long( &s_line->text, j, o );
+					al_set_long( &s_line->color, j, o_c );
+				}
+			}
+		}
+//		debug( 0, L"frum frum %d %d", al_get_count( &o_line->text ), al_get_count( &s_line->text ) );
+		if( al_get_count( &s_line->text ) > al_get_count( &o_line->text ) )
+		{
+			s_move( scr, &output, al_get_count( &o_line->text ), i );
+			s_write_mbs( &output, clr_eol);
+			al_truncate( &s_line->text, al_get_count( &o_line->text ) );
+//			debug( 0, L"YAY DELETE from %d", al_get_count( &o_line->text ) );			
+		}
+		
+	}
+	for( i=al_get_count( &scr->output ); i< al_get_count( &scr->screen ); i++ )
+	{
+		line_t *s_line = (line_t *)al_get( &scr->screen, i );
+		s_move( scr, &output, 0, i );
+		s_write_mbs( &output, clr_eol);
+		al_truncate( &s_line->text, 0 );
+	}
+	
+	s_move( scr, &output, scr->output_cursor[0], scr->output_cursor[1] );
+	
+	s_set_color( scr, &output, 0xffffffff);
+
+	if( output.used )
+	{
+		write( 1, output.buff, output.used );
+	}
+	
+	b_destroy( &output );
+	
+}
+
+
+void s_write( screen_t *s, wchar_t *prompt, wchar_t *b, int *c, int cursor, int flags )
+{
+	int i;
+	int cursor_arr[2];
+
+	int prompt_width = calc_prompt_width( prompt );
+
+//	debug( 0, L"Prompt width is %d", prompt_width );
+	
+	s_reset_arr( &s->output );
+	s->output_cursor[0] = s->output_cursor[1] = 0;
+	
+	for( i=0; i<prompt_width; i++ )
+	{
+		s_output_append_char( s, L' ', 0, prompt_width );
+	}
+	
+	for( i=0; b[i]; i++ )
+	{
+		int col = c[i];
+		
+		if( i == cursor )
+		{
+			memcpy(cursor_arr, s->output_cursor, sizeof(int)*2);
+			col = 0;
+		}
+
+		s_output_append_char( s, b[i], col, prompt_width );
+		
+	}
+	if( i == cursor )
+	{
+		memcpy(cursor_arr, s->output_cursor, sizeof(int)*2);
+	}
+	
+	memcpy( s->output_cursor, cursor_arr, sizeof(int)*2 );
+	s_update( s, prompt );
+	
+}
+
+void s_reset( screen_t *s )
+{
+	s_reset_arr( &s->screen );
+	s->screen_cursor[0] = s->screen_cursor[1] = 0;
+	sb_clear( &s->prompt_buff );
+}
+
diff --git a/screen.h b/screen.h
new file mode 100644
index 00000000..4a85edc0
--- /dev/null
+++ b/screen.h
@@ -0,0 +1,30 @@
+#ifndef FISH_SCREEN_H
+#define FISH_SCREEN_H
+
+#define SCREEN_REPAINT 1
+#define SCREEN_SKIP_RETURN 2
+
+typedef struct
+{
+	array_list_t output;
+	array_list_t screen;
+	int output_cursor[2];
+	int screen_cursor[2];
+	string_buffer_t prompt_buff;
+}
+	screen_t;
+
+typedef struct
+{
+	array_list_t text;
+	array_list_t color;
+}
+	line_t;
+
+void s_init( screen_t *s );
+void s_destroy( screen_t *s );
+
+void s_write( screen_t *s, wchar_t *prompt, wchar_t *b, int *c, int cursor, int flags );
+void s_reset( screen_t *s );
+
+#endif
diff --git a/tokenizer.c b/tokenizer.c
index 9be49c72..ae9e5bb8 100644
--- a/tokenizer.c
+++ b/tokenizer.c
@@ -234,132 +234,140 @@ static void read_string( tokenizer *tok )
 		{
 //			debug(1, L"%lc", *tok->buff );
 
-		if( *tok->buff == L'\\' )
-		{
-			tok->buff++;
-			if( *tok->buff == L'\0' )
+			if( *tok->buff == L'\\' )
 			{
-				tok_error( tok, EOL_ERROR );
-				return;
+				tok->buff++;
+				if( *tok->buff == L'\0' )
+				{
+					tok_error( tok, EOL_ERROR );
+					return;
+				}
+				else if( *tok->buff == L'\n' && mode == 0)
+				{
+					tok->buff--;
+					do_loop = 0;
+					break;
+				}
+				
+				tok->buff++;
+				continue;
 			}
-			tok->buff++;
-			continue;
-		}
-
-		/*
-		  The modes are as follows:
-
-		  0: regular text
-		  1: inside of subshell
-		  2: inside of array brackets
-		  3: inside of array brackets and subshell, like in '$foo[(ech'
-		*/
-		switch( mode )
-		{
-			case 0:
+			
+			
+			/*
+			  The modes are as follows:
+			  
+			  0: regular text
+			  1: inside of subshell
+			  2: inside of array brackets
+			  3: inside of array brackets and subshell, like in '$foo[(ech'
+			*/
+			switch( mode )
 			{
-				switch( *tok->buff )
+				case 0:
 				{
-					case L'(':
-					{
-						paran_count=1;
-						mode = 1;
-						break;
-					}
-
-					case L'[':
-					{
-						if( tok->buff != start )
-							mode=2;
-						break;
-					}
-
-					case L'\'':
-					case L'"':
+					switch( *tok->buff )
 					{
-
-						const wchar_t *end = quote_end( tok->buff );
-						tok->last_quote = *tok->buff;
-						if( end )
+						case L'(':
 						{
-							tok->buff=(wchar_t *)end;
+							paran_count=1;
+							mode = 1;
+							break;
 						}
-						else
+						
+						case L'[':
 						{
-							tok->buff += wcslen( tok->buff );
-
-							if( (!tok->accept_unfinished) )
+							if( tok->buff != start )
+								mode=2;
+							break;
+						}
+						
+						case L'\'':
+						case L'"':
+						{
+							
+							const wchar_t *end = quote_end( tok->buff );
+							tok->last_quote = *tok->buff;
+							if( end )
 							{
-								tok_error( tok, EOL_ERROR );
-								return;
+								tok->buff=(wchar_t *)end;
 							}
-							do_loop = 0;
+							else
+							{
+								tok->buff += wcslen( tok->buff );
+								
+								if( (!tok->accept_unfinished) )
+								{
+									tok_error( tok, EOL_ERROR );
+									return;
+								}
+								do_loop = 0;
 
+							}
+							break;
 						}
-						break;
-					}
 
-					default:
-					{
-						if( !is_string_char(*(tok->buff)) )
+						default:
 						{
-							do_loop=0;
+							if( !is_string_char(*(tok->buff)) )
+							{
+								do_loop=0;
+							}
 						}
 					}
+					break;
 				}
-				break;
-			}
 
-			case 3:
-			case 1:
-				switch( *tok->buff )
-				{
-					case L'\'':
-					case L'\"':
+				case 3:
+				case 1:
+					switch( *tok->buff )
 					{
-						const wchar_t *end = quote_end( tok->buff );
-						if( end )
+						case L'\'':
+						case L'\"':
 						{
-							tok->buff=(wchar_t *)end;
+							const wchar_t *end = quote_end( tok->buff );
+							if( end )
+							{
+								tok->buff=(wchar_t *)end;
+							}
+							else
+								do_loop = 0;
+							break;
 						}
-						else
+
+						case L'(':
+							paran_count++;
+							break;
+						case L')':
+							paran_count--;
+							if( paran_count == 0 )
+							{
+								mode--;
+							}
+							break;
+						case L'\0':
 							do_loop = 0;
-						break;
+							break;
 					}
+					break;
+				case 2:
+					switch( *tok->buff )
+					{
+						case L'(':
+							paran_count=1;
+							mode = 3;
+							break;
 
-					case L'(':
-						paran_count++;
-						break;
-					case L')':
-						paran_count--;
-						if( paran_count == 0 )
-						{
-							mode--;
-						}
-						break;
-					case L'\0':
-						do_loop = 0;
-						break;
-				}
-				break;
-			case 2:
-				switch( *tok->buff )
-				{
-					case L'(':
-						paran_count=1;
-						mode = 3;
-						break;
-
-					case L']':
-						mode=0;
-						break;
-
-					case L'\0':
-						do_loop = 0;
-						break;
-				}
-				break;
-		}
+						case L']':
+							mode=0;
+							break;
+
+						case L'\0':
+							do_loop = 0;
+							break;
+					}
+					break;
+			}
 		}
 
 
@@ -531,6 +539,8 @@ void tok_next( tokenizer *tok )
 			if(( *(tok->buff) == L'\\') &&( *(tok->buff+1) == L'\n') )
 			{
 				tok->buff+=2;
+				tok->last_type = TOK_END;
+				return;
 			}
 			break;
 		}
@@ -594,7 +604,6 @@ void tok_next( tokenizer *tok )
 
 		default:
 		{
-
 			if( iswdigit( *tok->buff ) )
 			{
 				wchar_t *orig = tok->buff;
diff --git a/util.c b/util.c
index 7c938bf3..1561eb13 100644
--- a/util.c
+++ b/util.c
@@ -923,16 +923,21 @@ int wcsfilecmp( const wchar_t *a, const wchar_t *b )
 	}
 
 	int res = wcsfilecmp( a+1, b+1 );
-	switch( abs(res) )
+
+	if( abs(res) < 2 )
 	{
-		case 2:
-			return res;
-		default:
-			if( secondary_diff )
-				return secondary_diff>0?1:-1;
+		/*
+		  No primary difference in rest of string.
+		  Use secondary difference on this element if found.
+		*/
+		if( secondary_diff )
+		{
+			return secondary_diff>0?1:-1;
+		}
 	}
-	return 0;
-
+	
+	return res;
+	
 }
 
 void sb_init( string_buffer_t * b)
diff --git a/util.h b/util.h
index 01cb8b91..13a16475 100644
--- a/util.h
+++ b/util.h
@@ -113,10 +113,20 @@ typedef struct array_list
 		Array containing the data
 	*/
 	anything_t *arr;
-	/** Position to append elements at*/
-	int pos;
-	/** Length of array */
-	int size;
+	
+	/** 
+		Internal cursor position of the array_list_t. This is the
+		position to append elements at. This is also what the
+		array_list_t considers to be its true size, as reported by
+		al_get_count(), etc. Calls to e.g. al_insert will preserve the
+		values of all elements up to pos.
+	*/
+	size_t pos;
+
+	/** 
+		Amount of memory allocated in arr, expressed in number of elements.
+	*/
+	size_t size;
 }
 array_list_t;