Remove trailing whitespaces and change tabs to spaces

1 files changed, 520 insertions, 520 deletions

diff --git a/tokenizer.cpp b/tokenizer.cpp
index 5af71bf9..537f1332 100644
--- a/tokenizer.cpp
+++ b/tokenizer.cpp
@@ -61,21 +61,21 @@ segments.
 */
 static const wchar_t *tok_desc[] =
 {
-	N_(L"Tokenizer not yet initialized"),
-	N_( L"Tokenizer error" ),
-	N_( L"Invalid token" ),
-	N_( L"String" ),
-	N_( L"Pipe" ),
-	N_( L"End of command" ),
-	N_( L"Redirect output to file" ),
-	N_( L"Append output to file" ),
-	N_( L"Redirect input to file" ),
-	N_( L"Redirect to file descriptor" ),
-	N_( L"Redirect output to file if file does not exist" ),
-	N_( L"Run job in background" ),
-	N_( L"Comment" )
+  N_(L"Tokenizer not yet initialized"),
+  N_( L"Tokenizer error" ),
+  N_( L"Invalid token" ),
+  N_( L"String" ),
+  N_( L"Pipe" ),
+  N_( L"End of command" ),
+  N_( L"Redirect output to file" ),
+  N_( L"Append output to file" ),
+  N_( L"Redirect input to file" ),
+  N_( L"Redirect to file descriptor" ),
+  N_( L"Redirect output to file if file does not exist" ),
+  N_( L"Run job in background" ),
+  N_( L"Comment" )
 }
-	;
+  ;
 
 /**
    Tests if the tokenizer buffer is large enough to hold contents of
@@ -85,19 +85,19 @@ static const wchar_t *tok_desc[] =
 */
 static int check_size( tokenizer *tok, size_t len )
 {
-	if( tok->last_len <= len )
-	{
-		wchar_t *tmp;
-		tok->last_len = len +1;
-		tmp = (wchar_t *)realloc( tok->last, sizeof(wchar_t)*tok->last_len );
-		if( tmp == 0 )
-		{
-			wperror( L"realloc" );
-			return 0;
-		}
-		tok->last = tmp;
-	}
-	return 1;
+  if( tok->last_len <= len )
+  {
+    wchar_t *tmp;
+    tok->last_len = len +1;
+    tmp = (wchar_t *)realloc( tok->last, sizeof(wchar_t)*tok->last_len );
+    if( tmp == 0 )
+    {
+      wperror( L"realloc" );
+      return 0;
+    }
+    tok->last = tmp;
+  }
+  return 1;
 }
 
 /**
@@ -105,21 +105,21 @@ static int check_size( tokenizer *tok, size_t len )
 */
 static void tok_call_error( tokenizer *tok, int error_type, const wchar_t *error_message )
 {
-	tok->last_type = TOK_ERROR;
-	tok->error = error_type;
-	if( !check_size( tok, wcslen( error_message)+1 ))
-	{
-		if( tok->last != 0 )
-			*tok->last=0;
-		return;
-	}
-
-	wcscpy( tok->last, error_message );
+  tok->last_type = TOK_ERROR;
+  tok->error = error_type;
+  if( !check_size( tok, wcslen( error_message)+1 ))
+  {
+    if( tok->last != 0 )
+      *tok->last=0;
+    return;
+  }
+
+  wcscpy( tok->last, error_message );
 }
 
 int tok_get_error( tokenizer *tok )
 {
-	return tok->error;
+  return tok->error;
 }
 
 
@@ -130,58 +130,58 @@ void tok_init( tokenizer *tok, const wchar_t *b, int flags )
     if (! (flags & TOK_SQUASH_ERRORS)) {
         ASSERT_IS_MAIN_THREAD();
     }
-    
-	CHECK( tok, );
 
-	memset( tok, 0, sizeof( tokenizer) );
+  CHECK( tok, );
+
+  memset( tok, 0, sizeof( tokenizer) );
 
-	CHECK( b, );
+  CHECK( b, );
 
 
-	tok->accept_unfinished = !! (flags & TOK_ACCEPT_UNFINISHED);
-	tok->show_comments = !! (flags & TOK_SHOW_COMMENTS);
+  tok->accept_unfinished = !! (flags & TOK_ACCEPT_UNFINISHED);
+  tok->show_comments = !! (flags & TOK_SHOW_COMMENTS);
     tok->squash_errors = !! (flags & TOK_SQUASH_ERRORS);
-	tok->has_next=true;
+  tok->has_next=true;
 
-	tok->has_next = (*b != L'\0');
-	tok->orig_buff = tok->buff = b;
+  tok->has_next = (*b != L'\0');
+  tok->orig_buff = tok->buff = b;
     tok->cached_lineno_offset = 0;
     tok->cached_lineno_count = 0;
-	tok_next( tok );
+  tok_next( tok );
 }
 
 void tok_destroy( tokenizer *tok )
 {
-	CHECK( tok, );
-	
-	free( tok->last );
+  CHECK( tok, );
+
+  free( tok->last );
 }
 
 int tok_last_type( tokenizer *tok )
 {
-	CHECK( tok, TOK_ERROR );
-	CHECK( tok->buff, TOK_ERROR );
-	
-	return tok->last_type;
+  CHECK( tok, TOK_ERROR );
+  CHECK( tok->buff, TOK_ERROR );
+
+  return tok->last_type;
 }
 
 wchar_t *tok_last( tokenizer *tok )
 {
-	CHECK( tok, 0 );
-	
-	return tok->last;
+  CHECK( tok, 0 );
+
+  return tok->last;
 }
 
 int tok_has_next( tokenizer *tok )
 {
-	/*
-	  Return 1 on broken tokenizer
-	*/
-	CHECK( tok, 1 );
-	CHECK( tok->buff, 1 );
-
-/*	fwprintf( stderr, L"has_next is %ls \n", tok->has_next?L"true":L"false" );*/
-	return 	tok->has_next;
+  /*
+    Return 1 on broken tokenizer
+  */
+  CHECK( tok, 1 );
+  CHECK( tok->buff, 1 );
+
+/*  fwprintf( stderr, L"has_next is %ls \n", tok->has_next?L"true":L"false" );*/
+  return   tok->has_next;
 }
 
 int tokenizer::line_number_of_character_at_offset(size_t offset)
@@ -189,13 +189,13 @@ int tokenizer::line_number_of_character_at_offset(size_t offset)
     // we want to return (one plus) the number of newlines at offsets less than the given offset
     // cached_lineno_count is the number of newlines at indexes less than cached_lineno_offset
     const wchar_t *str = orig_buff;
-	if (! str)
-		return 0;
-    
+  if (! str)
+    return 0;
+
     // easy hack to handle 0
     if (offset == 0)
         return 1;
-    
+
     size_t i;
     if (offset > cached_lineno_offset)
     {
@@ -217,7 +217,7 @@ int tokenizer::line_number_of_character_at_offset(size_t offset)
         }
         cached_lineno_offset = offset;
     }
-	return cached_lineno_count + 1;
+  return cached_lineno_count + 1;
 }
 
 /**
@@ -240,14 +240,14 @@ static bool is_string_char( wchar_t c, bool is_first )
         case L'>':
         case L'&':
             return false;
-            
+
         /* Conditional separator */
         case L'^':
             return ! is_first;
-        
+
         default:
             return true;
-            
+
     }
 }
 
@@ -259,7 +259,7 @@ static bool is_string_char( wchar_t c, bool is_first )
 */
 static int myal( wchar_t c )
 {
-	return (c>=L'a' && c<=L'z') || (c>=L'A'&&c<=L'Z');
+  return (c>=L'a' && c<=L'z') || (c>=L'A'&&c<=L'Z');
 }
 
 /**
@@ -267,199 +267,199 @@ static int myal( wchar_t c )
 */
 static void read_string( tokenizer *tok )
 {
-	const wchar_t *start;
-	long len;
-	int mode=0;
-	int do_loop=1;
-	int paran_count=0;
+  const wchar_t *start;
+  long len;
+  int mode=0;
+  int do_loop=1;
+  int paran_count=0;
 
-	start = tok->buff;
+  start = tok->buff;
     bool is_first = true;
-    
-	while( 1 )
-	{
-
-		if( !myal( *tok->buff ) )
-		{
-//			debug(1, L"%lc", *tok->buff );
-
-			if( *tok->buff == L'\\' )
-			{
-				tok->buff++;
-				if( *tok->buff == L'\0' )
-				{
-					if( (!tok->accept_unfinished) )
-					{
-						TOK_CALL_ERROR( tok, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR );
-						return;
-					}
-					else
-					{
-						do_loop = 0;
-					}
-					
-
-				}
-				else if( *tok->buff == L'\n' && mode == 0)
-				{
-					tok->buff--;
-					do_loop = 0;
-					break;
-				}
-				
-				tok->buff++;
-				continue;
-			}
-			
-			
-			/*
-			  The modes are as follows:
-			  
-			  0: regular text
-			  1: inside of subshell
-			  2: inside of array brackets
-			  3: inside of array brackets and subshell, like in '$foo[(ech'
-			*/
-			switch( mode )
-			{
-				case 0:
-				{
-					switch( *tok->buff )
-					{
-						case L'(':
-						{
-							paran_count=1;
-							mode = 1;
-							break;
-						}
-						
-						case L'[':
-						{
-							if( tok->buff != start )
-								mode=2;
-							break;
-						}
-						
-						case L'\'':
-						case L'"':
-						{
-							
-							const wchar_t *end = quote_end( tok->buff );
-							tok->last_quote = *tok->buff;
-							if( end )
-							{
-								tok->buff=(wchar_t *)end;
-							}
-							else
-							{
-								tok->buff += wcslen( tok->buff );
-								
-								if( (!tok->accept_unfinished) )
-								{
-									TOK_CALL_ERROR( tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR );
-									return;
-								}
-								do_loop = 0;
-
-							}
-							break;
-						}
-
-						default:
-						{
-							if( !is_string_char(*(tok->buff), is_first) )
-							{
-								do_loop=0;
-							}
-						}
-					}
-					break;
-				}
-
-				case 3:
-				case 1:
-					switch( *tok->buff )
-					{
-						case L'\'':
-						case L'\"':
-						{
-							const wchar_t *end = quote_end( tok->buff );
-							if( end )
-							{
-								tok->buff=(wchar_t *)end;
-							}
-							else
-							{
-								tok->buff += wcslen( tok->buff );
-								if( (!tok->accept_unfinished) )
-								{
-									TOK_CALL_ERROR( tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR );
-									return;
-								}
-								do_loop = 0;
-							}
-							
-							break;
-						}
-
-						case L'(':
-							paran_count++;
-							break;
-						case L')':
-							paran_count--;
-							if( paran_count == 0 )
-							{
-								mode--;
-							}
-							break;
-						case L'\0':
-							do_loop = 0;
-							break;
-					}
-					break;
-				case 2:
-					switch( *tok->buff )
-					{
-						case L'(':
-							paran_count=1;
-							mode = 3;
-							break;
-
-						case L']':
-							mode=0;
-							break;
-
-						case L'\0':
-							do_loop = 0;
-							break;
-					}
-					break;
-			}
-		}
-
-
-		if( !do_loop )
-			break;
-
-		tok->buff++;
+
+  while( 1 )
+  {
+
+    if( !myal( *tok->buff ) )
+    {
+//      debug(1, L"%lc", *tok->buff );
+
+      if( *tok->buff == L'\\' )
+      {
+        tok->buff++;
+        if( *tok->buff == L'\0' )
+        {
+          if( (!tok->accept_unfinished) )
+          {
+            TOK_CALL_ERROR( tok, TOK_UNTERMINATED_ESCAPE, QUOTE_ERROR );
+            return;
+          }
+          else
+          {
+            do_loop = 0;
+          }
+
+
+        }
+        else if( *tok->buff == L'\n' && mode == 0)
+        {
+          tok->buff--;
+          do_loop = 0;
+          break;
+        }
+
+        tok->buff++;
+        continue;
+      }
+
+
+      /*
+        The modes are as follows:
+
+        0: regular text
+        1: inside of subshell
+        2: inside of array brackets
+        3: inside of array brackets and subshell, like in '$foo[(ech'
+      */
+      switch( mode )
+      {
+        case 0:
+        {
+          switch( *tok->buff )
+          {
+            case L'(':
+            {
+              paran_count=1;
+              mode = 1;
+              break;
+            }
+
+            case L'[':
+            {
+              if( tok->buff != start )
+                mode=2;
+              break;
+            }
+
+            case L'\'':
+            case L'"':
+            {
+
+              const wchar_t *end = quote_end( tok->buff );
+              tok->last_quote = *tok->buff;
+              if( end )
+              {
+                tok->buff=(wchar_t *)end;
+              }
+              else
+              {
+                tok->buff += wcslen( tok->buff );
+
+                if( (!tok->accept_unfinished) )
+                {
+                  TOK_CALL_ERROR( tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR );
+                  return;
+                }
+                do_loop = 0;
+
+              }
+              break;
+            }
+
+            default:
+            {
+              if( !is_string_char(*(tok->buff), is_first) )
+              {
+                do_loop=0;
+              }
+            }
+          }
+          break;
+        }
+
+        case 3:
+        case 1:
+          switch( *tok->buff )
+          {
+            case L'\'':
+            case L'\"':
+            {
+              const wchar_t *end = quote_end( tok->buff );
+              if( end )
+              {
+                tok->buff=(wchar_t *)end;
+              }
+              else
+              {
+                tok->buff += wcslen( tok->buff );
+                if( (!tok->accept_unfinished) )
+                {
+                  TOK_CALL_ERROR( tok, TOK_UNTERMINATED_QUOTE, QUOTE_ERROR );
+                  return;
+                }
+                do_loop = 0;
+              }
+
+              break;
+            }
+
+            case L'(':
+              paran_count++;
+              break;
+            case L')':
+              paran_count--;
+              if( paran_count == 0 )
+              {
+                mode--;
+              }
+              break;
+            case L'\0':
+              do_loop = 0;
+              break;
+          }
+          break;
+        case 2:
+          switch( *tok->buff )
+          {
+            case L'(':
+              paran_count=1;
+              mode = 3;
+              break;
+
+            case L']':
+              mode=0;
+              break;
+
+            case L'\0':
+              do_loop = 0;
+              break;
+          }
+          break;
+      }
+    }
+
+
+    if( !do_loop )
+      break;
+
+    tok->buff++;
         is_first = false;
-	}
+  }
 
-	if( (!tok->accept_unfinished) && (mode!=0) )
-	{
-		TOK_CALL_ERROR( tok, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR );
-		return;
-	}
+  if( (!tok->accept_unfinished) && (mode!=0) )
+  {
+    TOK_CALL_ERROR( tok, TOK_UNTERMINATED_SUBSHELL, PARAN_ERROR );
+    return;
+  }
 
 
-	len = tok->buff - start;
+  len = tok->buff - start;
 
-	if( !check_size( tok, len ))
-		return;
+  if( !check_size( tok, len ))
+    return;
 
-	memcpy( tok->last, start, sizeof(wchar_t)*len );
-	tok->last[len] = L'\0';
-	tok->last_type = TOK_STRING;
+  memcpy( tok->last, start, sizeof(wchar_t)*len );
+  tok->last[len] = L'\0';
+  tok->last_type = TOK_STRING;
 }
 
 /**
@@ -467,19 +467,19 @@ static void read_string( tokenizer *tok )
 */
 static void read_comment( tokenizer *tok )
 {
-	const wchar_t *start;
+  const wchar_t *start;
 
-	start = tok->buff;
-	while( *(tok->buff)!= L'\n' && *(tok->buff)!= L'\0' )
-		tok->buff++;
+  start = tok->buff;
+  while( *(tok->buff)!= L'\n' && *(tok->buff)!= L'\0' )
+    tok->buff++;
 
-	size_t len = tok->buff - start;
-	if( !check_size( tok, len ))
-		return;
+  size_t len = tok->buff - start;
+  if( !check_size( tok, len ))
+    return;
 
-	memcpy( tok->last, start, sizeof(wchar_t)*len );
-	tok->last[len] = L'\0';
-	tok->last_type = TOK_COMMENT;
+  memcpy( tok->last, start, sizeof(wchar_t)*len );
+  tok->last[len] = L'\0';
+  tok->last_type = TOK_COMMENT;
 }
 
 /**
@@ -487,74 +487,74 @@ static void read_comment( tokenizer *tok )
 */
 static void read_redirect( tokenizer *tok, int fd )
 {
-	int mode = -1;
-
-	if( (*tok->buff == L'>') ||
-		(*tok->buff == L'^') )
-	{
-		tok->buff++;
-		if( *tok->buff == *(tok->buff-1) )
-		{
-			tok->buff++;
-			mode = 1;
-		}
-		else
-		{
-			mode = 0;
-		}
-
-		if( *tok->buff == L'|' )
-		{
-			if( fd == 0 )
-			{
-				TOK_CALL_ERROR( tok, TOK_OTHER, PIPE_ERROR );
-				return;
-			}
-			check_size( tok, FD_STR_MAX_LEN );
-			tok->buff++;
-			swprintf( tok->last, FD_STR_MAX_LEN, L"%d", fd );
-			tok->last_type = TOK_PIPE;
-			return;
-		}
-	}
-	else if( *tok->buff == L'<' )
-	{
-		tok->buff++;
-		mode = 2;
-	}
-	else
-	{
-		TOK_CALL_ERROR( tok, TOK_OTHER, REDIRECT_ERROR);
-	}
-
-	if( !check_size( tok, 2 ))
-	{
-		return;
-	}
-
-	swprintf( tok->last, tok->last_len, L"%d", fd );
-
-	if( *tok->buff == L'&' )
-	{
-		tok->buff++;
-		tok->last_type = TOK_REDIRECT_FD;
-	}
-	else if( *tok->buff == L'?' )
-	{
-		tok->buff++;
-		tok->last_type = TOK_REDIRECT_NOCLOB;
-	}
-	else
-	{
-		tok->last_type = TOK_REDIRECT_OUT + mode;
-	}
+  int mode = -1;
+
+  if( (*tok->buff == L'>') ||
+    (*tok->buff == L'^') )
+  {
+    tok->buff++;
+    if( *tok->buff == *(tok->buff-1) )
+    {
+      tok->buff++;
+      mode = 1;
+    }
+    else
+    {
+      mode = 0;
+    }
+
+    if( *tok->buff == L'|' )
+    {
+      if( fd == 0 )
+      {
+        TOK_CALL_ERROR( tok, TOK_OTHER, PIPE_ERROR );
+        return;
+      }
+      check_size( tok, FD_STR_MAX_LEN );
+      tok->buff++;
+      swprintf( tok->last, FD_STR_MAX_LEN, L"%d", fd );
+      tok->last_type = TOK_PIPE;
+      return;
+    }
+  }
+  else if( *tok->buff == L'<' )
+  {
+    tok->buff++;
+    mode = 2;
+  }
+  else
+  {
+    TOK_CALL_ERROR( tok, TOK_OTHER, REDIRECT_ERROR);
+  }
+
+  if( !check_size( tok, 2 ))
+  {
+    return;
+  }
+
+  swprintf( tok->last, tok->last_len, L"%d", fd );
+
+  if( *tok->buff == L'&' )
+  {
+    tok->buff++;
+    tok->last_type = TOK_REDIRECT_FD;
+  }
+  else if( *tok->buff == L'?' )
+  {
+    tok->buff++;
+    tok->last_type = TOK_REDIRECT_NOCLOB;
+  }
+  else
+  {
+    tok->last_type = TOK_REDIRECT_OUT + mode;
+  }
 }
 
 wchar_t tok_last_quote( tokenizer *tok )
 {
-	CHECK( tok, 0 );
-	
-	return tok->last_quote;
+  CHECK( tok, 0 );
+
+  return tok->last_quote;
 }
 
 /**
@@ -563,189 +563,189 @@ wchar_t tok_last_quote( tokenizer *tok )
 */
 static int my_iswspace( wchar_t c )
 {
-	if( c == L'\n' )
-		return 0;
-	else
-		return iswspace( c );
+  if( c == L'\n' )
+    return 0;
+  else
+    return iswspace( c );
 }
 
 
 const wchar_t *tok_get_desc( int type )
 {
-	if( type < 0 || (size_t)type >= sizeof( tok_desc ) )
-	{
-		return _(L"Invalid token type");
-	}
-	return _(tok_desc[type]);
+  if( type < 0 || (size_t)type >= sizeof( tok_desc ) )
+  {
+    return _(L"Invalid token type");
+  }
+  return _(tok_desc[type]);
 }
 
 
 void tok_next( tokenizer *tok )
 {
 
-	CHECK( tok, );
-	CHECK( tok->buff, );
-	
-	if( tok_last_type( tok ) == TOK_ERROR )
-	{
-		tok->has_next=false;
-		return;
-	}
-
-	if( !tok->has_next )
-	{
-/*		wprintf( L"EOL\n" );*/
-		tok->last_type = TOK_END;
-		return;
-	}
-
-	while( 1 )
-	{
-		if( my_iswspace(*(tok->buff) ) )
-		{
-			tok->buff++;
-		}
-		else
-		{
-			if(( *(tok->buff) == L'\\') &&( *(tok->buff+1) == L'\n') )
-			{
-				tok->last_pos = tok->buff - tok->orig_buff;
-				tok->buff+=2;
-				tok->last_type = TOK_END;
-				return;
-			}
-			break;
-		}
-	}
-	
-
-	if( *tok->buff == L'#')
-	{
-		if( tok->show_comments )
-		{
-			tok->last_pos = tok->buff - tok->orig_buff;
-			read_comment( tok );
-			return;
-		}
-		else
-		{
-			while( *(tok->buff)!= L'\n' && *(tok->buff)!= L'\0' )
-				tok->buff++;
-		}
-
-		while( my_iswspace(*(tok->buff) ) )
-			tok->buff++;
-	}
-
-	tok->last_pos = tok->buff - tok->orig_buff;
-
-	switch( *tok->buff )
-	{
-
-		case L'\0':
-			tok->last_type = TOK_END;
-			/*fwprintf( stderr, L"End of string\n" );*/
-			tok->has_next = false;
-			break;
-		case 13:
-		case L'\n':
-		case L';':
-			tok->last_type = TOK_END;
-			tok->buff++;
-			break;
-		case L'&':
-			tok->last_type = TOK_BACKGROUND;
-			tok->buff++;
-			break;
-
-		case L'|':
-			check_size( tok, 2 );
-
-			tok->last[0]=L'1';
-			tok->last[1]=L'\0';
-			tok->last_type = TOK_PIPE;
-			tok->buff++;
-			break;
-
-		case L'>':
-			read_redirect( tok, 1 );
-			return;
-		case L'<':
-			read_redirect( tok, 0 );
-			return;
-		case L'^':
-			read_redirect( tok, 2 );
-			return;
-
-		default:
-		{
-			if( iswdigit( *tok->buff ) )
-			{
-				const wchar_t *orig = tok->buff;
-				int fd = 0;
-				while( iswdigit( *tok->buff ) )
-					fd = (fd*10) + (*(tok->buff++) - L'0');
-
-				switch( *(tok->buff))
-				{
-					case L'^':
-					case L'>':
-					case L'<':
-						read_redirect( tok, fd );
-						return;
-				}
-				tok->buff = orig;
-			}
-			read_string( tok );
-		}
-
-	}
+  CHECK( tok, );
+  CHECK( tok->buff, );
+
+  if( tok_last_type( tok ) == TOK_ERROR )
+  {
+    tok->has_next=false;
+    return;
+  }
+
+  if( !tok->has_next )
+  {
+/*    wprintf( L"EOL\n" );*/
+    tok->last_type = TOK_END;
+    return;
+  }
+
+  while( 1 )
+  {
+    if( my_iswspace(*(tok->buff) ) )
+    {
+      tok->buff++;
+    }
+    else
+    {
+      if(( *(tok->buff) == L'\\') &&( *(tok->buff+1) == L'\n') )
+      {
+        tok->last_pos = tok->buff - tok->orig_buff;
+        tok->buff+=2;
+        tok->last_type = TOK_END;
+        return;
+      }
+      break;
+    }
+  }
+
+
+  if( *tok->buff == L'#')
+  {
+    if( tok->show_comments )
+    {
+      tok->last_pos = tok->buff - tok->orig_buff;
+      read_comment( tok );
+      return;
+    }
+    else
+    {
+      while( *(tok->buff)!= L'\n' && *(tok->buff)!= L'\0' )
+        tok->buff++;
+    }
+
+    while( my_iswspace(*(tok->buff) ) )
+      tok->buff++;
+  }
+
+  tok->last_pos = tok->buff - tok->orig_buff;
+
+  switch( *tok->buff )
+  {
+
+    case L'\0':
+      tok->last_type = TOK_END;
+      /*fwprintf( stderr, L"End of string\n" );*/
+      tok->has_next = false;
+      break;
+    case 13:
+    case L'\n':
+    case L';':
+      tok->last_type = TOK_END;
+      tok->buff++;
+      break;
+    case L'&':
+      tok->last_type = TOK_BACKGROUND;
+      tok->buff++;
+      break;
+
+    case L'|':
+      check_size( tok, 2 );
+
+      tok->last[0]=L'1';
+      tok->last[1]=L'\0';
+      tok->last_type = TOK_PIPE;
+      tok->buff++;
+      break;
+
+    case L'>':
+      read_redirect( tok, 1 );
+      return;
+    case L'<':
+      read_redirect( tok, 0 );
+      return;
+    case L'^':
+      read_redirect( tok, 2 );
+      return;
+
+    default:
+    {
+      if( iswdigit( *tok->buff ) )
+      {
+        const wchar_t *orig = tok->buff;
+        int fd = 0;
+        while( iswdigit( *tok->buff ) )
+          fd = (fd*10) + (*(tok->buff++) - L'0');
+
+        switch( *(tok->buff))
+        {
+          case L'^':
+          case L'>':
+          case L'<':
+            read_redirect( tok, fd );
+            return;
+        }
+        tok->buff = orig;
+      }
+      read_string( tok );
+    }
+
+  }
 
 }
 
 const wchar_t *tok_string( tokenizer *tok )
 {
-	return tok?tok->orig_buff:0;
+  return tok?tok->orig_buff:0;
 }
 
 wchar_t *tok_first( const wchar_t *str )
 {
-	tokenizer t;
-	wchar_t *res=0;
-
-	CHECK( str, 0 );
-	
-	tok_init( &t, str, TOK_SQUASH_ERRORS);
-
-	switch( tok_last_type( &t ) )
-	{
-		case TOK_STRING:
-//				fwprintf( stderr, L"Got token %ls\n", tok_last( &t ));
-			res = wcsdup(tok_last( &t ));
-			break;
-		default:
-			break;
-	}
-
-	tok_destroy( &t );
-	return res;
+  tokenizer t;
+  wchar_t *res=0;
+
+  CHECK( str, 0 );
+
+  tok_init( &t, str, TOK_SQUASH_ERRORS);
+
+  switch( tok_last_type( &t ) )
+  {
+    case TOK_STRING:
+//        fwprintf( stderr, L"Got token %ls\n", tok_last( &t ));
+      res = wcsdup(tok_last( &t ));
+      break;
+    default:
+      break;
+  }
+
+  tok_destroy( &t );
+  return res;
 }
 
 int tok_get_pos( tokenizer *tok )
 {
-	CHECK( tok, 0 );
-	
-	return (int)tok->last_pos;
+  CHECK( tok, 0 );
+
+  return (int)tok->last_pos;
 }
 
 
 void tok_set_pos( tokenizer *tok, int pos )
 {
-	CHECK( tok, );
-	
-	tok->buff = tok->orig_buff + pos;
-	tok->has_next = true;
-	tok_next( tok );
+  CHECK( tok, );
+
+  tok->buff = tok->orig_buff + pos;
+  tok->has_next = true;
+  tok_next( tok );
 }
 
 
@@ -756,38 +756,38 @@ void tok_set_pos( tokenizer *tok, int pos )
 */
 int main( int argc, char **argv )
 {
-	tokenizer tok;
-	int i;
-	for ( i=1; i<argc; i++ )
-	{
-		wprintf( L"Tokenizing string %s\n", argv[i] );
-		for( tok_init( &tok, str2wcs(argv[i]), 0 ); tok_has_next( &tok ); tok_next( &tok ) )
-		{
-			switch( tok_last_type( &tok ) )
- 			{
-				case TOK_INVALID:
-					wprintf( L"Type: INVALID\n" );
-					break;
-				case TOK_STRING:
-					wprintf( L"Type: STRING\t Value: %ls\n", tok_last( &tok ) );
-					break;
-				case TOK_PIPE:
-					wprintf( L"Type: PIPE\n" );
-					break;
-				case TOK_END:
-					wprintf( L"Type: END\n" );
-					break;
-				case TOK_ERROR:
-					wprintf( L"Type: ERROR\n" );
-					break;
-				default:
-					wprintf( L"Type: Unknown\n"  );
-					break;
-			}
-		}
-		tok_destroy( &tok );
-
-	}
+  tokenizer tok;
+  int i;
+  for ( i=1; i<argc; i++ )
+  {
+    wprintf( L"Tokenizing string %s\n", argv[i] );
+    for( tok_init( &tok, str2wcs(argv[i]), 0 ); tok_has_next( &tok ); tok_next( &tok ) )
+    {
+      switch( tok_last_type( &tok ) )
+       {
+        case TOK_INVALID:
+          wprintf( L"Type: INVALID\n" );
+          break;
+        case TOK_STRING:
+          wprintf( L"Type: STRING\t Value: %ls\n", tok_last( &tok ) );
+          break;
+        case TOK_PIPE:
+          wprintf( L"Type: PIPE\n" );
+          break;
+        case TOK_END:
+          wprintf( L"Type: END\n" );
+          break;
+        case TOK_ERROR:
+          wprintf( L"Type: ERROR\n" );
+          break;
+        default:
+          wprintf( L"Type: Unknown\n"  );
+          break;
+      }
+    }
+    tok_destroy( &tok );
+
+  }
 }
 
 #endif