Remove tokenizer_test target and codeT

1 files changed, 0 insertions, 41 deletions

diff --git a/tokenizer.cpp b/tokenizer.cpp
index a727743a..db56748b 100644
--- a/tokenizer.cpp
+++ b/tokenizer.cpp
@@ -747,44 +747,3 @@ bool move_word_state_machine_t::consume_char(wchar_t c)
     return consumed;
 }
 
-
-#ifdef TOKENIZER_TEST
-
-/**
-   This main function is used for compiling the tokenizer_test command, used for testing the tokenizer.
-*/
-int main(int argc, char **argv)
-{
-    tokenizer tok;
-    int i;
-    for (i=1; i<argc; i++)
-    {
-        wprintf(L"Tokenizing string %s\n", argv[i]);
-        for (tok_init(&tok, str2wcs(argv[i]), 0); tok_has_next(&tok); tok_next(&tok))
-        {
-            switch (tok_last_type(&tok))
-            {
-                case TOK_INVALID:
-                    wprintf(L"Type: INVALID\n");
-                    break;
-                case TOK_STRING:
-                    wprintf(L"Type: STRING\t Value: %ls\n", tok_last(&tok));
-                    break;
-                case TOK_PIPE:
-                    wprintf(L"Type: PIPE\n");
-                    break;
-                case TOK_END:
-                    wprintf(L"Type: END\n");
-                    break;
-                case TOK_ERROR:
-                    wprintf(L"Type: ERROR\n");
-                    break;
-                default:
-                    wprintf(L"Type: Unknown\n");
-                    break;
-            }
-        }
-    }
-}
-
-#endif