From 54f683fc6b41bd5575a8b3eb7307b69a1b725df6 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sun, 26 May 2013 12:12:16 -0700 Subject: Initial work on parser bringup (squash) --- fish.xcodeproj/project.pbxproj | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index b48ea577..23b06b5e 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -72,6 +72,7 @@ D07D266D15E33B86009E43F6 /* functions in Copy Files */ = {isa = PBXBuildFile; fileRef = D025C02815D1FEA100B9DB63 /* functions */; }; D07D266E15E33B86009E43F6 /* tools in Copy Files */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; }; D07D267215E34171009E43F6 /* config.fish in Copy Files */ = {isa = PBXBuildFile; fileRef = D0CBD580159EE48F0024809C /* config.fish */; }; + D07FEA311751E6AF003066C3 /* expression.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D07FEA2F1751E6AF003066C3 /* expression.cpp */; }; D0879AC816BF9AAB00E98E56 /* fish_term_icon.icns in Resources */ = {isa = PBXBuildFile; fileRef = D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */; }; D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; @@ -336,6 +337,8 @@ D03EE83814DF88B200FC7150 /* lru.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lru.h; sourceTree = ""; }; D07B247215BCC15700D4ADB4 /* add-shell */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = "add-shell"; path = "build_tools/osx_package_scripts/add-shell"; sourceTree = ""; }; D07B247515BCC4BE00D4ADB4 /* install.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = install.sh; path = osx/install.sh; sourceTree = ""; }; + D07FEA2F1751E6AF003066C3 /* expression.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = expression.cpp; sourceTree = ""; }; + D07FEA301751E6AF003066C3 /* expression.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = expression.h; sourceTree = ""; }; D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; name = fish_term_icon.icns; path = osx/fish_term_icon.icns; sourceTree = ""; }; D09B1C1914FC7B5B00F91077 /* postfork.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = postfork.cpp; sourceTree = ""; }; D09B1C1A14FC7B5B00F91077 /* postfork.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = postfork.h; sourceTree = ""; }; @@ -582,6 +585,8 @@ D0A0853C13B3ACEE0099B651 /* exec.cpp */, D0A0850C13B3ACEE0099B651 /* expand.h */, D0A0853D13B3ACEE0099B651 /* expand.cpp */, + D07FEA301751E6AF003066C3 /* expression.h */, + D07FEA2F1751E6AF003066C3 /* expression.cpp */, D0A0850D13B3ACEE0099B651 /* fallback.h */, D0A0853E13B3ACEE0099B651 /* fallback.cpp */, D0A0850E13B3ACEE0099B651 /* function.h */, @@ -1102,6 +1107,7 @@ D0D02A7A15983916008E62BD /* env_universal.cpp in Sources */, D0D02A7B15983928008E62BD /* env_universal_common.cpp in Sources */, D0D02A89159839DF008E62BD /* fish.cpp in Sources */, + D07FEA311751E6AF003066C3 /* expression.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1189,7 +1195,7 @@ "PREFIX=L\\\"/usr/local\\\"", "DATADIR=L\\\"/usr/local/share\\\"", "SYSCONFDIR=L\\\"/usr/local/etc\\\"", - "BINDIR=L\\\"/usr/local/bin\\\"", + "BINDIR=L\\\"/usr/local/bin\\\"", ); GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES; @@ -1339,7 +1345,7 @@ "PREFIX=L\\\"/usr/local\\\"", "DATADIR=L\\\"/usr/local/share\\\"", "SYSCONFDIR=L\\\"/usr/local/etc\\\"", - "BINDIR=L\\\"/usr/local/bin\\\"", + "BINDIR=L\\\"/usr/local/bin\\\"", ); GCC_SYMBOLS_PRIVATE_EXTERN = NO; GCC_WARN_64_TO_32_BIT_CONVERSION = YES; @@ -1365,7 +1371,7 @@ "PREFIX=L\\\"/usr/local\\\"", "DATADIR=L\\\"/usr/local/share\\\"", "SYSCONFDIR=L\\\"/usr/local/etc\\\"", - "BINDIR=L\\\"/usr/local/bin\\\"", + "BINDIR=L\\\"/usr/local/bin\\\"", ); GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES; -- cgit v1.2.3 From 048f08080d3d2fe60bcc6690c8ef124194980c32 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sun, 9 Jun 2013 14:21:24 -0700 Subject: Rename expression.h to parse_tree.h and parse_exec.h --- Makefile.in | 2 +- expression.cpp | 789 ----------------------------------------- expression.h | 81 ----- fish.xcodeproj/project.pbxproj | 16 +- fish_tests.cpp | 3 +- parse_exec.cpp | 1 + parse_exec.h | 11 + parse_tree.cpp | 773 ++++++++++++++++++++++++++++++++++++++++ parse_tree.h | 69 ++++ parser.cpp | 9 +- parser.h | 4 +- 11 files changed, 872 insertions(+), 886 deletions(-) delete mode 100644 expression.cpp delete mode 100644 expression.h create mode 100644 parse_exec.cpp create mode 100644 parse_exec.h create mode 100644 parse_tree.cpp create mode 100644 parse_tree.h (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/Makefile.in b/Makefile.in index 61a9ef27..56e32e59 100644 --- a/Makefile.in +++ b/Makefile.in @@ -95,7 +95,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o \ env_universal.o env_universal_common.o input_common.o event.o \ signal.o io.o parse_util.o common.o screen.o path.o autoload.o \ parser_keywords.o iothread.o color.o postfork.o \ - builtin_test.o expression.o + builtin_test.o parse_tree.o parse_exec.o FISH_INDENT_OBJS := fish_indent.o print_help.o common.o \ parser_keywords.o wutil.o tokenizer.o diff --git a/expression.cpp b/expression.cpp deleted file mode 100644 index 75245bd1..00000000 --- a/expression.cpp +++ /dev/null @@ -1,789 +0,0 @@ -#include "expression.h" -#include "tokenizer.h" -#include - - -/* Fish grammar: - -# A statement_list is a list of statements, separated by semicolons or newlines - - statement_list = | - statement statement_list - -# A statement is a normal job, or an if / while / and etc. - - statement = boolean_statement | block_statement | decorated_statement - -# A block is a conditional, loop, or begin/end - - block_statement = block_header statement_list END arguments_or_redirections_list - block_header = if_header | for_header | while_header | function_header | begin_header - if_header = IF statement - for_header = FOR var_name IN arguments_or_redirections_list STATEMENT_TERMINATOR - while_header = WHILE statement - begin_header = BEGIN STATEMENT_TERMINATOR - function_header = FUNCTION arguments_or_redirections_list STATEMENT_TERMINATOR - -# A boolean statement is AND or OR or NOT - - boolean_statement = AND statement | OR statement | NOT statement - -# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" - - decorated_statement = COMMAND plain_statement | BUILTIN plain_statement | plain_statement - plain_statement = command arguments_or_redirections_list terminator - - arguments_or_redirections_list = | - argument_or_redirection arguments_or_redirections_list - argument_or_redirection = redirection | - redirection = - - terminator = | - -*/ - -typedef size_t node_offset_t; - -#define PARSE_ASSERT(a) assert(a) - -#define PARSER_DIE() assert(0) - -class parse_command_t; - -enum parse_token_type_t -{ - token_type_invalid, - - // Non-terminal tokens - symbol_statement_list, - symbol_statement, - symbol_block_statement, - symbol_block_header, - symbol_if_header, - symbol_for_header, - symbol_while_header, - symbol_begin_header, - symbol_function_header, - symbol_boolean_statement, - symbol_decorated_statement, - symbol_plain_statement, - symbol_arguments_or_redirections_list, - symbol_argument_or_redirection, - - // Terminal types - parse_token_type_string, - parse_token_type_pipe, - parse_token_type_redirection, - parse_token_background, - parse_token_type_end, - parse_token_type_terminate, - - FIRST_PARSE_TOKEN_TYPE = parse_token_type_string -}; - -static wcstring token_type_description(parse_token_type_t type) -{ - switch (type) - { - case token_type_invalid: return L"invalid"; - - case symbol_statement_list: return L"statement_list"; - case symbol_statement: return L"statement"; - case symbol_block_statement: return L"block_statement"; - case symbol_block_header: return L"block_header"; - case symbol_if_header: return L"if_header"; - case symbol_for_header: return L"for_header"; - case symbol_while_header: return L"while_header"; - case symbol_begin_header: return L"begin_header"; - case symbol_function_header: return L"function_header"; - case symbol_boolean_statement: return L"boolean_statement"; - case symbol_decorated_statement: return L"decorated_statement"; - case symbol_plain_statement: return L"plain_statement"; - case symbol_arguments_or_redirections_list: return L"arguments_or_redirections_list"; - case symbol_argument_or_redirection: return L"argument_or_redirection"; - - case parse_token_type_string: return L"token_string"; - case parse_token_type_pipe: return L"token_pipe"; - case parse_token_type_redirection: return L"token_redirection"; - case parse_token_background: return L"token_background"; - case parse_token_type_end: return L"token_end"; - case parse_token_type_terminate: return L"token_terminate"; - - default: return format_string(L"Unknown token type %ld", static_cast(type)); - } -} - -enum parse_keyword_t -{ - parse_keyword_none, - parse_keyword_if, - parse_keyword_else, - parse_keyword_for, - parse_keyword_in, - parse_keyword_while, - parse_keyword_begin, - parse_keyword_function, - parse_keyword_switch, - parse_keyword_end, - parse_keyword_and, - parse_keyword_or, - parse_keyword_not, - parse_keyword_command, - parse_keyword_builtin -}; - - -struct parse_token_t -{ - enum parse_token_type_t type; // The type of the token as represented by the parser - enum token_type tokenizer_type; // The type of the token as represented by the tokenizer - enum parse_keyword_t keyword; // Any keyword represented by this parser - size_t source_start; - size_t source_length; -}; - -// Convert from tokenizer_t's token type to our token -static parse_token_t parse_token_from_tokenizer_token(enum token_type tokenizer_token_type) -{ - parse_token_t result = {}; - result.tokenizer_type = tokenizer_token_type; - switch (tokenizer_token_type) - { - case TOK_STRING: - result.type = parse_token_type_string; - break; - - case TOK_PIPE: - result.type = parse_token_type_pipe; - break; - - case TOK_END: - result.type = parse_token_type_end; - break; - - case TOK_BACKGROUND: - result.type = parse_token_background; - break; - - default: - fprintf(stderr, "Bad token type %d passed to %s\n", (int)tokenizer_token_type, __FUNCTION__); - assert(0); - break; - } - return result; -} - -/** Base class for nodes of a parse tree */ -class parse_node_t -{ - public: - - /* Start in the source code */ - size_t source_start; - - /* Length of our range in the source code */ - size_t source_length; - - /* Children */ - node_offset_t child_start; - node_offset_t child_count; - - /* Type-dependent data */ - uint32_t tag; - - /* Type of the node */ - enum parse_token_type_t type; - - - /* Description */ - wcstring describe(void) const - { - wcstring result = token_type_description(type); - return result; - } - - /* Constructor */ - explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0) - { - } -}; - -static void dump_tree_recursive(const std::vector &nodes, const wcstring &src, size_t start, size_t indent, wcstring *result, size_t *line) -{ - assert(start < nodes.size()); - const parse_node_t &node = nodes.at(start); - - append_format(*result, L"%2lu ", *line); - result->append(indent, L' ');; - result->append(node.describe()); - if (node.child_count > 0) - { - append_format(*result, L" <%lu children>", node.child_count); - } - if (node.type == parse_token_type_string) - { - result->append(L": \""); - result->append(src, node.source_start, node.source_length); - result->append(L"\""); - } - result->push_back(L'\n'); - ++*line; - for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++) - { - dump_tree_recursive(nodes, src, child_idx, indent + 2, result, line); - } -} - -static wcstring dump_tree(const std::vector &nodes, const wcstring &src) -{ - if (nodes.empty()) - return L"(empty!)"; - - size_t line = 0; - wcstring result; - dump_tree_recursive(nodes, src, 0, 0, &result, &line); - return result; -} - - -struct parse_stack_element_t -{ - enum parse_token_type_t type; - enum parse_keyword_t keyword; - node_offset_t node_idx; - - parse_stack_element_t(parse_token_type_t t) : type(t), keyword(parse_keyword_none), node_idx(-1) - { - } - - parse_stack_element_t(parse_keyword_t k) : type(parse_token_type_string), keyword(k), node_idx(-1) - { - } -}; - -class parse_ll_t -{ - friend class parse_t; - - std::vector symbol_stack; // LL parser stack - std::vector nodes; - bool errored; - - // Constructor - parse_ll_t() : errored(false) - { - // initial node - parse_stack_element_t elem = symbol_statement_list; - elem.node_idx = 0; - symbol_stack.push_back(elem); // goal token - nodes.push_back(parse_node_t(symbol_statement_list)); - } - - bool top_node_match_token(parse_token_t token); - - // implementation of certain parser constructions - void accept_token(parse_token_t token); - void accept_token_statement_list(parse_token_t token); - void accept_token_statement(parse_token_t token); - void accept_token_block_header(parse_token_t token); - void accept_token_boolean_statement(parse_token_t token); - void accept_token_decorated_statement(parse_token_t token); - void accept_token_plain_statement(parse_token_t token); - void accept_token_arguments_or_redirections_list(parse_token_t token); - void accept_token_argument_or_redirection(parse_token_t token); - bool accept_token_string(parse_token_t token); - - void token_unhandled(parse_token_t token, const char *function); - - void parse_error(const wchar_t *expected, parse_token_t token); - - // Get the node corresponding to the top element of the stack - parse_node_t &node_for_top_symbol() - { - PARSE_ASSERT(! symbol_stack.empty()); - const parse_stack_element_t &top_symbol = symbol_stack.back(); - PARSE_ASSERT(top_symbol.node_idx != -1); - PARSE_ASSERT(top_symbol.node_idx < nodes.size()); - return nodes.at(top_symbol.node_idx); - } - - parse_token_type_t stack_top_type() const - { - return symbol_stack.back().type; - } - - void top_node_set_tag(uint32_t tag) - { - this->node_for_top_symbol().tag = tag; - } - - inline void add_child_to_node(size_t parent_node_idx, parse_stack_element_t *tok) - { - PARSE_ASSERT(tok->type != token_type_invalid); - tok->node_idx = nodes.size(); - nodes.push_back(parse_node_t(tok->type)); - nodes.at(parent_node_idx).child_count += 1; - } - - - // Pop from the top of the symbol stack, then push, updating node counts. Note that these are pushed in reverse order, so the first argument will be on the top of the stack. - inline void symbol_stack_pop_push(parse_stack_element_t tok1 = token_type_invalid, parse_stack_element_t tok2 = token_type_invalid, parse_stack_element_t tok3 = token_type_invalid, parse_stack_element_t tok4 = token_type_invalid, parse_stack_element_t tok5 = token_type_invalid) - { - // Get the node for the top symbol and tell it about its children - size_t node_idx = symbol_stack.back().node_idx; - parse_node_t &node = nodes.at(node_idx); - - // Should have no children yet - PARSE_ASSERT(node.child_count == 0); - - // Tell the node where its children start - node.child_start = nodes.size(); - - // Add nodes for the children - // Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack) - if (tok1.type != token_type_invalid) add_child_to_node(node_idx, &tok1); - if (tok2.type != token_type_invalid) add_child_to_node(node_idx, &tok2); - if (tok3.type != token_type_invalid) add_child_to_node(node_idx, &tok3); - if (tok4.type != token_type_invalid) add_child_to_node(node_idx, &tok4); - if (tok5.type != token_type_invalid) add_child_to_node(node_idx, &tok5); - - // The above set the node_idx. Now replace the top of the stack. - symbol_stack.pop_back(); - if (tok5.type != token_type_invalid) symbol_stack.push_back(tok5); - if (tok4.type != token_type_invalid) symbol_stack.push_back(tok4); - if (tok3.type != token_type_invalid) symbol_stack.push_back(tok3); - if (tok2.type != token_type_invalid) symbol_stack.push_back(tok2); - if (tok1.type != token_type_invalid) symbol_stack.push_back(tok1); - } -}; - -void parse_ll_t::token_unhandled(parse_token_t token, const char *function) -{ - fprintf(stderr, "Unhandled token with type %d in function %s\n", (int)token.type, function); - PARSER_DIE(); -} - -void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) -{ - fprintf(stderr, "Expected a %ls, instead got a token of type %d\n", expected, (int)token.type); -} - -void parse_ll_t::accept_token_statement_list(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_statement_list); - switch (token.type) - { - case parse_token_type_string: - case parse_token_type_pipe: - case parse_token_type_redirection: - case parse_token_background: - case parse_token_type_end: - symbol_stack_pop_push(symbol_statement, symbol_statement_list); - break; - - case parse_token_type_terminate: - // no more commands, just transition to empty - symbol_stack_pop_push(); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_statement); - switch (token.type) - { - case parse_token_type_string: - switch (token.keyword) - { - case parse_keyword_and: - case parse_keyword_or: - case parse_keyword_not: - symbol_stack_pop_push(symbol_boolean_statement); - break; - - case parse_keyword_if: - case parse_keyword_else: - case parse_keyword_for: - case parse_keyword_in: - case parse_keyword_while: - case parse_keyword_begin: - case parse_keyword_function: - case parse_keyword_switch: - symbol_stack_pop_push(symbol_block_statement); - assert(0 && "Need assignment"); - break; - - case parse_keyword_end: - // TODO - break; - - case parse_keyword_none: - case parse_keyword_command: - case parse_keyword_builtin: - symbol_stack_pop_push(symbol_decorated_statement); - break; - - } - break; - - case parse_token_type_pipe: - case parse_token_type_redirection: - case parse_token_background: - case parse_token_type_end: - case parse_token_type_terminate: - parse_error(L"command", token); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_block_header(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_block_header); - switch (token.type) - { - case parse_token_type_string: - switch (token.keyword) - { - case parse_keyword_if: - symbol_stack_pop_push(symbol_if_header); - break; - - case parse_keyword_else: - //todo - break; - - case parse_keyword_for: - symbol_stack_pop_push(symbol_for_header); - break; - - - case parse_keyword_while: - symbol_stack_pop_push(symbol_while_header); - break; - - case parse_keyword_begin: - symbol_stack_pop_push(symbol_begin_header); - break; - - case parse_keyword_function: - symbol_stack_pop_push(symbol_function_header); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - - } - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_boolean_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_boolean_statement); - switch (token.type) - { - case parse_token_type_string: - switch (token.keyword) - { - case parse_keyword_and: - case parse_keyword_or: - case parse_keyword_not: - top_node_set_tag(token.keyword); - symbol_stack_pop_push(token.keyword, symbol_statement); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_decorated_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_decorated_statement); - switch (token.type) - { - case parse_token_type_string: - switch (token.keyword) - { - case parse_keyword_command: - top_node_set_tag(parse_keyword_command); - symbol_stack_pop_push(parse_keyword_command, symbol_plain_statement); - break; - - case parse_keyword_builtin: - top_node_set_tag(parse_keyword_builtin); - symbol_stack_pop_push(parse_keyword_builtin, symbol_plain_statement); - break; - - default: - top_node_set_tag(parse_keyword_none); - symbol_stack_pop_push(symbol_plain_statement); - break; - } - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_plain_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_plain_statement); - symbol_stack_pop_push(parse_token_type_string, symbol_arguments_or_redirections_list, parse_token_type_end); -} - -void parse_ll_t::accept_token_arguments_or_redirections_list(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_arguments_or_redirections_list); - switch (token.type) - { - case parse_token_type_string: - case parse_token_type_redirection: - symbol_stack_pop_push(symbol_argument_or_redirection, symbol_arguments_or_redirections_list); - break; - - default: - // Some other token, end of list - symbol_stack_pop_push(); - break; - } -} - -void parse_ll_t::accept_token_argument_or_redirection(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_argument_or_redirection); - switch (token.type) - { - case parse_token_type_string: - symbol_stack_pop_push(parse_token_type_string); - // Got an argument - break; - - case parse_token_type_redirection: - symbol_stack_pop_push(parse_token_type_redirection); - // Got a redirection - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -bool parse_ll_t::accept_token_string(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == parse_token_type_string); - bool result = false; - switch (token.type) - { - case parse_token_type_string: - // Got our string - symbol_stack_pop_push(); - result = true; - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } - return result; -} - -bool parse_ll_t::top_node_match_token(parse_token_t token) -{ - PARSE_ASSERT(! symbol_stack.empty()); - PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE); - bool result = false; - parse_stack_element_t &stack_top = symbol_stack.back(); - if (stack_top.type == token.type) - { - // So far so good. See if we need a particular keyword. - if (stack_top.keyword == parse_keyword_none || stack_top.keyword == token.keyword) - { - // Success. Tell the node that it matched this token - parse_node_t &node = node_for_top_symbol(); - node.source_start = token.source_start; - node.source_length = token.source_length; - - // We consumed this symbol - symbol_stack.pop_back(); - result = true; - } - } - return result; -} - -void parse_ll_t::accept_token(parse_token_t token) -{ - PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE); - PARSE_ASSERT(! symbol_stack.empty()); - bool consumed = false; - while (! consumed && ! this->errored) - { - fprintf(stderr, "Top type %ls\n", token_type_description(stack_top_type()).c_str()); - if (top_node_match_token(token)) - { - consumed = true; - break; - } - - switch (stack_top_type()) - { - /* Symbols */ - case symbol_statement_list: - accept_token_statement_list(token); - break; - - case symbol_statement: - accept_token_statement(token); - break; - - case symbol_block_statement: - symbol_stack_pop_push(symbol_block_header, symbol_statement_list, parse_keyword_end, symbol_arguments_or_redirections_list); - break; - - case symbol_block_header: - accept_token_block_header(token); - break; - - case symbol_if_header: - break; - - case symbol_for_header: - symbol_stack_pop_push(parse_keyword_for, parse_token_type_string, parse_keyword_in, symbol_arguments_or_redirections_list, parse_token_type_end); - break; - - case symbol_while_header: - symbol_stack_pop_push(parse_keyword_while, symbol_statement); - break; - - case symbol_begin_header: - symbol_stack_pop_push(parse_keyword_begin, parse_token_type_end); - break; - - case symbol_function_header: - symbol_stack_pop_push(parse_keyword_function, symbol_arguments_or_redirections_list, parse_token_type_end); - break; - - case symbol_boolean_statement: - accept_token_boolean_statement(token); - break; - - case symbol_decorated_statement: - accept_token_decorated_statement(token); - break; - - case symbol_plain_statement: - accept_token_plain_statement(token); - break; - - case symbol_arguments_or_redirections_list: - accept_token_arguments_or_redirections_list(token); - break; - - case symbol_argument_or_redirection: - accept_token_argument_or_redirection(token); - break; - - /* Tokens */ - case parse_token_type_string: - consumed = accept_token_string(token); - break; - - default: - fprintf(stderr, "Bailing with token type %d\n", (int)token.type); - break; - } - } -} - -parse_t::parse_t() : parser(new parse_ll_t()) -{ -} - -static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt) -{ - parse_keyword_t result = parse_keyword_none; - if (tok == TOK_STRING) - { - - const struct { - const wchar_t *txt; - parse_keyword_t keyword; - } keywords[] = { - {L"if", parse_keyword_if}, - {L"else", parse_keyword_else}, - {L"for", parse_keyword_for}, - {L"in", parse_keyword_in}, - {L"while", parse_keyword_while}, - {L"begin", parse_keyword_begin}, - {L"function", parse_keyword_function}, - {L"switch", parse_keyword_switch}, - {L"end", parse_keyword_end}, - {L"and", parse_keyword_and}, - {L"or", parse_keyword_or}, - {L"not", parse_keyword_not}, - {L"command", parse_keyword_command}, - {L"builtin", parse_keyword_builtin} - }; - - for (size_t i=0; i < sizeof keywords / sizeof *keywords; i++) - { - if (! wcscmp(keywords[i].txt, tok_txt)) - { - result = keywords[i].keyword; - break; - } - } - } - return result; -} - -void parse_t::parse(const wcstring &str) -{ - tokenizer_t tok = tokenizer_t(str.c_str(), 0); - for (; tok_has_next(&tok); tok_next(&tok)) - { - token_type tok_type = static_cast(tok_last_type(&tok)); - const wchar_t *tok_txt = tok_last(&tok); - int tok_start = tok_get_pos(&tok); - - if (tok_type == TOK_ERROR) - { - fprintf(stderr, "Tokenizer error\n"); - break; - } - - parse_token_t token = parse_token_from_tokenizer_token(tok_type); - token.tokenizer_type = tok_type; - token.source_start = (size_t)tok_start; - token.source_length = wcslen(tok_txt); - token.keyword = keyword_for_token(tok_type, tok_txt); - this->parser->accept_token(token); - } - wcstring result = dump_tree(this->parser->nodes, str); - fprintf(stderr, "Tree (%ld nodes):\n%ls", this->parser->nodes.size(), result.c_str()); - fprintf(stderr, "node size %ld", sizeof(parse_node_t)); -} diff --git a/expression.h b/expression.h deleted file mode 100644 index 379314cd..00000000 --- a/expression.h +++ /dev/null @@ -1,81 +0,0 @@ -/**\file expression.h - - Programmatic representation of fish code. - -*/ - -#ifndef FISH_EXPRESSION_H -#define FISH_EXPRESSION_H - -#include - -#include "config.h" -#include "util.h" -#include "common.h" -#include "tokenizer.h" - - -class parse_ll_t; -class parse_sr_t; -class parse_t -{ - parse_ll_t * const parser; - - public: - parse_t(); - void parse(const wcstring &str); -}; - - - -/* Fish grammar: - -# A statement_list is a list of statements, separated by semicolons or newlines - - statement_list = | - statement statement_list - -# A statement is a normal job, or an if / while / and etc. - - statement = boolean_statement | block_statement | decorated_statement - -# A block is a conditional, loop, or begin/end - - block_statement = block_header statement_list END arguments_or_redirections_list - block_header = if_header | for_header | while_header | function_header | begin_header - if_header = IF statement - for_header = FOR var_name IN arguments_or_redirections_list STATEMENT_TERMINATOR - while_header = WHILE statement - begin_header = BEGIN STATEMENT_TERMINATOR - function_header = FUNCTION arguments_or_redirections_list STATEMENT_TERMINATOR - -# A boolean statement is AND or OR or NOT - - boolean_statement = AND statement | OR statement | NOT statement - -# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" - - decorated_statement = COMMAND plain_statement | BUILTIN plain_statement | plain_statement - plain_statement = command arguments_or_redirections_list terminator - - arguments_or_redirections_list = | - argument_or_redirection arguments_or_redirections_list - argument_or_redirection = redirection | - redirection = - - terminator = | - -*/ - - -/* fish Shift-Reduce grammar: - - - IF <- if_statement - FOR <- for_statement - - - -*/ - -#endif diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 23b06b5e..165fb278 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -72,10 +72,10 @@ D07D266D15E33B86009E43F6 /* functions in Copy Files */ = {isa = PBXBuildFile; fileRef = D025C02815D1FEA100B9DB63 /* functions */; }; D07D266E15E33B86009E43F6 /* tools in Copy Files */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; }; D07D267215E34171009E43F6 /* config.fish in Copy Files */ = {isa = PBXBuildFile; fileRef = D0CBD580159EE48F0024809C /* config.fish */; }; - D07FEA311751E6AF003066C3 /* expression.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D07FEA2F1751E6AF003066C3 /* expression.cpp */; }; D0879AC816BF9AAB00E98E56 /* fish_term_icon.icns in Resources */ = {isa = PBXBuildFile; fileRef = D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */; }; D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; + D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; }; D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; }; D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; }; @@ -337,8 +337,6 @@ D03EE83814DF88B200FC7150 /* lru.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lru.h; sourceTree = ""; }; D07B247215BCC15700D4ADB4 /* add-shell */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = "add-shell"; path = "build_tools/osx_package_scripts/add-shell"; sourceTree = ""; }; D07B247515BCC4BE00D4ADB4 /* install.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = install.sh; path = osx/install.sh; sourceTree = ""; }; - D07FEA2F1751E6AF003066C3 /* expression.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = expression.cpp; sourceTree = ""; }; - D07FEA301751E6AF003066C3 /* expression.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = expression.h; sourceTree = ""; }; D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; name = fish_term_icon.icns; path = osx/fish_term_icon.icns; sourceTree = ""; }; D09B1C1914FC7B5B00F91077 /* postfork.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = postfork.cpp; sourceTree = ""; }; D09B1C1A14FC7B5B00F91077 /* postfork.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = postfork.h; sourceTree = ""; }; @@ -444,6 +442,10 @@ D0B6B0FE14E88BA400AD6C10 /* color.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = color.cpp; sourceTree = ""; }; D0B6B0FF14E88BA400AD6C10 /* color.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = color.h; sourceTree = ""; }; D0C4FD9415A7D7EE00212EF1 /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = etc/config.fish; sourceTree = ""; }; + D0C52F331765281F00BFAB82 /* parse_exec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_exec.cpp; sourceTree = ""; }; + D0C52F341765281F00BFAB82 /* parse_exec.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_exec.h; sourceTree = ""; }; + D0C52F351765284C00BFAB82 /* parse_tree.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_tree.cpp; sourceTree = ""; }; + D0C52F361765284C00BFAB82 /* parse_tree.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_tree.h; sourceTree = ""; }; D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = ""; }; D0C6FCCB14CFA4B7004CE8AD /* autoload.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = autoload.h; sourceTree = ""; }; D0C861EA16CC7054003B5A04 /* builtin_set_color.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_set_color.cpp; sourceTree = ""; }; @@ -585,8 +587,10 @@ D0A0853C13B3ACEE0099B651 /* exec.cpp */, D0A0850C13B3ACEE0099B651 /* expand.h */, D0A0853D13B3ACEE0099B651 /* expand.cpp */, - D07FEA301751E6AF003066C3 /* expression.h */, - D07FEA2F1751E6AF003066C3 /* expression.cpp */, + D0C52F361765284C00BFAB82 /* parse_tree.h */, + D0C52F351765284C00BFAB82 /* parse_tree.cpp */, + D0C52F341765281F00BFAB82 /* parse_exec.h */, + D0C52F331765281F00BFAB82 /* parse_exec.cpp */, D0A0850D13B3ACEE0099B651 /* fallback.h */, D0A0853E13B3ACEE0099B651 /* fallback.cpp */, D0A0850E13B3ACEE0099B651 /* function.h */, @@ -1107,7 +1111,7 @@ D0D02A7A15983916008E62BD /* env_universal.cpp in Sources */, D0D02A7B15983928008E62BD /* env_universal_common.cpp in Sources */, D0D02A89159839DF008E62BD /* fish.cpp in Sources */, - D07FEA311751E6AF003066C3 /* expression.cpp in Sources */, + D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/fish_tests.cpp b/fish_tests.cpp index 69368993..35c1e327 100644 --- a/fish_tests.cpp +++ b/fish_tests.cpp @@ -60,7 +60,8 @@ #include "postfork.h" #include "signal.h" #include "highlight.h" -#include "expression.h" +#include "parse_tree.h" +#include "parse_exec.h" /** The number of tests to run diff --git a/parse_exec.cpp b/parse_exec.cpp new file mode 100644 index 00000000..5f4cf2d1 --- /dev/null +++ b/parse_exec.cpp @@ -0,0 +1 @@ +#include "parse_exec.h" diff --git a/parse_exec.h b/parse_exec.h new file mode 100644 index 00000000..50a0ddd6 --- /dev/null +++ b/parse_exec.h @@ -0,0 +1,11 @@ +/**\file parse_exec.h + + Programmatic execution of a parse tree +*/ + +#ifndef FISH_PARSE_TREE_H +#define FISH_PARSE_TREE_H + +#include "parse_tree.h" + +#endif diff --git a/parse_tree.cpp b/parse_tree.cpp new file mode 100644 index 00000000..bdc3cf2b --- /dev/null +++ b/parse_tree.cpp @@ -0,0 +1,773 @@ +#include "expression.h" +#include "tokenizer.h" +#include + +struct parse_node_t; +typedef size_t node_offset_t; +typedef std::vector parse_node_tree_t; + +#define PARSE_ASSERT(a) assert(a) + +#define PARSER_DIE() assert(0) + +class parse_command_t; + +enum parse_token_type_t +{ + token_type_invalid, + + // Non-terminal tokens + symbol_statement_list, + symbol_statement, + symbol_block_statement, + symbol_block_header, + symbol_if_header, + symbol_for_header, + symbol_while_header, + symbol_begin_header, + symbol_function_header, + symbol_boolean_statement, + symbol_decorated_statement, + symbol_plain_statement, + symbol_arguments_or_redirections_list, + symbol_argument_or_redirection, + + // Terminal types + parse_token_type_string, + parse_token_type_pipe, + parse_token_type_redirection, + parse_token_background, + parse_token_type_end, + parse_token_type_terminate, + + FIRST_PARSE_TOKEN_TYPE = parse_token_type_string +}; + +static wcstring token_type_description(parse_token_type_t type) +{ + switch (type) + { + case token_type_invalid: return L"invalid"; + + case symbol_statement_list: return L"statement_list"; + case symbol_statement: return L"statement"; + case symbol_block_statement: return L"block_statement"; + case symbol_block_header: return L"block_header"; + case symbol_if_header: return L"if_header"; + case symbol_for_header: return L"for_header"; + case symbol_while_header: return L"while_header"; + case symbol_begin_header: return L"begin_header"; + case symbol_function_header: return L"function_header"; + case symbol_boolean_statement: return L"boolean_statement"; + case symbol_decorated_statement: return L"decorated_statement"; + case symbol_plain_statement: return L"plain_statement"; + case symbol_arguments_or_redirections_list: return L"arguments_or_redirections_list"; + case symbol_argument_or_redirection: return L"argument_or_redirection"; + + case parse_token_type_string: return L"token_string"; + case parse_token_type_pipe: return L"token_pipe"; + case parse_token_type_redirection: return L"token_redirection"; + case parse_token_background: return L"token_background"; + case parse_token_type_end: return L"token_end"; + case parse_token_type_terminate: return L"token_terminate"; + + default: return format_string(L"Unknown token type %ld", static_cast(type)); + } +} + +enum parse_keyword_t +{ + parse_keyword_none, + parse_keyword_if, + parse_keyword_else, + parse_keyword_for, + parse_keyword_in, + parse_keyword_while, + parse_keyword_begin, + parse_keyword_function, + parse_keyword_switch, + parse_keyword_end, + parse_keyword_and, + parse_keyword_or, + parse_keyword_not, + parse_keyword_command, + parse_keyword_builtin +}; + + +struct parse_token_t +{ + enum parse_token_type_t type; // The type of the token as represented by the parser + enum token_type tokenizer_type; // The type of the token as represented by the tokenizer + enum parse_keyword_t keyword; // Any keyword represented by this parser + size_t source_start; + size_t source_length; +}; + +// Convert from tokenizer_t's token type to our token +static parse_token_t parse_token_from_tokenizer_token(enum token_type tokenizer_token_type) +{ + parse_token_t result = {}; + result.tokenizer_type = tokenizer_token_type; + switch (tokenizer_token_type) + { + case TOK_STRING: + result.type = parse_token_type_string; + break; + + case TOK_PIPE: + result.type = parse_token_type_pipe; + break; + + case TOK_END: + result.type = parse_token_type_end; + break; + + case TOK_BACKGROUND: + result.type = parse_token_background; + break; + + default: + fprintf(stderr, "Bad token type %d passed to %s\n", (int)tokenizer_token_type, __FUNCTION__); + assert(0); + break; + } + return result; +} + +/** Base class for nodes of a parse tree */ +class parse_node_t +{ + public: + + /* Start in the source code */ + size_t source_start; + + /* Length of our range in the source code */ + size_t source_length; + + /* Children */ + node_offset_t child_start; + node_offset_t child_count; + + /* Type-dependent data */ + uint32_t tag; + + /* Type of the node */ + enum parse_token_type_t type; + + + /* Description */ + wcstring describe(void) const + { + wcstring result = token_type_description(type); + return result; + } + + /* Constructor */ + explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0) + { + } +}; + +static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &src, size_t start, size_t indent, wcstring *result, size_t *line) +{ + assert(start < nodes.size()); + const parse_node_t &node = nodes.at(start); + + append_format(*result, L"%2lu ", *line); + result->append(indent, L' ');; + result->append(node.describe()); + if (node.child_count > 0) + { + append_format(*result, L" <%lu children>", node.child_count); + } + if (node.type == parse_token_type_string) + { + result->append(L": \""); + result->append(src, node.source_start, node.source_length); + result->append(L"\""); + } + result->push_back(L'\n'); + ++*line; + for (size_t child_idx = node.child_start; child_idx < node.child_start + node.child_count; child_idx++) + { + dump_tree_recursive(nodes, src, child_idx, indent + 2, result, line); + } +} + +static wcstring dump_tree(const parse_node_tree_t &nodes, const wcstring &src) +{ + if (nodes.empty()) + return L"(empty!)"; + + size_t line = 0; + wcstring result; + dump_tree_recursive(nodes, src, 0, 0, &result, &line); + return result; +} + +struct parse_stack_element_t +{ + enum parse_token_type_t type; + enum parse_keyword_t keyword; + node_offset_t node_idx; + + parse_stack_element_t(parse_token_type_t t) : type(t), keyword(parse_keyword_none), node_idx(-1) + { + } + + parse_stack_element_t(parse_keyword_t k) : type(parse_token_type_string), keyword(k), node_idx(-1) + { + } +}; + +class parse_execution_context_t +{ + wcstring src; + const parse_node_tree_t nodes; + size_t node_idx; + + public: + parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s) : src(s), nodes(n), node_idx(0) + { + } + + wcstring simulate(void); +}; + +wcstring parse_execution_context_t::simulate() +{ + if (nodes.empty()) + return L"(empty!"); + + PARSE_ASSERT(node_idx < nodes.size()); + PARSE_ASSERT(nodes.at(node_idx).type == symbol_statement_list); + +} + +class parse_ll_t +{ + friend class parse_t; + + std::vector symbol_stack; // LL parser stack + parse_node_tree_t nodes; + bool errored; + + // Constructor + parse_ll_t() : errored(false) + { + // initial node + parse_stack_element_t elem = symbol_statement_list; + elem.node_idx = 0; + symbol_stack.push_back(elem); // goal token + nodes.push_back(parse_node_t(symbol_statement_list)); + } + + bool top_node_match_token(parse_token_t token); + + // implementation of certain parser constructions + void accept_token(parse_token_t token); + void accept_token_statement_list(parse_token_t token); + void accept_token_statement(parse_token_t token); + void accept_token_block_header(parse_token_t token); + void accept_token_boolean_statement(parse_token_t token); + void accept_token_decorated_statement(parse_token_t token); + void accept_token_plain_statement(parse_token_t token); + void accept_token_arguments_or_redirections_list(parse_token_t token); + void accept_token_argument_or_redirection(parse_token_t token); + bool accept_token_string(parse_token_t token); + + void token_unhandled(parse_token_t token, const char *function); + + void parse_error(const wchar_t *expected, parse_token_t token); + + // Get the node corresponding to the top element of the stack + parse_node_t &node_for_top_symbol() + { + PARSE_ASSERT(! symbol_stack.empty()); + const parse_stack_element_t &top_symbol = symbol_stack.back(); + PARSE_ASSERT(top_symbol.node_idx != -1); + PARSE_ASSERT(top_symbol.node_idx < nodes.size()); + return nodes.at(top_symbol.node_idx); + } + + parse_token_type_t stack_top_type() const + { + return symbol_stack.back().type; + } + + void top_node_set_tag(uint32_t tag) + { + this->node_for_top_symbol().tag = tag; + } + + inline void add_child_to_node(size_t parent_node_idx, parse_stack_element_t *tok) + { + PARSE_ASSERT(tok->type != token_type_invalid); + tok->node_idx = nodes.size(); + nodes.push_back(parse_node_t(tok->type)); + nodes.at(parent_node_idx).child_count += 1; + } + + + // Pop from the top of the symbol stack, then push, updating node counts. Note that these are pushed in reverse order, so the first argument will be on the top of the stack. + inline void symbol_stack_pop_push(parse_stack_element_t tok1 = token_type_invalid, parse_stack_element_t tok2 = token_type_invalid, parse_stack_element_t tok3 = token_type_invalid, parse_stack_element_t tok4 = token_type_invalid, parse_stack_element_t tok5 = token_type_invalid) + { + // Get the node for the top symbol and tell it about its children + size_t node_idx = symbol_stack.back().node_idx; + parse_node_t &node = nodes.at(node_idx); + + // Should have no children yet + PARSE_ASSERT(node.child_count == 0); + + // Tell the node where its children start + node.child_start = nodes.size(); + + // Add nodes for the children + // Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack) + if (tok1.type != token_type_invalid) add_child_to_node(node_idx, &tok1); + if (tok2.type != token_type_invalid) add_child_to_node(node_idx, &tok2); + if (tok3.type != token_type_invalid) add_child_to_node(node_idx, &tok3); + if (tok4.type != token_type_invalid) add_child_to_node(node_idx, &tok4); + if (tok5.type != token_type_invalid) add_child_to_node(node_idx, &tok5); + + // The above set the node_idx. Now replace the top of the stack. + symbol_stack.pop_back(); + if (tok5.type != token_type_invalid) symbol_stack.push_back(tok5); + if (tok4.type != token_type_invalid) symbol_stack.push_back(tok4); + if (tok3.type != token_type_invalid) symbol_stack.push_back(tok3); + if (tok2.type != token_type_invalid) symbol_stack.push_back(tok2); + if (tok1.type != token_type_invalid) symbol_stack.push_back(tok1); + } +}; + +void parse_ll_t::token_unhandled(parse_token_t token, const char *function) +{ + fprintf(stderr, "Unhandled token with type %d in function %s\n", (int)token.type, function); + PARSER_DIE(); +} + +void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) +{ + fprintf(stderr, "Expected a %ls, instead got a token of type %d\n", expected, (int)token.type); +} + +void parse_ll_t::accept_token_statement_list(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_statement_list); + switch (token.type) + { + case parse_token_type_string: + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_background: + case parse_token_type_end: + symbol_stack_pop_push(symbol_statement, symbol_statement_list); + break; + + case parse_token_type_terminate: + // no more commands, just transition to empty + symbol_stack_pop_push(); + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } +} + +void parse_ll_t::accept_token_statement(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_statement); + switch (token.type) + { + case parse_token_type_string: + switch (token.keyword) + { + case parse_keyword_and: + case parse_keyword_or: + case parse_keyword_not: + symbol_stack_pop_push(symbol_boolean_statement); + break; + + case parse_keyword_if: + case parse_keyword_else: + case parse_keyword_for: + case parse_keyword_in: + case parse_keyword_while: + case parse_keyword_begin: + case parse_keyword_function: + case parse_keyword_switch: + symbol_stack_pop_push(symbol_block_statement); + assert(0 && "Need assignment"); + break; + + case parse_keyword_end: + // TODO + break; + + case parse_keyword_none: + case parse_keyword_command: + case parse_keyword_builtin: + symbol_stack_pop_push(symbol_decorated_statement); + break; + + } + break; + + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_background: + case parse_token_type_end: + case parse_token_type_terminate: + parse_error(L"command", token); + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } +} + +void parse_ll_t::accept_token_block_header(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_block_header); + switch (token.type) + { + case parse_token_type_string: + switch (token.keyword) + { + case parse_keyword_if: + symbol_stack_pop_push(symbol_if_header); + break; + + case parse_keyword_else: + //todo + break; + + case parse_keyword_for: + symbol_stack_pop_push(symbol_for_header); + break; + + + case parse_keyword_while: + symbol_stack_pop_push(symbol_while_header); + break; + + case parse_keyword_begin: + symbol_stack_pop_push(symbol_begin_header); + break; + + case parse_keyword_function: + symbol_stack_pop_push(symbol_function_header); + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + + } + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } +} + +void parse_ll_t::accept_token_boolean_statement(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_boolean_statement); + switch (token.type) + { + case parse_token_type_string: + switch (token.keyword) + { + case parse_keyword_and: + case parse_keyword_or: + case parse_keyword_not: + top_node_set_tag(token.keyword); + symbol_stack_pop_push(token.keyword, symbol_statement); + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } + + default: + token_unhandled(token, __FUNCTION__); + break; + } +} + +void parse_ll_t::accept_token_decorated_statement(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_decorated_statement); + switch (token.type) + { + case parse_token_type_string: + switch (token.keyword) + { + case parse_keyword_command: + top_node_set_tag(parse_keyword_command); + symbol_stack_pop_push(parse_keyword_command, symbol_plain_statement); + break; + + case parse_keyword_builtin: + top_node_set_tag(parse_keyword_builtin); + symbol_stack_pop_push(parse_keyword_builtin, symbol_plain_statement); + break; + + default: + top_node_set_tag(parse_keyword_none); + symbol_stack_pop_push(symbol_plain_statement); + break; + } + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } +} + +void parse_ll_t::accept_token_plain_statement(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_plain_statement); + symbol_stack_pop_push(parse_token_type_string, symbol_arguments_or_redirections_list, parse_token_type_end); +} + +void parse_ll_t::accept_token_arguments_or_redirections_list(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_arguments_or_redirections_list); + switch (token.type) + { + case parse_token_type_string: + case parse_token_type_redirection: + symbol_stack_pop_push(symbol_argument_or_redirection, symbol_arguments_or_redirections_list); + break; + + default: + // Some other token, end of list + symbol_stack_pop_push(); + break; + } +} + +void parse_ll_t::accept_token_argument_or_redirection(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == symbol_argument_or_redirection); + switch (token.type) + { + case parse_token_type_string: + symbol_stack_pop_push(parse_token_type_string); + // Got an argument + break; + + case parse_token_type_redirection: + symbol_stack_pop_push(parse_token_type_redirection); + // Got a redirection + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } +} + +bool parse_ll_t::accept_token_string(parse_token_t token) +{ + PARSE_ASSERT(stack_top_type() == parse_token_type_string); + bool result = false; + switch (token.type) + { + case parse_token_type_string: + // Got our string + symbol_stack_pop_push(); + result = true; + break; + + default: + token_unhandled(token, __FUNCTION__); + break; + } + return result; +} + +bool parse_ll_t::top_node_match_token(parse_token_t token) +{ + PARSE_ASSERT(! symbol_stack.empty()); + PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE); + bool result = false; + parse_stack_element_t &stack_top = symbol_stack.back(); + if (stack_top.type == token.type) + { + // So far so good. See if we need a particular keyword. + if (stack_top.keyword == parse_keyword_none || stack_top.keyword == token.keyword) + { + // Success. Tell the node that it matched this token + parse_node_t &node = node_for_top_symbol(); + node.source_start = token.source_start; + node.source_length = token.source_length; + + // We consumed this symbol + symbol_stack.pop_back(); + result = true; + } + } + return result; +} + +void parse_ll_t::accept_token(parse_token_t token) +{ + PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE); + PARSE_ASSERT(! symbol_stack.empty()); + bool consumed = false; + while (! consumed && ! this->errored) + { + if (top_node_match_token(token)) + { + consumed = true; + break; + } + + switch (stack_top_type()) + { + /* Symbols */ + case symbol_statement_list: + accept_token_statement_list(token); + break; + + case symbol_statement: + accept_token_statement(token); + break; + + case symbol_block_statement: + symbol_stack_pop_push(symbol_block_header, symbol_statement_list, parse_keyword_end, symbol_arguments_or_redirections_list); + break; + + case symbol_block_header: + accept_token_block_header(token); + break; + + case symbol_if_header: + break; + + case symbol_for_header: + symbol_stack_pop_push(parse_keyword_for, parse_token_type_string, parse_keyword_in, symbol_arguments_or_redirections_list, parse_token_type_end); + break; + + case symbol_while_header: + symbol_stack_pop_push(parse_keyword_while, symbol_statement); + break; + + case symbol_begin_header: + symbol_stack_pop_push(parse_keyword_begin, parse_token_type_end); + break; + + case symbol_function_header: + symbol_stack_pop_push(parse_keyword_function, symbol_arguments_or_redirections_list, parse_token_type_end); + break; + + case symbol_boolean_statement: + accept_token_boolean_statement(token); + break; + + case symbol_decorated_statement: + accept_token_decorated_statement(token); + break; + + case symbol_plain_statement: + accept_token_plain_statement(token); + break; + + case symbol_arguments_or_redirections_list: + accept_token_arguments_or_redirections_list(token); + break; + + case symbol_argument_or_redirection: + accept_token_argument_or_redirection(token); + break; + + /* Tokens */ + case parse_token_type_string: + consumed = accept_token_string(token); + break; + + default: + fprintf(stderr, "Bailing with token type %d\n", (int)token.type); + break; + } + } +} + +parse_t::parse_t() : parser(new parse_ll_t()) +{ +} + +static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt) +{ + parse_keyword_t result = parse_keyword_none; + if (tok == TOK_STRING) + { + + const struct { + const wchar_t *txt; + parse_keyword_t keyword; + } keywords[] = { + {L"if", parse_keyword_if}, + {L"else", parse_keyword_else}, + {L"for", parse_keyword_for}, + {L"in", parse_keyword_in}, + {L"while", parse_keyword_while}, + {L"begin", parse_keyword_begin}, + {L"function", parse_keyword_function}, + {L"switch", parse_keyword_switch}, + {L"end", parse_keyword_end}, + {L"and", parse_keyword_and}, + {L"or", parse_keyword_or}, + {L"not", parse_keyword_not}, + {L"command", parse_keyword_command}, + {L"builtin", parse_keyword_builtin} + }; + + for (size_t i=0; i < sizeof keywords / sizeof *keywords; i++) + { + if (! wcscmp(keywords[i].txt, tok_txt)) + { + result = keywords[i].keyword; + break; + } + } + } + return result; +} + +void parse_t::parse(const wcstring &str) +{ + tokenizer_t tok = tokenizer_t(str.c_str(), 0); + for (; tok_has_next(&tok); tok_next(&tok)) + { + token_type tok_type = static_cast(tok_last_type(&tok)); + const wchar_t *tok_txt = tok_last(&tok); + int tok_start = tok_get_pos(&tok); + + if (tok_type == TOK_ERROR) + { + fprintf(stderr, "Tokenizer error\n"); + break; + } + + parse_token_t token = parse_token_from_tokenizer_token(tok_type); + token.tokenizer_type = tok_type; + token.source_start = (size_t)tok_start; + token.source_length = wcslen(tok_txt); + token.keyword = keyword_for_token(tok_type, tok_txt); + this->parser->accept_token(token); + } + wcstring result = dump_tree(this->parser->nodes, str); + fprintf(stderr, "Tree (%ld nodes):\n%ls", this->parser->nodes.size(), result.c_str()); + fprintf(stderr, "node size %ld\n", sizeof(parse_node_t)); +} diff --git a/parse_tree.h b/parse_tree.h new file mode 100644 index 00000000..9536479b --- /dev/null +++ b/parse_tree.h @@ -0,0 +1,69 @@ +/**\file parse_tree.h + + Programmatic representation of fish code. +*/ + +#ifndef FISH_PARSE_TREE_H +#define FISH_PARSE_TREE_H + +#include + +#include "config.h" +#include "util.h" +#include "common.h" +#include "tokenizer.h" + + +class parse_ll_t; +class parse_sr_t; +class parse_t +{ + parse_ll_t * const parser; + + public: + parse_t(); + void parse(const wcstring &str); +}; + + + +/* Fish grammar: + +# A statement_list is a list of statements, separated by semicolons or newlines + + statement_list = | + statement statement_list + +# A statement is a normal job, or an if / while / and etc. + + statement = boolean_statement | block_statement | decorated_statement + +# A block is a conditional, loop, or begin/end + + block_statement = block_header statement_list END arguments_or_redirections_list + block_header = if_header | for_header | while_header | function_header | begin_header + if_header = IF statement + for_header = FOR var_name IN arguments_or_redirections_list STATEMENT_TERMINATOR + while_header = WHILE statement + begin_header = BEGIN STATEMENT_TERMINATOR + function_header = FUNCTION arguments_or_redirections_list STATEMENT_TERMINATOR + +# A boolean statement is AND or OR or NOT + + boolean_statement = AND statement | OR statement | NOT statement + +# A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" + + decorated_statement = COMMAND plain_statement | BUILTIN plain_statement | plain_statement + plain_statement = command arguments_or_redirections_list terminator + + arguments_or_redirections_list = | + argument_or_redirection arguments_or_redirections_list + argument_or_redirection = redirection | + redirection = + + terminator = | + +*/ + +#endif diff --git a/parser.cpp b/parser.cpp index 53c68c3c..d2b0f069 100644 --- a/parser.cpp +++ b/parser.cpp @@ -722,7 +722,7 @@ void parser_t::print_errors_stderr() } -int parser_t::eval_args(const wchar_t *line, std::vector &args) +void parser_t::eval_args(const wchar_t *line, std::vector &args) { expand_flags_t eflags = 0; @@ -731,10 +731,9 @@ int parser_t::eval_args(const wchar_t *line, std::vector &args) if (this->parser_type != PARSER_TYPE_GENERAL) eflags |= EXPAND_SKIP_CMDSUBST; - int do_loop=1; + bool do_loop=1; - CHECK(line, 1); -// CHECK( args, 1 ); + if (! line) return; // PCA we need to suppress calling proc_push_interactive off of the main thread. I'm not sure exactly what it does. if (this->parser_type == PARSER_TYPE_GENERAL) @@ -803,8 +802,6 @@ int parser_t::eval_args(const wchar_t *line, std::vector &args) if (this->parser_type == PARSER_TYPE_GENERAL) proc_pop_interactive(); - - return 1; } void parser_t::stack_trace(block_t *b, wcstring &buff) diff --git a/parser.h b/parser.h index c2ad6c9b..0c90641b 100644 --- a/parser.h +++ b/parser.h @@ -396,7 +396,7 @@ public: /** Evaluate line as a list of parameters, i.e. tokenize it and perform parameter expansion and cmdsubst execution on the tokens. - The output is inserted into output, and should be freed by the caller. + The output is inserted into output. \param line Line to evaluate \param output List to insert output to @@ -405,7 +405,7 @@ public: \param line Line to evaluate \param output List to insert output to */ - int eval_args(const wchar_t *line, std::vector &output); + void eval_args(const wchar_t *line, std::vector &output); /** Sets the current evaluation error. This function should only be used by libraries that are called by -- cgit v1.2.3 From e2a506e54a297dcbd13cdb881936e50bbefceb28 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 11 Jun 2013 09:37:51 -0700 Subject: More work on new parser --- fish.xcodeproj/project.pbxproj | 2 + parse_exec.cpp | 147 +++++++++++++++++++++++++++++++++++++++++ parse_exec.h | 15 ++++- parse_tree.cpp | 121 ++------------------------------- parse_tree.h | 90 +++++++++++++++++++++++++ 5 files changed, 257 insertions(+), 118 deletions(-) (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 165fb278..6f6c83ba 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -76,6 +76,7 @@ D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; + D0C52F381765720600BFAB82 /* parse_exec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F331765281F00BFAB82 /* parse_exec.cpp */; }; D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; }; D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; }; D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; }; @@ -1112,6 +1113,7 @@ D0D02A7B15983928008E62BD /* env_universal_common.cpp in Sources */, D0D02A89159839DF008E62BD /* fish.cpp in Sources */, D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */, + D0C52F381765720600BFAB82 /* parse_exec.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/parse_exec.cpp b/parse_exec.cpp index 5f4cf2d1..30503857 100644 --- a/parse_exec.cpp +++ b/parse_exec.cpp @@ -1 +1,148 @@ #include "parse_exec.h" +#include + + +struct exec_node_t +{ + node_offset_t parse_node_idx; + + exec_node_t(size_t pni) : parse_node_idx(pni) + { + } + + virtual ~exec_node_t(); +}; + +exec_node_t::~exec_node_t() +{ +} + +struct exec_redirection_t : public exec_node_t +{ + +}; + +struct exec_argument_t : public exec_node_t +{ + +}; + +struct exec_statement_t +{ + enum + { + decoration_plain, + decoration_command, + decoration_builtin + } decoration; + + std::vector arguments; + std::vector redirections; +}; + +class parse_exec_t +{ + parse_node_tree_t parse_tree; + wcstring src; + std::vector exec_nodes; + + parse_exec_t(const parse_node_tree_t &tree, const wcstring &s) : parse_tree(tree), src(s) + { + } + + void pop_push(uint32_t child_idx) + { + exec_node_t &top = exec_nodes.back(); + const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); + PARSE_ASSERT(child_idx < parse_node.child_count); + node_offset_t child_node_idx = parse_node.child_start + child_idx; + exec_nodes.pop_back(); + exec_nodes.push_back(child_node_idx); + + } + + void simulate(void); + void enter_parse_node(size_t idx); + void run_top_node(void); +}; + +void parse_exec_t::run_top_node() +{ + PARSE_ASSERT(! exec_nodes.empty()); + exec_node_t &top = exec_nodes.back(); + const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); + + switch (parse_node.type) + { + case symbol_statement_list: + PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 2); + if (parse_node.child_count == 0) + { + // Statement list done + exec_nodes.pop_back(); + } + else + { + // First child is a statement, next is the rest of the list + node_offset_t head = parse_node.child_start; + node_offset_t tail = parse_node.child_start + 1; + exec_nodes.pop_back(); + exec_nodes.push_back(tail); + exec_nodes.push_back(head); + } + break; + + case symbol_statement: + PARSE_ASSERT(parse_node.child_count == 1); + pop_push(0); + break; + + case decorated_statement: + PARSE_ASSERT(parse_node.child_count == 1 || parse_node.child_count == 2 ); + pop_push(0); + break; + + default: + fprintf(stderr, "Unhandled token type %ld\n", (long)parse_node.type); + PARSER_DIE(); + break; + + } +} + +void parse_exec_t::enter_parse_node(size_t idx) +{ + PARSE_ASSERT(idx < parse_tree.size()); + exec_node_t exec(idx); + exec_nodes.push_back(exec); +} + +wcstring parse_exec_t::simulate(void) +{ + PARSE_ASSERT(exec_nodes.empty()); + assemble_statement_list(0); + enter_parse_node(0); + run_node(); +} + +wcstring parse_execution_context_t::simulate() +{ + if (parse_tree.empty()) + return L"(empty!)"; + + PARSE_ASSERT(node_idx < nodes.size()); + PARSE_ASSERT(nodes.at(node_idx).type == symbol_statement_list); + + wcstring result; + +} + +parse_execution_context_t::parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s) +{ + ctx = new parse_exec_t(n, s); +} + +wcstring parse_execution_context_t::simulate(void) +{ + return ctx->simulate(); +} diff --git a/parse_exec.h b/parse_exec.h index 50a0ddd6..635ebb83 100644 --- a/parse_exec.h +++ b/parse_exec.h @@ -3,9 +3,20 @@ Programmatic execution of a parse tree */ -#ifndef FISH_PARSE_TREE_H -#define FISH_PARSE_TREE_H +#ifndef FISH_PARSE_EXEC_H +#define FISH_PARSE_EXEC_H #include "parse_tree.h" +class parse_exec_t; +class parse_execution_context_t +{ + parse_exec_t *ctx; + + public: + parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s); + wcstring simulate(void); +}; + + #endif diff --git a/parse_tree.cpp b/parse_tree.cpp index bdc3cf2b..8c38ff0c 100644 --- a/parse_tree.cpp +++ b/parse_tree.cpp @@ -1,48 +1,10 @@ -#include "expression.h" +#include "parse_tree.h" #include "tokenizer.h" #include -struct parse_node_t; -typedef size_t node_offset_t; -typedef std::vector parse_node_tree_t; - -#define PARSE_ASSERT(a) assert(a) - -#define PARSER_DIE() assert(0) class parse_command_t; -enum parse_token_type_t -{ - token_type_invalid, - - // Non-terminal tokens - symbol_statement_list, - symbol_statement, - symbol_block_statement, - symbol_block_header, - symbol_if_header, - symbol_for_header, - symbol_while_header, - symbol_begin_header, - symbol_function_header, - symbol_boolean_statement, - symbol_decorated_statement, - symbol_plain_statement, - symbol_arguments_or_redirections_list, - symbol_argument_or_redirection, - - // Terminal types - parse_token_type_string, - parse_token_type_pipe, - parse_token_type_redirection, - parse_token_background, - parse_token_type_end, - parse_token_type_terminate, - - FIRST_PARSE_TOKEN_TYPE = parse_token_type_string -}; - static wcstring token_type_description(parse_token_type_t type) { switch (type) @@ -75,25 +37,11 @@ static wcstring token_type_description(parse_token_type_t type) } } -enum parse_keyword_t +wcstring parse_node_t::describe(void) const { - parse_keyword_none, - parse_keyword_if, - parse_keyword_else, - parse_keyword_for, - parse_keyword_in, - parse_keyword_while, - parse_keyword_begin, - parse_keyword_function, - parse_keyword_switch, - parse_keyword_end, - parse_keyword_and, - parse_keyword_or, - parse_keyword_not, - parse_keyword_command, - parse_keyword_builtin -}; - + wcstring result = token_type_description(type); + return result; +} struct parse_token_t { @@ -135,41 +83,6 @@ static parse_token_t parse_token_from_tokenizer_token(enum token_type tokenizer_ return result; } -/** Base class for nodes of a parse tree */ -class parse_node_t -{ - public: - - /* Start in the source code */ - size_t source_start; - - /* Length of our range in the source code */ - size_t source_length; - - /* Children */ - node_offset_t child_start; - node_offset_t child_count; - - /* Type-dependent data */ - uint32_t tag; - - /* Type of the node */ - enum parse_token_type_t type; - - - /* Description */ - wcstring describe(void) const - { - wcstring result = token_type_description(type); - return result; - } - - /* Constructor */ - explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0) - { - } -}; - static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring &src, size_t start, size_t indent, wcstring *result, size_t *line) { assert(start < nodes.size()); @@ -222,30 +135,6 @@ struct parse_stack_element_t } }; -class parse_execution_context_t -{ - wcstring src; - const parse_node_tree_t nodes; - size_t node_idx; - - public: - parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s) : src(s), nodes(n), node_idx(0) - { - } - - wcstring simulate(void); -}; - -wcstring parse_execution_context_t::simulate() -{ - if (nodes.empty()) - return L"(empty!"); - - PARSE_ASSERT(node_idx < nodes.size()); - PARSE_ASSERT(nodes.at(node_idx).type == symbol_statement_list); - -} - class parse_ll_t { friend class parse_t; diff --git a/parse_tree.h b/parse_tree.h index 9536479b..892c36cd 100644 --- a/parse_tree.h +++ b/parse_tree.h @@ -12,6 +12,10 @@ #include "util.h" #include "common.h" #include "tokenizer.h" +#include + +#define PARSE_ASSERT(a) assert(a) +#define PARSER_DIE() assert(0) class parse_ll_t; @@ -25,6 +29,92 @@ class parse_t void parse(const wcstring &str); }; +class parse_node_t; +typedef std::vector parse_node_tree_t; +typedef size_t node_offset_t; + + +enum parse_token_type_t +{ + token_type_invalid, + + // Non-terminal tokens + symbol_statement_list, + symbol_statement, + symbol_block_statement, + symbol_block_header, + symbol_if_header, + symbol_for_header, + symbol_while_header, + symbol_begin_header, + symbol_function_header, + symbol_boolean_statement, + symbol_decorated_statement, + symbol_plain_statement, + symbol_arguments_or_redirections_list, + symbol_argument_or_redirection, + + // Terminal types + parse_token_type_string, + parse_token_type_pipe, + parse_token_type_redirection, + parse_token_background, + parse_token_type_end, + parse_token_type_terminate, + + FIRST_PARSE_TOKEN_TYPE = parse_token_type_string +}; + +enum parse_keyword_t +{ + parse_keyword_none, + parse_keyword_if, + parse_keyword_else, + parse_keyword_for, + parse_keyword_in, + parse_keyword_while, + parse_keyword_begin, + parse_keyword_function, + parse_keyword_switch, + parse_keyword_end, + parse_keyword_and, + parse_keyword_or, + parse_keyword_not, + parse_keyword_command, + parse_keyword_builtin +}; + +/** Base class for nodes of a parse tree */ +class parse_node_t +{ + public: + + /* Type of the node */ + enum parse_token_type_t type; + + /* Start in the source code */ + size_t source_start; + + /* Length of our range in the source code */ + size_t source_length; + + /* Children */ + node_offset_t child_start; + node_offset_t child_count; + + /* Type-dependent data */ + uint32_t tag; + + + /* Description */ + wcstring describe(void) const; + + /* Constructor */ + explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0) + { + } +}; + /* Fish grammar: -- cgit v1.2.3 From 3e3eefc2dcb2e0e31b224703a063e05dc8c67996 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Mon, 22 Jul 2013 18:26:15 -0700 Subject: Improvements to new parser. All functions and completions now parse. --- builtin.cpp | 61 ++-- builtin_printf.cpp | 2 +- exec.cpp | 16 +- fish.xcodeproj/project.pbxproj | 2 + fish_tests.cpp | 6 +- parse_exec.cpp | 170 +++++------ parse_exec.h | 49 ++-- parse_tree.cpp | 628 +++++++++++++++++++---------------------- parse_tree.h | 414 +++------------------------ parse_tree_construction.h | 586 ++++++++++++++++++++++++++++++++++++++ parse_util.cpp | 10 +- tokenizer.cpp | 10 +- tokenizer.h | 5 +- 13 files changed, 1080 insertions(+), 879 deletions(-) create mode 100644 parse_tree_construction.h (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/builtin.cpp b/builtin.cpp index 3b40be3c..d2a80a8c 100644 --- a/builtin.cpp +++ b/builtin.cpp @@ -3946,53 +3946,53 @@ static int builtin_history(parser_t &parser, wchar_t **argv) struct parse_execution_simulator_t : public parse_execution_visitor_t { wcstring_list_t result; - + wcstring &back() { assert(! result.empty()); return result.back(); } - + void append_src(node_offset_t idx) { wcstring tmp; context->get_source(idx, &tmp); back().append(tmp); } - + void append(const wchar_t *s) { back().append(s); } - + bool enter_job_list(void) { return true; } - + bool enter_job(void) { result.resize(result.size() + 1); return true; } - + void visit_statement(void) { } - + virtual void visit_boolean_statement(void) { } - + virtual void enter_if_clause(const exec_if_clause_t &statement) { } - + virtual void exit_if_clause(const exec_if_clause_t &statement) { append_format(back(), L"\nIF successful jump to %lu", (unsigned long)statement.body); } - + void visit_basic_statement(const exec_basic_statement_t &statement) { wcstring &line = this->back(); @@ -4005,34 +4005,35 @@ struct parse_execution_simulator_t : public parse_execution_visitor_t case exec_basic_statement_t::decoration_builtin: line.append(L" "); break; - + case exec_basic_statement_t::decoration_command: line.append(L" "); break; - + default: - break; + break; } - + line.append(L"cmd:"); this->append_src(statement.command_idx); for (size_t i=0; i < statement.arguments().size(); i++) { const exec_argument_t &arg = statement.arguments().at(i); append(L" "); - append(L"arg:"); + append(L"arg:"); append_src(arg.parse_node_idx); } } - - void visit_function(const exec_function_header_t &function) { + + void visit_function(const exec_function_header_t &function) + { wcstring &line = this->back(); line.append(L"define function: "); wcstring tmp; context->get_source(function.name_idx, &tmp); line.append(tmp); } - + void exit_job_list(void) { } @@ -4074,17 +4075,19 @@ int builtin_parse(parser_t &parser, wchar_t **argv) } else { - parse_execution_context_t ctx(parse_tree, src); - parse_execution_simulator_t sim; - sim.context = &ctx; - while (ctx.visit_next_node(&sim)) - { - } - stdout_buffer.append(L"Simulating execution:\n"); - for (size_t i=0; i < sim.result.size(); i++) - { - stdout_buffer.append(sim.result.at(i)); - stdout_buffer.push_back(L'\n'); + if (0) { + parse_execution_context_t ctx(parse_tree, src); + parse_execution_simulator_t sim; + sim.context = &ctx; + while (ctx.visit_next_node(&sim)) + { + } + stdout_buffer.append(L"Simulating execution:\n"); + for (size_t i=0; i < sim.result.size(); i++) + { + stdout_buffer.append(sim.result.at(i)); + stdout_buffer.push_back(L'\n'); + } } } } diff --git a/builtin_printf.cpp b/builtin_printf.cpp index efe4a211..b7df7fa8 100644 --- a/builtin_printf.cpp +++ b/builtin_printf.cpp @@ -632,7 +632,7 @@ int builtin_printf_state_t::print_formatted(const wchar_t *format, int argc, wch } break; } - + modify_allowed_format_specifiers(ok, "aAcdeEfFgGiosuxX", true); for (;; f++, direc_length++) diff --git a/exec.cpp b/exec.cpp index d5c7d4bf..5f467663 100644 --- a/exec.cpp +++ b/exec.cpp @@ -537,7 +537,7 @@ static bool can_use_posix_spawn_for_job(const job_t *job, const process_t *proce return false; } } - + /* Now see if we have a redirection involving a file. The only one we allow is /dev/null, which we assume will not fail. */ bool result = true; for (size_t idx = 0; idx < job->io.size(); idx++) @@ -545,8 +545,8 @@ static bool can_use_posix_spawn_for_job(const job_t *job, const process_t *proce const shared_ptr &io = job->io.at(idx); if (redirection_is_to_real_file(io.get())) { - result = false; - break; + result = false; + break; } } return result; @@ -1192,16 +1192,16 @@ void exec(parser_t &parser, job_t *j) forking is expensive, fish tries to avoid it when possible. */ - + bool fork_was_skipped = false; - + const shared_ptr stdout_io = io_chain_get(j->io, STDOUT_FILENO); const shared_ptr stderr_io = io_chain_get(j->io, STDERR_FILENO); - + /* If we are outputting to a file, we have to actually do it, even if we have no output, so that we can truncate the file. Does not apply to /dev/null. */ bool must_fork = redirection_is_to_real_file(stdout_io.get()) || redirection_is_to_real_file(stderr_io.get()); if (! must_fork) - { + { if (p->next == NULL) { const bool stdout_is_to_buffer = stdout_io && stdout_io->io_mode == IO_BUFFER; @@ -1250,7 +1250,7 @@ void exec(parser_t &parser, job_t *j) } } } - + if (fork_was_skipped) { diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 93293e50..5ae10bfc 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -465,6 +465,7 @@ D0D2693C159835CA005D9B9C /* fish */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish; sourceTree = BUILT_PRODUCTS_DIR; }; D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_test.cpp; sourceTree = ""; }; D0F5E28415A7A32D00315DFF /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = ""; }; + D0FE8EE6179CA8A5008C9F21 /* parse_tree_construction.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_tree_construction.h; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -589,6 +590,7 @@ D0A0850C13B3ACEE0099B651 /* expand.h */, D0A0853D13B3ACEE0099B651 /* expand.cpp */, D0C52F361765284C00BFAB82 /* parse_tree.h */, + D0FE8EE6179CA8A5008C9F21 /* parse_tree_construction.h */, D0C52F351765284C00BFAB82 /* parse_tree.cpp */, D0C52F341765281F00BFAB82 /* parse_exec.h */, D0C52F331765281F00BFAB82 /* parse_exec.cpp */, diff --git a/fish_tests.cpp b/fish_tests.cpp index 739b47b0..6ebd3d22 100644 --- a/fish_tests.cpp +++ b/fish_tests.cpp @@ -534,7 +534,7 @@ static void test_utils() { say(L"Testing utils"); const wchar_t *a = L"echo (echo (echo hi"; - + const wchar_t *begin = NULL, *end = NULL; parse_util_cmdsubst_extent(a, 0, &begin, &end); if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__); @@ -544,7 +544,7 @@ static void test_utils() if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__); parse_util_cmdsubst_extent(a, 3, &begin, &end); if (begin != a || end != begin + wcslen(begin)) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__); - + parse_util_cmdsubst_extent(a, 8, &begin, &end); if (begin != a + wcslen(L"echo (")) err(L"parse_util_cmdsubst_extent failed on line %ld", (long)__LINE__); @@ -1842,7 +1842,7 @@ int main(int argc, char **argv) builtin_init(); reader_init(); env_init(); - + test_new_parser(); return 0; diff --git a/parse_exec.cpp b/parse_exec.cpp index 0028d530..3f2074f4 100644 --- a/parse_exec.cpp +++ b/parse_exec.cpp @@ -6,11 +6,11 @@ struct exec_node_t node_offset_t parse_node_idx; node_offset_t body_parse_node_idx; bool visited; - + explicit exec_node_t(node_offset_t pni) : parse_node_idx(pni), body_parse_node_idx(NODE_OFFSET_INVALID), visited(false) { } - + explicit exec_node_t(node_offset_t pni, node_offset_t body_pni) : parse_node_idx(pni), body_parse_node_idx(body_pni), visited(false) { } @@ -18,7 +18,7 @@ struct exec_node_t exec_basic_statement_t::exec_basic_statement_t() : command_idx(0), decoration(decoration_plain) { - + } @@ -26,21 +26,21 @@ class parse_exec_t { parse_node_tree_t parse_tree; wcstring src; - + /* The stack of nodes as we execute them */ std::vector exec_nodes; - + /* The stack of commands being built */ std::vector assembling_statements; - + /* Current visitor (very transient) */ struct parse_execution_visitor_t * visitor; - + const parse_node_t &get_child(const parse_node_t &parent, node_offset_t which) const { return parse_tree.at(parent.child_offset(which)); } - + void pop_push_specific(node_offset_t idx1, node_offset_t idx2 = NODE_OFFSET_INVALID, node_offset_t idx3 = NODE_OFFSET_INVALID, node_offset_t idx4 = NODE_OFFSET_INVALID, node_offset_t idx5 = NODE_OFFSET_INVALID) { PARSE_ASSERT(! exec_nodes.empty()); @@ -48,10 +48,10 @@ class parse_exec_t exec_node_t &top = exec_nodes.back(); const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); node_offset_t child_node_idx = parse_node.child_start; - + // Remove the top node exec_nodes.pop_back(); - + // Append the given children, backwards const node_offset_t idxs[] = {idx5, idx4, idx3, idx2, idx1}; for (size_t q=0; q < sizeof idxs / sizeof *idxs; q++) @@ -65,18 +65,18 @@ class parse_exec_t } } - + void push(node_offset_t global_idx) { exec_nodes.push_back(exec_node_t(global_idx)); } - + void push(const exec_node_t &node) { exec_nodes.push_back(node); } - + void pop_push(node_offset_t child_idx, node_offset_t child_count = 1) { PARSE_ASSERT(! exec_nodes.empty()); @@ -92,10 +92,10 @@ class parse_exec_t const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); PARSE_ASSERT(child_idx < parse_node.child_count); node_offset_t child_node_idx = parse_node.child_start + child_idx; - + // Remove the top node exec_nodes.pop_back(); - + // Append the given children, backwards node_offset_t cursor = child_count; while (cursor--) @@ -104,20 +104,20 @@ class parse_exec_t } } } - + void pop() { PARSE_ASSERT(! exec_nodes.empty()); exec_nodes.pop_back(); } - + void pop_push_all() { exec_node_t &top = exec_nodes.back(); const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); pop_push(0, parse_node.child_count); } - + void assemble_1_argument_or_redirection(node_offset_t idx, exec_arguments_and_redirections_t *output) const { const parse_node_t &node = parse_tree.at(idx); @@ -130,28 +130,28 @@ class parse_exec_t { case parse_token_type_string: // Argument - { - exec_argument_t arg = exec_argument_t(); - arg.parse_node_idx = child_idx; - output->arguments.push_back(arg); - } - break; - + { + exec_argument_t arg = exec_argument_t(); + arg.parse_node_idx = child_idx; + output->arguments.push_back(arg); + } + break; + case parse_token_type_redirection: // Redirection - { - exec_redirection_t redirect = exec_redirection_t(); - redirect.parse_node_idx = child_idx; - output->redirections.push_back(redirect); - } - break; - + { + exec_redirection_t redirect = exec_redirection_t(); + redirect.parse_node_idx = child_idx; + output->redirections.push_back(redirect); + } + break; + default: PARSER_DIE(); break; } } - + void assemble_arguments_and_redirections(node_offset_t start_idx, exec_arguments_and_redirections_t *output) const { node_offset_t idx = start_idx; @@ -173,7 +173,7 @@ class parse_exec_t } } } - + void assemble_command_for_plain_statement(node_offset_t idx, parse_keyword_t decoration) { const parse_node_t &node = parse_tree.at(idx); @@ -185,10 +185,10 @@ class parse_exec_t assemble_arguments_and_redirections(node.child_offset(1), &statement.arguments_and_redirections); visitor->visit_basic_statement(statement); } - + void assemble_block_statement(node_offset_t parse_node_idx) { - + const parse_node_t &node = parse_tree.at(parse_node_idx); PARSE_ASSERT(node.type == symbol_block_statement); PARSE_ASSERT(node.child_count == 5); @@ -196,24 +196,24 @@ class parse_exec_t // Fetch arguments and redirections. These ought to be evaluated before the job list exec_block_statement_t statement; assemble_arguments_and_redirections(node.child_offset(4), &statement.arguments_and_redirections); - + // Generic visit visitor->enter_block_statement(statement); - + // Dig into the header to discover the type const parse_node_t &header_parent = parse_tree.at(node.child_offset(0)); PARSE_ASSERT(header_parent.type == symbol_block_header); - PARSE_ASSERT(header_parent.child_count == 1); + PARSE_ASSERT(header_parent.child_count == 1); const node_offset_t header_idx = header_parent.child_offset(0); - + // Fetch body (job list) node_offset_t body_idx = node.child_offset(2); PARSE_ASSERT(parse_tree.at(body_idx).type == symbol_job_list); - + pop(); push(exec_node_t(header_idx, body_idx)); } - + /* which: 0 -> if, 1 -> else if, 2 -> else */ void assemble_if_else_clause(exec_node_t &exec_node, const parse_node_t &node, int which) { @@ -227,7 +227,7 @@ class parse_exec_t PARSE_ASSERT(node.type == symbol_else_continuation); PARSE_ASSERT(node.child_count == 2); } - + struct exec_if_clause_t clause; if (which == 0) { @@ -252,7 +252,7 @@ class parse_exec_t pop(); } } - + void assemble_arguments(node_offset_t start_idx, exec_argument_list_t *output) const { node_offset_t idx = start_idx; @@ -282,24 +282,24 @@ class parse_exec_t } } } - + void assemble_1_case_item(exec_switch_statement_t *statement, node_offset_t node_idx) { const parse_node_t &node = parse_tree.at(node_idx); PARSE_ASSERT(node.type == symbol_case_item); - + // add a new case size_t len = statement->cases.size(); statement->cases.resize(len + 1); exec_switch_case_t &new_case = statement->cases.back(); - + // assemble it new_case.body = node.child_offset(3); assemble_arguments(node.child_offset(1), &new_case.arguments); - - + + } - + void assemble_case_item_list(exec_switch_statement_t *statement, node_offset_t node_idx) { const parse_node_t &node = parse_tree.at(node_idx); @@ -311,21 +311,21 @@ class parse_exec_t assemble_case_item_list(statement, node.child_offset(1)); } } - + void assemble_switch_statement(const exec_node_t &exec_node, const parse_node_t &parse_node) { PARSE_ASSERT(parse_node.type == symbol_switch_statement); exec_switch_statement_t statement; - + statement.argument.parse_node_idx = parse_node.child_offset(1); assemble_case_item_list(&statement, parse_node.child_offset(3)); - + visitor->visit_switch_statement(statement); - + // pop off the switch pop(); } - + void assemble_function_header(const exec_node_t &exec_node, const parse_node_t &header) { PARSE_ASSERT(header.type == symbol_function_header); @@ -336,17 +336,17 @@ class parse_exec_t function_info.body_idx = exec_node.body_parse_node_idx; assemble_arguments(header.child_offset(2), &function_info.arguments); visitor->visit_function(function_info); - + // Always pop pop(); } - + void enter_parse_node(size_t idx); void run_top_node(void); - - public: - + +public: + void get_node_string(node_offset_t idx, wcstring *output) const { const parse_node_t &node = parse_tree.at(idx); @@ -354,9 +354,9 @@ class parse_exec_t PARSE_ASSERT(node.source_start + node.source_length <= src.size()); output->assign(src, node.source_start, node.source_length); } - + bool visit_next_node(parse_execution_visitor_t *v); - + parse_exec_t(const parse_node_tree_t &tree, const wcstring &s) : parse_tree(tree), src(s), visitor(NULL) { if (! parse_tree.empty()) @@ -373,7 +373,7 @@ void parse_exec_t::run_top_node() const node_offset_t parse_node_idx = exec_node.parse_node_idx; const parse_node_t &parse_node = parse_tree.at(exec_node.parse_node_idx); bool log = true; - + if (log) { wcstring tmp; @@ -381,7 +381,7 @@ void parse_exec_t::run_top_node() tmp.append(parse_node.describe()); printf("%ls\n", tmp.c_str()); } - + switch (parse_node.type) { case symbol_job_list: @@ -404,7 +404,7 @@ void parse_exec_t::run_top_node() pop_push(0, 2); } break; - + case symbol_job: { PARSE_ASSERT(parse_node.child_count == 2); @@ -412,7 +412,7 @@ void parse_exec_t::run_top_node() pop_push_all(); break; } - + case symbol_job_continuation: PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 3); if (parse_node.child_count == 0) @@ -426,51 +426,51 @@ void parse_exec_t::run_top_node() // Skip the pipe pop_push(1, 2); } - break; - + break; + case symbol_statement: { PARSE_ASSERT(parse_node.child_count == 1); pop_push_all(); break; } - + case symbol_block_statement: { PARSE_ASSERT(parse_node.child_count == 5); assemble_block_statement(parse_node_idx); break; } - + case symbol_block_header: { PARSE_ASSERT(parse_node.child_count == 1); pop_push_all(); break; } - + case symbol_function_header: { PARSE_ASSERT(parse_node.child_count == 3); assemble_function_header(exec_node, parse_node); break; } - + case symbol_if_statement: { - PARSE_ASSERT(parse_node.child_count == 3); + PARSE_ASSERT(parse_node.child_count == 4); pop_push(0, 2); break; } - + case symbol_if_clause: { PARSE_ASSERT(parse_node.child_count == 4); assemble_if_else_clause(exec_node, parse_node, 0); pop(); - break; + break; } - + case symbol_else_clause: { PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 2); @@ -486,7 +486,7 @@ void parse_exec_t::run_top_node() } break; } - + case symbol_else_continuation: { // Figure out if this is an else if or a terminating else @@ -505,17 +505,17 @@ void parse_exec_t::run_top_node() } break; } - + case symbol_switch_statement: { assemble_switch_statement(exec_node, parse_node); break; } - + case symbol_decorated_statement: { PARSE_ASSERT(parse_node.child_count == 1 || parse_node.child_count == 2); - + node_offset_t plain_statement_idx = parse_node.child_offset(parse_node.child_count - 1); parse_keyword_t decoration = static_cast(parse_node.tag); assemble_command_for_plain_statement(plain_statement_idx, decoration); @@ -528,20 +528,20 @@ void parse_exec_t::run_top_node() case symbol_plain_statement: case symbol_arguments_or_redirections_list: case symbol_argument_or_redirection: - fprintf(stderr, "Unhandled token type %ls at index %ld\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx); + fprintf(stderr, "Unexpected token type %ls at index %ld. This should have been handled by the parent.\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx); PARSER_DIE(); break; - + case parse_token_type_end: PARSE_ASSERT(parse_node.child_count == 0); pop(); break; - + default: fprintf(stderr, "Unhandled token type %ls at index %ld\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx); PARSER_DIE(); break; - + } } @@ -553,7 +553,7 @@ bool parse_exec_t::visit_next_node(parse_execution_visitor_t *v) { return false; } - + visitor = v; run_top_node(); visitor = NULL; diff --git a/parse_exec.h b/parse_exec.h index 197f656d..1eea99ab 100644 --- a/parse_exec.h +++ b/parse_exec.h @@ -13,13 +13,13 @@ class parse_exec_t; class parse_execution_context_t { parse_exec_t *ctx; //owned - - public: + +public: parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s); ~parse_execution_context_t(); - + bool visit_next_node(parse_execution_visitor_t *visitor); - + // Gets the source for a node at a given index void get_source(node_offset_t idx, wcstring *result) const; }; @@ -53,10 +53,10 @@ struct exec_basic_statement_t { // Node containing the command node_offset_t command_idx; - + // Arguments exec_arguments_and_redirections_t arguments_and_redirections; - + // Decoration enum { @@ -64,9 +64,9 @@ struct exec_basic_statement_t decoration_command, decoration_builtin } decoration; - + exec_basic_statement_t(); - + void set_decoration(uint32_t k) { PARSE_ASSERT(k == parse_keyword_none || k == parse_keyword_command || k == parse_keyword_builtin); @@ -86,12 +86,12 @@ struct exec_basic_statement_t break; } } - + const exec_argument_list_t &arguments() const { return arguments_and_redirections.arguments; } - + const exec_redirection_list_t &redirections() const { return arguments_and_redirections.redirections; @@ -102,10 +102,10 @@ struct exec_function_header_t { // Node containing the function name node_offset_t name_idx; - + // Node containing the function body node_offset_t body_idx; - + // Arguments exec_argument_list_t arguments; }; @@ -138,23 +138,32 @@ struct parse_execution_visitor_t { node_offset_t node_idx; parse_execution_context_t *context; - + parse_execution_visitor_t() : node_idx(0), context(NULL) { } - - virtual bool enter_job_list(void) { return true; } - virtual bool enter_job(void) { return true; } + + virtual bool enter_job_list(void) + { + return true; + } + virtual bool enter_job(void) + { + return true; + } virtual void visit_statement(void) { } virtual void visit_function(const exec_function_header_t &function) { } - virtual bool enter_block_statement(const exec_block_statement_t &statement) { return true; } - + virtual bool enter_block_statement(const exec_block_statement_t &statement) + { + return true; + } + virtual void enter_if_clause(const exec_if_clause_t &statement) { } virtual void exit_if_clause(const exec_if_clause_t &statement) { } - + virtual void visit_switch_statement(const exec_switch_statement_t &header) { } - + virtual void visit_boolean_statement(void) { } virtual void visit_basic_statement(const exec_basic_statement_t &statement) { } virtual void exit_job(void) { } diff --git a/parse_tree.cpp b/parse_tree.cpp index 24bf41e7..aea3e729 100644 --- a/parse_tree.cpp +++ b/parse_tree.cpp @@ -1,4 +1,4 @@ -#include "parse_tree.h" +#include "parse_tree_construction.h" #include "tokenizer.h" #include @@ -11,26 +11,28 @@ wcstring parse_error_t::describe(const wcstring &src) const { // Locate the beginning of this line of source size_t line_start = 0; - + // Look for a newline prior to source_start. If we don't find one, start at the beginning of the string; otherwise start one past the newline size_t newline = src.find_last_of(L'\n', source_start); + fprintf(stderr, "newline: %lu, source_start %lu, source_length %lu\n", newline, source_start, source_length); if (newline != wcstring::npos) { - line_start = newline + 1; + line_start = newline;// + 1; } - + size_t line_end = src.find(L'\n', source_start + source_length); if (line_end == wcstring::npos) { line_end = src.size(); } assert(line_end >= line_start); + fprintf(stderr, "source start: %lu, line start %lu\n", source_start, line_start); assert(source_start >= line_start); - + // Append the line of text result.push_back(L'\n'); result.append(src, line_start, line_end - line_start); - + // Append the caret line result.push_back(L'\n'); result.append(source_start - line_start, L' '); @@ -43,68 +45,115 @@ wcstring token_type_description(parse_token_type_t type) { switch (type) { - case token_type_invalid: return L"invalid"; - - case symbol_job_list: return L"job_list"; - case symbol_job: return L"job"; - case symbol_job_continuation: return L"job_continuation"; - - case symbol_statement: return L"statement"; - case symbol_block_statement: return L"block_statement"; - case symbol_block_header: return L"block_header"; - case symbol_for_header: return L"for_header"; - case symbol_while_header: return L"while_header"; - case symbol_begin_header: return L"begin_header"; - case symbol_function_header: return L"function_header"; - - case symbol_if_statement: return L"if_statement"; - case symbol_if_clause: return L"if_clause"; - case symbol_else_clause: return L"else_clause"; - case symbol_else_continuation: return L"else_continuation"; - - case symbol_switch_statement: return L"switch_statement"; - case symbol_case_item_list: return L"case_item_list"; - case symbol_case_item: return L"case_item"; - - case symbol_argument_list_nonempty: return L"argument_list_nonempty"; - case symbol_argument_list: return L"argument_list"; - - case symbol_boolean_statement: return L"boolean_statement"; - case symbol_decorated_statement: return L"decorated_statement"; - case symbol_plain_statement: return L"plain_statement"; - case symbol_arguments_or_redirections_list: return L"arguments_or_redirections_list"; - case symbol_argument_or_redirection: return L"argument_or_redirection"; - - case parse_token_type_string: return L"token_string"; - case parse_token_type_pipe: return L"token_pipe"; - case parse_token_type_redirection: return L"token_redirection"; - case parse_token_background: return L"token_background"; - case parse_token_type_end: return L"token_end"; - case parse_token_type_terminate: return L"token_terminate"; - - default: return format_string(L"Unknown token type %ld", static_cast(type)); + case token_type_invalid: + return L"invalid"; + + case symbol_job_list: + return L"job_list"; + case symbol_job: + return L"job"; + case symbol_job_continuation: + return L"job_continuation"; + + case symbol_statement: + return L"statement"; + case symbol_block_statement: + return L"block_statement"; + case symbol_block_header: + return L"block_header"; + case symbol_for_header: + return L"for_header"; + case symbol_while_header: + return L"while_header"; + case symbol_begin_header: + return L"begin_header"; + case symbol_function_header: + return L"function_header"; + + case symbol_if_statement: + return L"if_statement"; + case symbol_if_clause: + return L"if_clause"; + case symbol_else_clause: + return L"else_clause"; + case symbol_else_continuation: + return L"else_continuation"; + + case symbol_switch_statement: + return L"switch_statement"; + case symbol_case_item_list: + return L"case_item_list"; + case symbol_case_item: + return L"case_item"; + + case symbol_argument_list_nonempty: + return L"argument_list_nonempty"; + case symbol_argument_list: + return L"argument_list"; + + case symbol_boolean_statement: + return L"boolean_statement"; + case symbol_decorated_statement: + return L"decorated_statement"; + case symbol_plain_statement: + return L"plain_statement"; + case symbol_arguments_or_redirections_list: + return L"arguments_or_redirections_list"; + case symbol_argument_or_redirection: + return L"argument_or_redirection"; + + case parse_token_type_string: + return L"token_string"; + case parse_token_type_pipe: + return L"token_pipe"; + case parse_token_type_redirection: + return L"token_redirection"; + case parse_token_type_background: + return L"token_background"; + case parse_token_type_end: + return L"token_end"; + case parse_token_type_terminate: + return L"token_terminate"; + case symbol_optional_background: + return L"optional_background"; } + return format_string(L"Unknown token type %ld", static_cast(type)); } wcstring keyword_description(parse_keyword_t k) { switch (k) { - case parse_keyword_none: return L"none"; - case parse_keyword_if: return L"if"; - case parse_keyword_else: return L"else"; - case parse_keyword_for: return L"for"; - case parse_keyword_in: return L"in"; - case parse_keyword_while: return L"while"; - case parse_keyword_begin: return L"begin"; - case parse_keyword_function: return L"function"; - case parse_keyword_switch: return L"switch"; - case parse_keyword_end: return L"end"; - case parse_keyword_and: return L"and"; - case parse_keyword_or: return L"or"; - case parse_keyword_not: return L"not"; - case parse_keyword_command: return L"command"; - case parse_keyword_builtin: return L"builtin"; + case parse_keyword_none: + return L"none"; + case parse_keyword_if: + return L"if"; + case parse_keyword_else: + return L"else"; + case parse_keyword_for: + return L"for"; + case parse_keyword_in: + return L"in"; + case parse_keyword_while: + return L"while"; + case parse_keyword_begin: + return L"begin"; + case parse_keyword_function: + return L"function"; + case parse_keyword_switch: + return L"switch"; + case parse_keyword_end: + return L"end"; + case parse_keyword_and: + return L"and"; + case parse_keyword_or: + return L"or"; + case parse_keyword_not: + return L"not"; + case parse_keyword_command: + return L"command"; + case parse_keyword_builtin: + return L"builtin"; default: return format_string(L"Unknown keyword type %ld", static_cast(k)); } @@ -123,7 +172,7 @@ struct parse_token_t enum parse_keyword_t keyword; // Any keyword represented by this parser size_t source_start; size_t source_length; - + wcstring describe() const; }; @@ -147,19 +196,28 @@ static parse_token_t parse_token_from_tokenizer_token(enum token_type tokenizer_ case TOK_STRING: result.type = parse_token_type_string; break; - + case TOK_PIPE: result.type = parse_token_type_pipe; break; - + case TOK_END: result.type = parse_token_type_end; break; - + case TOK_BACKGROUND: - result.type = parse_token_background; + result.type = parse_token_type_background; break; + case TOK_REDIRECT_OUT: + case TOK_REDIRECT_APPEND: + case TOK_REDIRECT_IN: + case TOK_REDIRECT_FD: + case TOK_REDIRECT_NOCLOB: + result.type = parse_token_type_redirection; + break; + + default: fprintf(stderr, "Bad token type %d passed to %s\n", (int)tokenizer_token_type, __FUNCTION__); assert(0); @@ -172,15 +230,15 @@ static void dump_tree_recursive(const parse_node_tree_t &nodes, const wcstring & { assert(start < nodes.size()); const parse_node_t &node = nodes.at(start); - + const size_t spacesPerIndent = 2; - + // unindent statement lists by 1 to flatten them if (node.type == symbol_job_list || node.type == symbol_arguments_or_redirections_list) { if (indent > 0) indent -= 1; } - + append_format(*result, L"%2lu - %l2u ", *line, start); result->append(indent * spacesPerIndent, L' ');; result->append(node.describe()); @@ -206,7 +264,7 @@ static wcstring dump_tree(const parse_node_tree_t &nodes, const wcstring &src) { if (nodes.empty()) return L"(empty!)"; - + size_t line = 0; wcstring result; dump_tree_recursive(nodes, src, 0, 0, &result, &line); @@ -218,15 +276,15 @@ struct parse_stack_element_t enum parse_token_type_t type; enum parse_keyword_t keyword; node_offset_t node_idx; - + parse_stack_element_t(parse_token_type_t t) : type(t), keyword(parse_keyword_none), node_idx(-1) { } - + parse_stack_element_t(parse_keyword_t k) : type(parse_token_type_string), keyword(k), node_idx(-1) { } - + wcstring describe(void) const { wcstring result = token_type_description(type); @@ -242,13 +300,13 @@ struct parse_stack_element_t class parse_ll_t { friend class parse_t; - + std::vector symbol_stack; // LL parser stack parse_node_tree_t nodes; - + bool fatal_errored; parse_error_list_t errors; - + // Constructor parse_ll_t() : fatal_errored(false) { @@ -258,33 +316,30 @@ class parse_ll_t symbol_stack.push_back(elem); // goal token nodes.push_back(parse_node_t(symbol_job_list)); } - + bool top_node_match_token(parse_token_t token); - + // implementation of certain parser constructions void accept_token(parse_token_t token, const wcstring &src); void accept_token_job_list(parse_token_t token); void accept_token_job(parse_token_t token); void accept_token_job_continuation(parse_token_t token); - void accept_token_statement(parse_token_t token); - void accept_token_block_header(parse_token_t token); void accept_token_else_clause(parse_token_t token); void accept_token_else_continuation(parse_token_t token); - void accept_token_boolean_statement(parse_token_t token); - void accept_token_decorated_statement(parse_token_t token); void accept_token_plain_statement(parse_token_t token); void accept_token_argument_list(parse_token_t token); void accept_token_arguments_or_redirections_list(parse_token_t token); void accept_token_argument_or_redirection(parse_token_t token); bool accept_token_string(parse_token_t token); - + void token_unhandled(parse_token_t token, const char *function); - + void parse_error(const wchar_t *expected, parse_token_t token); + void parse_error(parse_token_t token, const wchar_t *format, ...); void append_error_callout(wcstring &error_message, parse_token_t token); - + void dump_stack(void) const; - + // Get the node corresponding to the top element of the stack parse_node_t &node_for_top_symbol() { @@ -294,17 +349,17 @@ class parse_ll_t PARSE_ASSERT(top_symbol.node_idx < nodes.size()); return nodes.at(top_symbol.node_idx); } - + parse_token_type_t stack_top_type() const { return symbol_stack.back().type; } - + void top_node_set_tag(uint32_t tag) { this->node_for_top_symbol().tag = tag; } - + inline void add_child_to_node(size_t parent_node_idx, parse_stack_element_t *tok) { PARSE_ASSERT(tok->type != token_type_invalid); @@ -312,19 +367,19 @@ class parse_ll_t nodes.push_back(parse_node_t(tok->type)); nodes.at(parent_node_idx).child_count += 1; } - + inline void symbol_stack_pop() { symbol_stack.pop_back(); } - + // Pop from the top of the symbol stack, then push, updating node counts. Note that these are pushed in reverse order, so the first argument will be on the top of the stack. inline void symbol_stack_pop_push_int(parse_stack_element_t tok1 = token_type_invalid, parse_stack_element_t tok2 = token_type_invalid, parse_stack_element_t tok3 = token_type_invalid, parse_stack_element_t tok4 = token_type_invalid, parse_stack_element_t tok5 = token_type_invalid) { - + // Logging? - if (1) + if (0) { fprintf(stderr, "Pop %ls (%lu)\n", token_type_description(symbol_stack.back().type).c_str(), symbol_stack.size()); if (tok5.type != token_type_invalid) fprintf(stderr, "Push %ls\n", tok5.describe().c_str()); @@ -333,17 +388,17 @@ class parse_ll_t if (tok2.type != token_type_invalid) fprintf(stderr, "Push %ls\n", tok2.describe().c_str()); if (tok1.type != token_type_invalid) fprintf(stderr, "Push %ls\n", tok1.describe().c_str()); } - + // Get the node for the top symbol and tell it about its children size_t node_idx = symbol_stack.back().node_idx; parse_node_t &node = nodes.at(node_idx); - + // Should have no children yet PARSE_ASSERT(node.child_count == 0); - + // Tell the node where its children start node.child_start = nodes.size(); - + // Add nodes for the children // Confusingly, we want our nodes to be in forwards order (last token last, so dumps look nice), but the symbols should be reverse order (last token first, so it's lowest on the stack) if (tok1.type != token_type_invalid) add_child_to_node(node_idx, &tok1); @@ -351,7 +406,7 @@ class parse_ll_t if (tok3.type != token_type_invalid) add_child_to_node(node_idx, &tok3); if (tok4.type != token_type_invalid) add_child_to_node(node_idx, &tok4); if (tok5.type != token_type_invalid) add_child_to_node(node_idx, &tok5); - + // The above set the node_idx. Now replace the top of the stack. symbol_stack.pop_back(); if (tok5.type != token_type_invalid) symbol_stack.push_back(tok5); @@ -360,31 +415,69 @@ class parse_ll_t if (tok2.type != token_type_invalid) symbol_stack.push_back(tok2); if (tok1.type != token_type_invalid) symbol_stack.push_back(tok1); } - + template - inline void symbol_stack_pop_push2() + inline void symbol_stack_pop_push2(typename T::magic_seq_type_t x = 0) { symbol_stack_pop_push_int(T::t0::get_token(), T::t1::get_token(), T::t2::get_token(), T::t3::get_token(), T::t4::get_token()); } - + template - inline void symbol_stack_pop_push_production(int which) + inline void symbol_stack_pop_push2(typename T::magic_symbol_type_t x = 0) { + symbol_stack_pop_push_int(T::get_token()); + } + + // Singular. Sole productions are always of type Seq. + template + inline void symbol_stack_produce(parse_token_t tok, typename T::sole_production::magic_seq_type_t magic=0) + { + typedef typename T::sole_production seq; + symbol_stack_pop_push_int(seq::t0::get_token(), seq::t1::get_token(), seq::t2::get_token(), seq::t3::get_token(), seq::t4::get_token()); + } + + // Plural productions, of type Or. + template + inline void symbol_stack_produce(parse_token_t tok, typename T::productions::magic_or_type_t magic=0) + { + typedef typename T::productions ors; + int which = T::production(tok.type, tok.keyword); switch (which) { - case 0: symbol_stack_pop_push2(); break; - case 1: symbol_stack_pop_push2(); break; - case 2: symbol_stack_pop_push2(); break; - case 3: symbol_stack_pop_push2(); break; - case 4: symbol_stack_pop_push2(); break; + case 0: + symbol_stack_pop_push2(); + break; + case 1: + symbol_stack_pop_push2(); + break; + case 2: + symbol_stack_pop_push2(); + break; + case 3: + symbol_stack_pop_push2(); + break; + case 4: + symbol_stack_pop_push2(); + break; + + case NO_PRODUCTION: + parse_error(tok, L"Failed to produce with stack top '%ls' for token '%ls'\n", symbol_stack.back().describe().c_str(), tok.describe().c_str()); + break; + + default: + parse_error(tok, L"Unexpected production %d for token %ls\n", which, tok.describe().c_str()); + break; } } + // Non-sequence basic productions template - inline void symbol_stack_produce(parse_token_t tok) + inline void symbol_stack_produce(parse_token_t tok, typename T::sole_production::magic_symbol_type_t magic=0) { - symbol_stack_pop_push_production(T::production(tok.type, tok.keyword)); + symbol_stack_pop_push_int(T::sole_production::get_token()); } + + }; void parse_ll_t::dump_stack(void) const @@ -410,7 +503,7 @@ void parse_ll_t::dump_stack(void) const } } } - + fprintf(stderr, "Stack dump (%lu elements):\n", symbol_stack.size()); for (size_t idx = 0; idx < lines.size(); idx++) { @@ -422,9 +515,31 @@ void parse_ll_t::token_unhandled(parse_token_t token, const char *function) { fprintf(stderr, "Unhandled token with type %ls in function %s\n", token_type_description(token.type).c_str(), function); this->dump_stack(); - PARSER_DIE(); + parse_error_t err; + err.text = format_string(L"Unhandled token with type %ls in function %s", token_type_description(token.type).c_str(), function); + err.source_start = token.source_start; + err.source_length = token.source_length; + this->errors.push_back(err); + this->fatal_errored = true; +} + +void parse_ll_t::parse_error(parse_token_t token, const wchar_t *fmt, ...) +{ + this->dump_stack(); + parse_error_t err; + + va_list va; + va_start(va, fmt); + err.text = vformat_string(fmt, va); + va_end(va); + + err.source_start = token.source_start; + err.source_length = token.source_length; + this->errors.push_back(err); + this->fatal_errored = true; } + void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) { wcstring desc = token_type_description(token.type); @@ -436,172 +551,6 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) fatal_errored = true; } -void parse_ll_t::accept_token_job_list(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_job_list); - switch (token.type) - { - case parse_token_type_string: - // 'end' is special - switch (token.keyword) - { - case parse_keyword_end: - case parse_keyword_else: - // End this job list - symbol_stack_pop_push_production(0); - break; - - default: - // Normal string - symbol_stack_pop_push_production(1); - break; - } - break; - - case parse_token_type_pipe: - case parse_token_type_redirection: - case parse_token_background: - symbol_stack_pop_push_production(1); - break; - - case parse_token_type_end: - // Empty line - symbol_stack_pop_push_production(2); - break; - - case parse_token_type_terminate: - // no more commands, just transition to empty - symbol_stack_pop_push_production(0); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_job_continuation(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_job_continuation); - switch (token.type) - { - case parse_token_type_pipe: - // Pipe, continuation - symbol_stack_pop_push_production(1); - break; - - default: - // Not a pipe, no job continuation - symbol_stack_pop_push_production(0); - break; - } -} - - -void parse_ll_t::accept_token_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_statement); - switch (token.type) - { - case parse_token_type_string: - switch (token.keyword) - { - case parse_keyword_and: - case parse_keyword_or: - case parse_keyword_not: - symbol_stack_pop_push_production(0); - break; - - case parse_keyword_for: - case parse_keyword_while: - case parse_keyword_function: - case parse_keyword_begin: - symbol_stack_pop_push_production(1); - break; - - case parse_keyword_if: - symbol_stack_pop_push_production(2); - break; - - case parse_keyword_else: - symbol_stack_pop(); - break; - - case parse_keyword_switch: - symbol_stack_pop_push_production(3); - break; - - case parse_keyword_end: - PARSER_DIE(); //todo - break; - - // 'in' is only special within a for_header - case parse_keyword_in: - case parse_keyword_none: - case parse_keyword_command: - case parse_keyword_builtin: - case parse_keyword_case: - symbol_stack_pop_push_production(4); - break; - - } - break; - - case parse_token_type_pipe: - case parse_token_type_redirection: - case parse_token_background: - case parse_token_type_terminate: - parse_error(L"statement", token); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - -void parse_ll_t::accept_token_block_header(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_block_header); - switch (token.type) - { - case parse_token_type_string: - switch (token.keyword) - { - case parse_keyword_else: - PARSER_DIE(); //todo - break; - - case parse_keyword_for: - symbol_stack_pop_push_production(0); - break; - - - case parse_keyword_while: - symbol_stack_pop_push_production(1); - break; - - case parse_keyword_function: - symbol_stack_pop_push_production(2); - break; - - case parse_keyword_begin: - symbol_stack_pop_push_production(3); - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - - } - break; - - default: - token_unhandled(token, __FUNCTION__); - break; - } -} - void parse_ll_t::accept_token_else_clause(parse_token_t token) { PARSE_ASSERT(stack_top_type() == symbol_else_clause); @@ -614,25 +563,6 @@ void parse_ll_t::accept_token_else_continuation(parse_token_t token) symbol_stack_produce(token); } -void parse_ll_t::accept_token_boolean_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_boolean_statement); - top_node_set_tag(token.keyword); - symbol_stack_produce(token); -} - -void parse_ll_t::accept_token_decorated_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_decorated_statement); - top_node_set_tag(token.keyword); - symbol_stack_produce(token); -} - -void parse_ll_t::accept_token_plain_statement(parse_token_t token) -{ - PARSE_ASSERT(stack_top_type() == symbol_plain_statement); - symbol_stack_produce(token); -} void parse_ll_t::accept_token_argument_list(parse_token_t token) { @@ -664,7 +594,7 @@ bool parse_ll_t::accept_token_string(parse_token_t token) symbol_stack_pop(); result = true; break; - + default: token_unhandled(token, __FUNCTION__); break; @@ -687,7 +617,7 @@ bool parse_ll_t::top_node_match_token(parse_token_t token) parse_node_t &node = node_for_top_symbol(); node.source_start = token.source_start; node.source_length = token.source_length; - + // We consumed this symbol symbol_stack.pop_back(); result = true; @@ -704,7 +634,7 @@ bool parse_ll_t::top_node_match_token(parse_token_t token) void parse_ll_t::accept_token(parse_token_t token, const wcstring &src) { - bool logit = true; + bool logit = false; if (logit) { const wcstring txt = wcstring(src, token.source_start, token.source_length); @@ -724,107 +654,113 @@ void parse_ll_t::accept_token(parse_token_t token, const wcstring &src) consumed = true; break; } - + switch (stack_top_type()) { - /* Symbols */ + /* Symbols */ case symbol_job_list: - accept_token_job_list(token); + symbol_stack_produce(token); break; - + case symbol_job: - symbol_stack_pop_push2(); + symbol_stack_produce(token); break; - + case symbol_job_continuation: - accept_token_job_continuation(token); + symbol_stack_produce(token); break; case symbol_statement: - accept_token_statement(token); + symbol_stack_produce(token); break; - + case symbol_if_statement: symbol_stack_produce(token); break; - + case symbol_if_clause: symbol_stack_produce(token); break; - + case symbol_else_clause: accept_token_else_clause(token); break; - + case symbol_else_continuation: accept_token_else_continuation(token); break; - + case symbol_block_statement: symbol_stack_produce(token); break; - + case symbol_block_header: symbol_stack_produce(token); break; - + case symbol_for_header: symbol_stack_produce(token); break; - + case symbol_while_header: symbol_stack_produce(token); break; - + case symbol_begin_header: symbol_stack_produce(token); break; - + case symbol_function_header: symbol_stack_produce(token); break; - + case symbol_switch_statement: symbol_stack_produce(token); break; - + case symbol_case_item_list: symbol_stack_produce(token); break; - + case symbol_case_item: symbol_stack_produce(token); break; - + case symbol_boolean_statement: - accept_token_boolean_statement(token); + top_node_set_tag(token.keyword); + symbol_stack_produce(token); break; - + case symbol_decorated_statement: - accept_token_decorated_statement(token); + top_node_set_tag(token.keyword); + symbol_stack_produce(token); break; - + case symbol_plain_statement: - accept_token_plain_statement(token); + symbol_stack_produce(token); break; - + case symbol_argument_list_nonempty: symbol_stack_produce(token); break; - + case symbol_argument_list: accept_token_argument_list(token); break; - + case symbol_arguments_or_redirections_list: accept_token_arguments_or_redirections_list(token); break; - + case symbol_argument_or_redirection: accept_token_argument_or_redirection(token); break; - - /* Tokens */ + + case symbol_optional_background: + symbol_stack_produce(token); + break; + + /* Tokens */ case parse_token_type_string: consumed = accept_token_string(token); break; @@ -846,11 +782,13 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt) parse_keyword_t result = parse_keyword_none; if (tok == TOK_STRING) { - - const struct { + + const struct + { const wchar_t *txt; parse_keyword_t keyword; - } keywords[] = { + } keywords[] = + { {L"if", parse_keyword_if}, {L"else", parse_keyword_else}, {L"for", parse_keyword_for}, @@ -867,7 +805,7 @@ static parse_keyword_t keyword_for_token(token_type tok, const wchar_t *tok_txt) {L"command", parse_keyword_command}, {L"builtin", parse_keyword_builtin} }; - + for (size_t i=0; i < sizeof keywords / sizeof *keywords; i++) { if (! wcscmp(keywords[i].txt, tok_txt)) @@ -888,36 +826,40 @@ bool parse_t::parse(const wcstring &str, parse_node_tree_t *output, parse_error_ token_type tok_type = static_cast(tok_last_type(&tok)); const wchar_t *tok_txt = tok_last(&tok); int tok_start = tok_get_pos(&tok); - + size_t tok_extent = tok_get_extent(&tok); + if (tok_type == TOK_ERROR) { fprintf(stderr, "Tokenizer error\n"); break; } - + parse_token_t token = parse_token_from_tokenizer_token(tok_type); token.tokenizer_type = tok_type; token.source_start = (size_t)tok_start; - token.source_length = wcslen(tok_txt); + token.source_length = tok_extent; token.keyword = keyword_for_token(tok_type, tok_txt); this->parser->accept_token(token, str); + + if (this->parser->fatal_errored) + break; } - wcstring result = dump_tree(this->parser->nodes, str); + wcstring result = L"";//dump_tree(this->parser->nodes, str); fprintf(stderr, "Tree (%ld nodes):\n%ls", this->parser->nodes.size(), result.c_str()); fprintf(stderr, "%lu nodes, node size %lu, %lu bytes\n", this->parser->nodes.size(), sizeof(parse_node_t), this->parser->nodes.size() * sizeof(parse_node_t)); - + if (output != NULL) { output->swap(this->parser->nodes); this->parser->nodes.clear(); } - + if (errors != NULL) { errors->swap(this->parser->errors); this->parser->errors.clear(); } - + return ! this->parser->fatal_errored; } diff --git a/parse_tree.h b/parse_tree.h index 39e370af..6b1fc0d1 100644 --- a/parse_tree.h +++ b/parse_tree.h @@ -26,11 +26,11 @@ struct parse_error_t { /** Text of the error */ wcstring text; - + /** Offset and length of the token in the source code that triggered this error */ size_t source_start; size_t source_length; - + /** Return a string describing the error, suitable for presentation to the user */ wcstring describe(const wcstring &src) const; }; @@ -40,8 +40,8 @@ class parse_ll_t; class parse_t { parse_ll_t * const parser; - - public: + +public: parse_t(); bool parse(const wcstring &str, parse_node_tree_t *output, parse_error_list_t *errors); }; @@ -49,7 +49,7 @@ class parse_t enum parse_token_type_t { token_type_invalid, - + // Non-terminal tokens symbol_job_list, symbol_job, @@ -61,33 +61,35 @@ enum parse_token_type_t symbol_while_header, symbol_begin_header, symbol_function_header, - + symbol_if_statement, symbol_if_clause, symbol_else_clause, symbol_else_continuation, - + symbol_switch_statement, symbol_case_item_list, symbol_case_item, - + symbol_boolean_statement, symbol_decorated_statement, symbol_plain_statement, symbol_arguments_or_redirections_list, symbol_argument_or_redirection, - + symbol_argument_list_nonempty, symbol_argument_list, + + symbol_optional_background, // Terminal types parse_token_type_string, parse_token_type_pipe, parse_token_type_redirection, - parse_token_background, + parse_token_type_background, parse_token_type_end, parse_token_type_terminate, - + FIRST_PARSE_TOKEN_TYPE = parse_token_type_string }; @@ -117,32 +119,32 @@ wcstring keyword_description(parse_keyword_t type); /** Base class for nodes of a parse tree */ class parse_node_t { - public: - +public: + /* Type of the node */ enum parse_token_type_t type; - + /* Start in the source code */ size_t source_start; - + /* Length of our range in the source code */ size_t source_length; /* Children */ node_offset_t child_start; node_offset_t child_count; - + /* Type-dependent data */ uint32_t tag; - + /* Description */ wcstring describe(void) const; - + /* Constructor */ explicit parse_node_t(parse_token_type_t ty) : type(ty), source_start(0), source_length(0), child_start(0), child_count(0), tag(0) { } - + node_offset_t child_offset(node_offset_t which) const { PARSE_ASSERT(which < child_count); @@ -154,360 +156,6 @@ class parse_node_tree_t : public std::vector { }; -namespace parse_symbols -{ - - #define SYMBOL(x) static inline parse_token_type_t get_token() { return x; } - - #define PRODUCE(X) static int production(parse_token_type_t tok, parse_keyword_t key) { return X; } - - #define NO_PRODUCTION (-1) - - - template - struct Token - { - SYMBOL(WHICH); - - typedef Token t0; - typedef Token t1; - typedef Token t2; - typedef Token t3; - typedef Token t4; - typedef Token t5; - }; - - /* Placeholder */ - typedef Token none; - - struct EMPTY - { - typedef none t0; - typedef none t1; - typedef none t2; - typedef none t3; - typedef none t4; - typedef none t5; - }; - - template - struct Seq - { - typedef T0 t0; - typedef T1 t1; - typedef T2 t2; - typedef T3 t3; - typedef T4 t4; - typedef T5 t5; - }; - - template - struct OR - { - typedef P0 p0; - typedef P1 p1; - typedef P2 p2; - typedef P3 p3; - typedef P4 p4; - typedef P5 p5; - }; - - template - struct Keyword - { - static inline parse_keyword_t get_token() { return WHICH; } - }; - - struct job; - struct statement; - struct job_continuation; - struct boolean_statement; - struct block_statement; - struct if_statement; - struct if_clause; - struct else_clause; - struct else_continuation; - struct switch_statement; - struct decorated_statement; - struct switch_statement; - struct case_item_list; - struct case_item; - struct argument_list_nonempty; - struct argument_list; - struct block_statement; - struct block_header; - struct for_header; - struct while_header; - struct begin_header; - struct function_header; - struct boolean_statement; - struct decorated_statement; - struct plain_statement; - struct arguments_or_redirections_list; - struct argument_or_redirection; - struct redirection; - struct statement_terminator; - - /* A job_list is a list of jobs, separated by semicolons or newlines */ - struct job_list : OR< - EMPTY, - Seq, - Seq, job_list> - > - { - SYMBOL(symbol_job_list) - }; - - /* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */ - struct job : Seq - { - SYMBOL(symbol_job); - }; - - struct job_continuation : OR< - EMPTY, - Seq, statement, job_continuation> - > - { - SYMBOL(symbol_job_continuation); - }; - - /* A statement is a normal command, or an if / while / and etc */ - struct statement : OR< - boolean_statement, - block_statement, - if_statement, - switch_statement, - decorated_statement - > - { - SYMBOL(symbol_statement); - }; - - struct if_statement : Seq > - { - SYMBOL(symbol_if_statement); - PRODUCE(0) - }; - - struct if_clause : Seq, job, statement_terminator, job_list> - { - SYMBOL(symbol_if_clause); - PRODUCE(0) - }; - - struct else_clause : OR< - EMPTY, - Keyword, else_continuation - > - { - SYMBOL(symbol_else_clause); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_else: return 1; - default: return 0; - } - } - }; - - struct else_continuation : OR< - Seq, - Seq - > - { - SYMBOL(symbol_else_continuation); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_if: return 0; - default: return 1; - } - } - }; - - struct switch_statement : Seq, Token, statement_terminator, case_item_list, Keyword - > - { - SYMBOL(symbol_switch_statement); - }; - - struct case_item_list : OR - < - EMPTY, - case_item, case_item_list - > - { - SYMBOL(symbol_case_item_list); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_case: return 1; - default: return 0; - } - } - }; - - struct case_item : Seq, argument_list, statement_terminator, job_list> - { - SYMBOL(symbol_case_item); - }; - - struct argument_list_nonempty : Seq, argument_list> - { - SYMBOL(symbol_argument_list_nonempty); - }; - - struct argument_list : OR - { - SYMBOL(symbol_argument_list); - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_string: return 1; - default: return 0; - } - } - }; - - struct block_statement : Seq, arguments_or_redirections_list> - { - SYMBOL(symbol_block_statement); - PRODUCE(0) - }; - - struct block_header : OR - { - SYMBOL(symbol_block_header); - }; - - struct for_header : Seq, Token, Keyword, arguments_or_redirections_list> - { - SYMBOL(symbol_for_header); - }; - - struct while_header : Seq, statement> - { - SYMBOL(symbol_while_header); - }; - - struct begin_header : Keyword - { - SYMBOL(symbol_begin_header); - }; - - struct function_header : Keyword - { - SYMBOL(symbol_function_header); - }; - - /* A boolean statement is AND or OR or NOT */ - struct boolean_statement : OR< - Seq, statement>, - Seq, statement>, - Seq, statement> - > - { - SYMBOL(symbol_boolean_statement); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_and: return 0; - case parse_keyword_or: return 1; - case parse_keyword_not: return 2; - default: return NO_PRODUCTION; - } - } - }; - - /* A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" */ - struct decorated_statement : OR< - Seq, plain_statement>, - Seq, plain_statement>, - plain_statement - > - { - SYMBOL(symbol_decorated_statement); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_command: return 0; - case parse_keyword_builtin: return 1; - default: return 2; - } - } - }; - - struct plain_statement : Seq, arguments_or_redirections_list> - { - SYMBOL(symbol_plain_statement); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - return 0; - } - - }; - - struct arguments_or_redirections_list : OR< - EMPTY, - Seq > - { - SYMBOL(symbol_arguments_or_redirections_list); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_string: - case parse_token_type_redirection: - return 1; - default: - return 0; - } - } - }; - - struct argument_or_redirection : OR< - Token, - redirection - > - { - SYMBOL(symbol_argument_or_redirection); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_string: return 0; - case parse_token_type_redirection: return 1; - default: return NO_PRODUCTION; - } - } - }; - - struct redirection : Token - { - SYMBOL(parse_token_type_redirection); - }; - - struct statement_terminator : Token - { - SYMBOL(parse_token_type_end); - }; -} - /* Fish grammar: @@ -520,45 +168,45 @@ namespace parse_symbols # A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation job = statement job_continuation - job_continuation = | + job_continuation = | statement job_continuation # A statement is a normal command, or an if / while / and etc statement = boolean_statement | block_statement | if_statement | switch_statement | decorated_statement - + # A block is a conditional, loop, or begin/end - if_statement = if_clause else_clause + if_statement = if_clause else_clause arguments_or_redirections_list if_clause = job STATEMENT_TERMINATOR job_list else_clause = | else_continuation else_continuation = if_clause else_clause | STATEMENT_TERMINATOR job_list - + switch_statement = SWITCH STATEMENT_TERMINATOR case_item_list case_item_list = | case_item case_item_list case_item = CASE argument_list STATEMENT_TERMINATOR job_list - + argument_list_nonempty = argument_list argument_list = | argument_list_nonempty - block_statement = block_header STATEMENT_TERMINATOR job_list arguments_or_redirections_list + block_statement = block_header job_list arguments_or_redirections_list block_header = for_header | while_header | function_header | begin_header for_header = FOR var_name IN arguments_or_redirections_list while_header = WHILE statement begin_header = BEGIN function_header = FUNCTION function_name argument_list - + # A boolean statement is AND or OR or NOT boolean_statement = AND statement | OR statement | NOT statement - + # A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" decorated_statement = COMMAND plain_statement | BUILTIN plain_statement | plain_statement - plain_statement = COMMAND arguments_or_redirections_list + plain_statement = COMMAND arguments_or_redirections_list optional_background arguments_or_redirections_list = | argument_or_redirection arguments_or_redirections_list @@ -567,6 +215,8 @@ namespace parse_symbols terminator = | + optional_background = | + */ #endif diff --git a/parse_tree_construction.h b/parse_tree_construction.h new file mode 100644 index 00000000..fb9e8dfb --- /dev/null +++ b/parse_tree_construction.h @@ -0,0 +1,586 @@ +/**\file parse_tree.h + + Programmatic representation of fish code. +*/ + +#ifndef FISH_PARSE_TREE_CONSTRUCTION_H +#define FISH_PARSE_TREE_CONSTRUCTION_H + +#include "parse_tree.h" + +/* Terrifying template black magic. */ + +namespace parse_symbols +{ + +#define SYMBOL(x) static inline parse_token_type_t get_token() { return x; } + +#define PRODUCE(X) static int production(parse_token_type_t tok, parse_keyword_t key) { return X; } + +#define NO_PRODUCTION (-1) + +struct Symbol +{ + typedef int magic_symbol_type_t; +}; + +template +struct Token : public Symbol +{ + SYMBOL(WHICH); +}; + +/* Placeholder */ +typedef Token none; + +typedef Token EMPTY; + +template +struct Seq +{ + typedef T0 t0; + typedef T1 t1; + typedef T2 t2; + typedef T3 t3; + typedef T4 t4; + typedef T5 t5; + + typedef int magic_seq_type_t; +}; + +template +struct OR +{ + typedef P0 p0; + typedef P1 p1; + typedef P2 p2; + typedef P3 p3; + typedef P4 p4; + typedef P5 p5; + + typedef int magic_or_type_t; +}; + +template +struct Keyword : public Symbol +{ + static inline parse_keyword_t get_token() + { + return WHICH; + } +}; + +struct job; +struct statement; +struct job_continuation; +struct boolean_statement; +struct block_statement; +struct if_statement; +struct if_clause; +struct else_clause; +struct else_continuation; +struct switch_statement; +struct decorated_statement; +struct switch_statement; +struct case_item_list; +struct case_item; +struct argument_list_nonempty; +struct argument_list; +struct block_statement; +struct block_header; +struct for_header; +struct while_header; +struct begin_header; +struct function_header; +struct boolean_statement; +struct decorated_statement; +struct plain_statement; +struct arguments_or_redirections_list; +struct argument_or_redirection; +struct redirection; +struct statement_terminator; +struct optional_background; + +/* A job_list is a list of jobs, separated by semicolons or newlines */ +struct job_list : public Symbol +{ + typedef OR< + EMPTY, + Seq, + Seq, job_list> + > productions; + + SYMBOL(symbol_job_list) + + static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_string: + // 'end' is special + switch (token_keyword) + { + case parse_keyword_end: + case parse_keyword_else: + // End this job list + return 0; + + default: + // Normal string + return 1; + } + + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_type_background: + return 1; + + case parse_token_type_end: + // Empty line + return 2; + + case parse_token_type_terminate: + // no more commands, just transition to empty + return 0; + break; + + default: + return NO_PRODUCTION; + } + } + +}; + +/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */ +struct job : public Symbol +{ + typedef Seq sole_production; + SYMBOL(symbol_job); +}; + +struct job_continuation : public Symbol +{ + typedef OR< + EMPTY, + Seq, statement, job_continuation> + > productions; + + SYMBOL(symbol_job_continuation); + + static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_pipe: + // Pipe, continuation + return 1; + + default: + // Not a pipe, no job continuation + return 0; + } + + } +}; + +/* A statement is a normal command, or an if / while / and etc */ +struct statement : public Symbol +{ + typedef OR< + boolean_statement, + block_statement, + if_statement, + switch_statement, + decorated_statement + > productions; + + SYMBOL(symbol_statement); + + static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_string: + switch (token_keyword) + { + case parse_keyword_and: + case parse_keyword_or: + case parse_keyword_not: + return 0; + + case parse_keyword_for: + case parse_keyword_while: + case parse_keyword_function: + case parse_keyword_begin: + return 1; + + case parse_keyword_if: + return 2; + + case parse_keyword_else: + //symbol_stack_pop(); + return NO_PRODUCTION; + + case parse_keyword_switch: + return 3; + + case parse_keyword_end: + PARSER_DIE(); //todo + return NO_PRODUCTION; + + // 'in' is only special within a for_header + case parse_keyword_in: + case parse_keyword_none: + case parse_keyword_command: + case parse_keyword_builtin: + case parse_keyword_case: + return 4; + } + break; + + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_type_background: + case parse_token_type_terminate: + return NO_PRODUCTION; + //parse_error(L"statement", token); + + default: + return NO_PRODUCTION; + } + } + +}; + +struct if_statement : public Symbol +{ + typedef Seq, arguments_or_redirections_list> sole_production; + SYMBOL(symbol_if_statement); +}; + +struct if_clause : public Symbol +{ + typedef Seq, job, statement_terminator, job_list> sole_production; + SYMBOL(symbol_if_clause); +}; + +struct else_clause : public Symbol +{ + typedef OR< + EMPTY, + Seq, else_continuation> + > productions; + + SYMBOL(symbol_else_clause); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_else: + return 1; + default: + return 0; + } + } +}; + +struct else_continuation : public Symbol +{ + typedef OR< + Seq, + Seq + > productions; + + SYMBOL(symbol_else_continuation); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_if: + return 0; + default: + return 1; + } + } +}; + +struct switch_statement : public Symbol +{ + typedef Seq, + Token, + statement_terminator, + case_item_list, + Keyword + > sole_production; + + SYMBOL(symbol_switch_statement); +}; + +struct case_item_list : public Symbol +{ + typedef OR + < + EMPTY, + Seq, + Seq, case_item_list> + > productions; + + SYMBOL(symbol_case_item_list); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_case: return 1; + + default: + if (tok == parse_token_type_end) + { + /* empty line */ + return 2; + } + else + { + return 0; + } + + } + } +}; + +struct case_item : public Symbol +{ + typedef Seq, argument_list, statement_terminator, job_list> sole_production; + + SYMBOL(symbol_case_item); +}; + +struct argument_list_nonempty : public Symbol +{ + typedef Seq, argument_list> sole_production; + SYMBOL(symbol_argument_list_nonempty); +}; + +struct argument_list : public Symbol +{ + typedef OR productions; + + SYMBOL(symbol_argument_list); + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_string: + return 1; + default: + return 0; + } + } +}; + +struct block_statement : public Symbol +{ + typedef Seq, arguments_or_redirections_list> sole_production; + + SYMBOL(symbol_block_statement); +}; + +struct block_header : public Symbol +{ + typedef OR productions; + + SYMBOL(symbol_block_header); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + // todo + case parse_keyword_else: + return NO_PRODUCTION; + case parse_keyword_for: + return 0; + case parse_keyword_while: + return 1; + case parse_keyword_function: + return 2; + case parse_keyword_begin: + return 3; + default: + return NO_PRODUCTION; + } + } +}; + +struct for_header : public Symbol +{ + typedef Seq, Token, Keyword, arguments_or_redirections_list> sole_production; + + SYMBOL(symbol_for_header); +}; + +struct while_header : public Symbol +{ + typedef Seq, statement> sole_production; + + SYMBOL(symbol_while_header); +}; + +struct begin_header : public Symbol +{ + typedef Keyword sole_production; + SYMBOL(symbol_begin_header); +}; + +struct function_header : public Symbol +{ + typedef Seq< Keyword, Token, argument_list> sole_production; + SYMBOL(symbol_function_header); +}; + +/* A boolean statement is AND or OR or NOT */ +struct boolean_statement : public Symbol +{ + typedef OR< + Seq, statement>, + Seq, statement>, + Seq, statement> + > productions; + + SYMBOL(symbol_boolean_statement); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_and: + return 0; + case parse_keyword_or: + return 1; + case parse_keyword_not: + return 2; + default: + return NO_PRODUCTION; + } + } +}; + +/* A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" */ +struct decorated_statement : public Symbol +{ + + typedef OR< + Seq, plain_statement>, + Seq, plain_statement>, + plain_statement + > productions; + + SYMBOL(symbol_decorated_statement); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_command: + return 0; + case parse_keyword_builtin: + return 1; + default: + return 2; + } + } +}; + +struct plain_statement : public Symbol +{ + + typedef Seq, arguments_or_redirections_list, optional_background> sole_production; + + SYMBOL(symbol_plain_statement); + +}; + +struct arguments_or_redirections_list : public Symbol +{ + typedef OR< + EMPTY, + Seq > + productions; + + SYMBOL(symbol_arguments_or_redirections_list); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_string: + case parse_token_type_redirection: + return 1; + default: + return 0; + } + } +}; + +struct argument_or_redirection : public Symbol +{ + typedef OR< + Token, + redirection + > productions; + + + SYMBOL(symbol_argument_or_redirection); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_string: + return 0; + case parse_token_type_redirection: + return 1; + default: + return NO_PRODUCTION; + } + } +}; + +struct redirection : public Symbol +{ + typedef Token production; + SYMBOL(parse_token_type_redirection); +}; + +struct statement_terminator : public Symbol +{ + typedef Token production; + SYMBOL(parse_token_type_end); +}; + +struct optional_background : public Symbol +{ + typedef OR< + EMPTY, + Token + > productions; + + SYMBOL(symbol_optional_background); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_background: + return 1; + default: + return 0; + } + } +}; + +} + +#endif diff --git a/parse_util.cpp b/parse_util.cpp index 5e6f4459..6f291345 100644 --- a/parse_util.cpp +++ b/parse_util.cpp @@ -245,10 +245,10 @@ void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wc const wchar_t * const cursor = buff + cursor_pos; CHECK(buff,); - + const size_t bufflen = wcslen(buff); assert(cursor_pos <= bufflen); - + /* ap and bp are the beginning and end of the tightest command substitition found so far */ const wchar_t *ap = buff, *bp = buff + bufflen; const wchar_t *pos = buff; @@ -260,13 +260,13 @@ void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wc /* No subshell found, all done */ break; } - + /* Intrepret NULL to mean the end */ if (end == NULL) { end = const_cast(buff) + bufflen; } - + if (begin < cursor && end >= cursor) { /* This command substitution surrounds the cursor, so it's a tighter fit */ @@ -288,7 +288,7 @@ void parse_util_cmdsubst_extent(const wchar_t *buff, size_t cursor_pos, const wc assert(pos <= buff + bufflen); } } - + if (a != NULL) *a = ap; if (b != NULL) *b = bp; } diff --git a/tokenizer.cpp b/tokenizer.cpp index 831197ee..6d99b46c 100644 --- a/tokenizer.cpp +++ b/tokenizer.cpp @@ -651,13 +651,19 @@ wcstring tok_first(const wchar_t *str) return result; } -int tok_get_pos(tokenizer_t *tok) +int tok_get_pos(const tokenizer_t *tok) { CHECK(tok, 0); - return (int)tok->last_pos; } +size_t tok_get_extent(const tokenizer_t *tok) +{ + CHECK(tok, 0); + size_t current_pos = tok->buff - tok->orig_buff; + return current_pos > tok->last_pos ? current_pos - tok->last_pos : 0; +} + void tok_set_pos(tokenizer_t *tok, int pos) { diff --git a/tokenizer.h b/tokenizer.h index f2d6c0c0..0f3ff369 100644 --- a/tokenizer.h +++ b/tokenizer.h @@ -142,7 +142,10 @@ int tok_has_next(tokenizer_t *tok); /** Returns the position of the beginning of the current token in the original string */ -int tok_get_pos(tokenizer_t *tok); +int tok_get_pos(const tokenizer_t *tok); + +/** Returns the extent of the current token */ +size_t tok_get_extent(const tokenizer_t *tok); /** Returns the original string to tokenizer -- cgit v1.2.3 From 4f8d4f378cffa71b4e80bfa2049e2152b429615c Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Thu, 25 Jul 2013 15:24:22 -0700 Subject: AST no templates --- fish.xcodeproj/project.pbxproj | 8 +- parse_productions.cpp | 63 ++++ parse_productions.h | 647 +++++++++++++++++++++++++++++++++++++++++ parse_tree.cpp | 2 +- parse_tree.h | 4 +- parse_tree_construction.h | 586 ------------------------------------- 6 files changed, 719 insertions(+), 591 deletions(-) create mode 100644 parse_productions.cpp create mode 100644 parse_productions.h delete mode 100644 parse_tree_construction.h (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 5ae10bfc..708e65e3 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -152,6 +152,7 @@ D0F019FD15A977CA0034B3B1 /* config.fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0C4FD9415A7D7EE00212EF1 /* config.fish */; }; D0F01A0315A978910034B3B1 /* osx_fish_launcher.m in Sources */ = {isa = PBXBuildFile; fileRef = D0D02AFA159871B2008E62BD /* osx_fish_launcher.m */; }; D0F01A0515A978A10034B3B1 /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0CBD583159EEE010024809C /* Foundation.framework */; }; + D0FE8EE8179FB760008C9F21 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -465,7 +466,8 @@ D0D2693C159835CA005D9B9C /* fish */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish; sourceTree = BUILT_PRODUCTS_DIR; }; D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_test.cpp; sourceTree = ""; }; D0F5E28415A7A32D00315DFF /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = ""; }; - D0FE8EE6179CA8A5008C9F21 /* parse_tree_construction.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_tree_construction.h; sourceTree = ""; }; + D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_productions.h; sourceTree = ""; }; + D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_productions.cpp; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -589,8 +591,9 @@ D0A0853C13B3ACEE0099B651 /* exec.cpp */, D0A0850C13B3ACEE0099B651 /* expand.h */, D0A0853D13B3ACEE0099B651 /* expand.cpp */, + D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */, + D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */, D0C52F361765284C00BFAB82 /* parse_tree.h */, - D0FE8EE6179CA8A5008C9F21 /* parse_tree_construction.h */, D0C52F351765284C00BFAB82 /* parse_tree.cpp */, D0C52F341765281F00BFAB82 /* parse_exec.h */, D0C52F331765281F00BFAB82 /* parse_exec.cpp */, @@ -1116,6 +1119,7 @@ D0D02A89159839DF008E62BD /* fish.cpp in Sources */, D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */, D0C52F381765720600BFAB82 /* parse_exec.cpp in Sources */, + D0FE8EE8179FB760008C9F21 /* parse_productions.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/parse_productions.cpp b/parse_productions.cpp new file mode 100644 index 00000000..82bdd0b9 --- /dev/null +++ b/parse_productions.cpp @@ -0,0 +1,63 @@ +#include "parse_productions.h" + +using namespace parse_productions; + +#define PRODUCTIONS(sym) static const Production_t sym##_productions + +PRODUCTIONS(job_list) = + { + {}, + {symbol_job, symbol_job_list}, + {parse_token_type_end, symbol_job_list} + }; + + + +/* A job_list is a list of jobs, separated by semicolons or newlines */ + +DEC(job_list) { + symbol_job_list, + { + {}, + {symbol_job, symbol_job_list}, + {parse_token_type_end, symbol_job_list} + }, + resolve_job_list +}; + +static int resolve_job_list(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_string: + // 'end' is special + switch (token_keyword) + { + case parse_keyword_end: + case parse_keyword_else: + // End this job list + return 0; + + default: + // Normal string + return 1; + } + + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_type_background: + return 1; + + case parse_token_type_end: + // Empty line + return 2; + + case parse_token_type_terminate: + // no more commands, just transition to empty + return 0; + break; + + default: + return NO_PRODUCTION; + } + } \ No newline at end of file diff --git a/parse_productions.h b/parse_productions.h new file mode 100644 index 00000000..5ded6af0 --- /dev/null +++ b/parse_productions.h @@ -0,0 +1,647 @@ +/**\file parse_tree.h + + Programmatic representation of fish code. +*/ + +#ifndef FISH_PARSE_TREE_CONSTRUCTION_H +#define FISH_PARSE_TREE_CONSTRUCTION_H + +#include "parse_tree.h" + +/* Terrifying template black magic. */ + +/* + +- Get info for symbol +- Resolve production from info +- Get productions for children +- Get symbols for productions + +Production may be: + +1. Single value +2. Sequence of values (possibly empty) +3. Options of Single / Sequence + +Info to specify: + +1. Number of different productions +2. Resolver function +3. Symbols for associated productions + +Choice: should info be a class or a data? + +data: + +struct Symbol_t +{ + enum parse_token_type_t token_type; + int (*resolver)(parse_token_type_t tok, parse_keyword_t key); //may be trivial + production productions[5]; +} + +struct Production_t +{ + enum parse_token_type_t symbols[5]; +} + +*/ + +namespace parse_productions +{ + +#define MAX_PRODUCTIONS 5 +#define MAX_SYMBOLS_PER_PRODUCTION 5 + + + +struct Production_t +{ + enum parse_token_type_t symbols[MAX_SYMBOLS_PER_PRODUCTION]; +}; + +struct Symbol_t +{ + enum parse_token_type_t token_type; + int (*resolver)(parse_token_type_t tok, parse_keyword_t key); + Production_t productions[MAX_PRODUCTIONS]; +}; + + + +} + +namespace parse_symbols +{ + +#define SYMBOL(x) static inline parse_token_type_t get_token() { return x; } + +#define PRODUCE(X) static int production(parse_token_type_t tok, parse_keyword_t key) { return X; } + +#define NO_PRODUCTION (-1) + +struct Symbol +{ + typedef int magic_symbol_type_t; +}; + +template +struct Token : public Symbol +{ + SYMBOL(WHICH); +}; + +/* Placeholder */ +typedef Token none; + +typedef Token EMPTY; + +template +struct Seq +{ + typedef T0 t0; + typedef T1 t1; + typedef T2 t2; + typedef T3 t3; + typedef T4 t4; + typedef T5 t5; + + typedef int magic_seq_type_t; +}; + +template +struct OR +{ + typedef P0 p0; + typedef P1 p1; + typedef P2 p2; + typedef P3 p3; + typedef P4 p4; + typedef P5 p5; + + typedef int magic_or_type_t; +}; + +template +struct Keyword : public Symbol +{ + static inline parse_keyword_t get_token() + { + return WHICH; + } +}; + +struct job; +struct statement; +struct job_continuation; +struct boolean_statement; +struct block_statement; +struct if_statement; +struct if_clause; +struct else_clause; +struct else_continuation; +struct switch_statement; +struct decorated_statement; +struct switch_statement; +struct case_item_list; +struct case_item; +struct argument_list_nonempty; +struct argument_list; +struct block_statement; +struct block_header; +struct for_header; +struct while_header; +struct begin_header; +struct function_header; +struct boolean_statement; +struct decorated_statement; +struct plain_statement; +struct arguments_or_redirections_list; +struct argument_or_redirection; +struct redirection; +struct statement_terminator; +struct optional_background; + +/* A job_list is a list of jobs, separated by semicolons or newlines */ +struct job_list : public Symbol +{ + typedef OR< + EMPTY, + Seq, + Seq, job_list> + > productions; + + SYMBOL(symbol_job_list) + + static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_string: + // 'end' is special + switch (token_keyword) + { + case parse_keyword_end: + case parse_keyword_else: + // End this job list + return 0; + + default: + // Normal string + return 1; + } + + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_type_background: + return 1; + + case parse_token_type_end: + // Empty line + return 2; + + case parse_token_type_terminate: + // no more commands, just transition to empty + return 0; + break; + + default: + return NO_PRODUCTION; + } + } + +}; + +/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */ +struct job : public Symbol +{ + typedef Seq sole_production; + SYMBOL(symbol_job); +}; + +struct job_continuation : public Symbol +{ + typedef OR< + EMPTY, + Seq, statement, job_continuation> + > productions; + + SYMBOL(symbol_job_continuation); + + static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_pipe: + // Pipe, continuation + return 1; + + default: + // Not a pipe, no job continuation + return 0; + } + + } +}; + +/* A statement is a normal command, or an if / while / and etc */ +struct statement : public Symbol +{ + typedef OR< + boolean_statement, + block_statement, + if_statement, + switch_statement, + decorated_statement + > productions; + + SYMBOL(symbol_statement); + + static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) + { + switch (token_type) + { + case parse_token_type_string: + switch (token_keyword) + { + case parse_keyword_and: + case parse_keyword_or: + case parse_keyword_not: + return 0; + + case parse_keyword_for: + case parse_keyword_while: + case parse_keyword_function: + case parse_keyword_begin: + return 1; + + case parse_keyword_if: + return 2; + + case parse_keyword_else: + //symbol_stack_pop(); + return NO_PRODUCTION; + + case parse_keyword_switch: + return 3; + + case parse_keyword_end: + PARSER_DIE(); //todo + return NO_PRODUCTION; + + // 'in' is only special within a for_header + case parse_keyword_in: + case parse_keyword_none: + case parse_keyword_command: + case parse_keyword_builtin: + case parse_keyword_case: + return 4; + } + break; + + case parse_token_type_pipe: + case parse_token_type_redirection: + case parse_token_type_background: + case parse_token_type_terminate: + return NO_PRODUCTION; + //parse_error(L"statement", token); + + default: + return NO_PRODUCTION; + } + } + +}; + +struct if_statement : public Symbol +{ + typedef Seq, arguments_or_redirections_list> sole_production; + SYMBOL(symbol_if_statement); +}; + +struct if_clause : public Symbol +{ + typedef Seq, job, statement_terminator, job_list> sole_production; + SYMBOL(symbol_if_clause); +}; + +struct else_clause : public Symbol +{ + typedef OR< + EMPTY, + Seq, else_continuation> + > productions; + + SYMBOL(symbol_else_clause); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_else: + return 1; + default: + return 0; + } + } +}; + +struct else_continuation : public Symbol +{ + typedef OR< + Seq, + Seq + > productions; + + SYMBOL(symbol_else_continuation); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_if: + return 0; + default: + return 1; + } + } +}; + +struct switch_statement : public Symbol +{ + typedef Seq, + Token, + statement_terminator, + case_item_list, + Keyword + > sole_production; + + SYMBOL(symbol_switch_statement); +}; + +struct case_item_list : public Symbol +{ + typedef OR + < + EMPTY, + Seq, + Seq, case_item_list> + > productions; + + SYMBOL(symbol_case_item_list); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_case: return 1; + + default: + if (tok == parse_token_type_end) + { + /* empty line */ + return 2; + } + else + { + return 0; + } + + } + } +}; + +struct case_item : public Symbol +{ + typedef Seq, argument_list, statement_terminator, job_list> sole_production; + + SYMBOL(symbol_case_item); +}; + +struct argument_list_nonempty : public Symbol +{ + typedef Seq, argument_list> sole_production; + SYMBOL(symbol_argument_list_nonempty); +}; + +struct argument_list : public Symbol +{ + typedef OR productions; + + SYMBOL(symbol_argument_list); + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_string: + return 1; + default: + return 0; + } + } +}; + +struct block_statement : public Symbol +{ + typedef Seq, arguments_or_redirections_list> sole_production; + + SYMBOL(symbol_block_statement); +}; + +struct block_header : public Symbol +{ + typedef OR productions; + + SYMBOL(symbol_block_header); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + // todo + case parse_keyword_else: + return NO_PRODUCTION; + case parse_keyword_for: + return 0; + case parse_keyword_while: + return 1; + case parse_keyword_function: + return 2; + case parse_keyword_begin: + return 3; + default: + return NO_PRODUCTION; + } + } +}; + +struct for_header : public Symbol +{ + typedef Seq, Token, Keyword, arguments_or_redirections_list> sole_production; + + SYMBOL(symbol_for_header); +}; + +struct while_header : public Symbol +{ + typedef Seq, statement> sole_production; + + SYMBOL(symbol_while_header); +}; + +struct begin_header : public Symbol +{ + typedef Keyword sole_production; + SYMBOL(symbol_begin_header); +}; + +struct function_header : public Symbol +{ + typedef Seq< Keyword, Token, argument_list> sole_production; + SYMBOL(symbol_function_header); +}; + +/* A boolean statement is AND or OR or NOT */ +struct boolean_statement : public Symbol +{ + typedef OR< + Seq, statement>, + Seq, statement>, + Seq, statement> + > productions; + + SYMBOL(symbol_boolean_statement); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_and: + return 0; + case parse_keyword_or: + return 1; + case parse_keyword_not: + return 2; + default: + return NO_PRODUCTION; + } + } +}; + +/* A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" */ +struct decorated_statement : public Symbol +{ + + typedef OR< + Seq, plain_statement>, + Seq, plain_statement>, + plain_statement + > productions; + + SYMBOL(symbol_decorated_statement); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (key) + { + case parse_keyword_command: + return 0; + case parse_keyword_builtin: + return 1; + default: + return 2; + } + } +}; + +struct plain_statement : public Symbol +{ + + typedef Seq, arguments_or_redirections_list, optional_background> sole_production; + + SYMBOL(symbol_plain_statement); + +}; + +struct arguments_or_redirections_list : public Symbol +{ + typedef OR< + EMPTY, + Seq > + productions; + + SYMBOL(symbol_arguments_or_redirections_list); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_string: + case parse_token_type_redirection: + return 1; + default: + return 0; + } + } +}; + +struct argument_or_redirection : public Symbol +{ + typedef OR< + Token, + redirection + > productions; + + + SYMBOL(symbol_argument_or_redirection); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_string: + return 0; + case parse_token_type_redirection: + return 1; + default: + return NO_PRODUCTION; + } + } +}; + +struct redirection : public Symbol +{ + typedef Token production; + SYMBOL(parse_token_type_redirection); +}; + +struct statement_terminator : public Symbol +{ + typedef Token production; + SYMBOL(parse_token_type_end); +}; + +struct optional_background : public Symbol +{ + typedef OR< + EMPTY, + Token + > productions; + + SYMBOL(symbol_optional_background); + + static int production(parse_token_type_t tok, parse_keyword_t key) + { + switch (tok) + { + case parse_token_type_background: + return 1; + default: + return 0; + } + } +}; + +} + +#endif diff --git a/parse_tree.cpp b/parse_tree.cpp index aea3e729..4df277d4 100644 --- a/parse_tree.cpp +++ b/parse_tree.cpp @@ -1,4 +1,4 @@ -#include "parse_tree_construction.h" +#include "parse_productions.h" #include "tokenizer.h" #include diff --git a/parse_tree.h b/parse_tree.h index 6b1fc0d1..dfe9f24e 100644 --- a/parse_tree.h +++ b/parse_tree.h @@ -3,8 +3,8 @@ Programmatic representation of fish code. */ -#ifndef FISH_PARSE_TREE_H -#define FISH_PARSE_TREE_H +#ifndef FISH_PARSE_PRODUCTIONS_H +#define FISH_PARSE_PRODUCTIONS_H #include diff --git a/parse_tree_construction.h b/parse_tree_construction.h deleted file mode 100644 index fb9e8dfb..00000000 --- a/parse_tree_construction.h +++ /dev/null @@ -1,586 +0,0 @@ -/**\file parse_tree.h - - Programmatic representation of fish code. -*/ - -#ifndef FISH_PARSE_TREE_CONSTRUCTION_H -#define FISH_PARSE_TREE_CONSTRUCTION_H - -#include "parse_tree.h" - -/* Terrifying template black magic. */ - -namespace parse_symbols -{ - -#define SYMBOL(x) static inline parse_token_type_t get_token() { return x; } - -#define PRODUCE(X) static int production(parse_token_type_t tok, parse_keyword_t key) { return X; } - -#define NO_PRODUCTION (-1) - -struct Symbol -{ - typedef int magic_symbol_type_t; -}; - -template -struct Token : public Symbol -{ - SYMBOL(WHICH); -}; - -/* Placeholder */ -typedef Token none; - -typedef Token EMPTY; - -template -struct Seq -{ - typedef T0 t0; - typedef T1 t1; - typedef T2 t2; - typedef T3 t3; - typedef T4 t4; - typedef T5 t5; - - typedef int magic_seq_type_t; -}; - -template -struct OR -{ - typedef P0 p0; - typedef P1 p1; - typedef P2 p2; - typedef P3 p3; - typedef P4 p4; - typedef P5 p5; - - typedef int magic_or_type_t; -}; - -template -struct Keyword : public Symbol -{ - static inline parse_keyword_t get_token() - { - return WHICH; - } -}; - -struct job; -struct statement; -struct job_continuation; -struct boolean_statement; -struct block_statement; -struct if_statement; -struct if_clause; -struct else_clause; -struct else_continuation; -struct switch_statement; -struct decorated_statement; -struct switch_statement; -struct case_item_list; -struct case_item; -struct argument_list_nonempty; -struct argument_list; -struct block_statement; -struct block_header; -struct for_header; -struct while_header; -struct begin_header; -struct function_header; -struct boolean_statement; -struct decorated_statement; -struct plain_statement; -struct arguments_or_redirections_list; -struct argument_or_redirection; -struct redirection; -struct statement_terminator; -struct optional_background; - -/* A job_list is a list of jobs, separated by semicolons or newlines */ -struct job_list : public Symbol -{ - typedef OR< - EMPTY, - Seq, - Seq, job_list> - > productions; - - SYMBOL(symbol_job_list) - - static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) - { - switch (token_type) - { - case parse_token_type_string: - // 'end' is special - switch (token_keyword) - { - case parse_keyword_end: - case parse_keyword_else: - // End this job list - return 0; - - default: - // Normal string - return 1; - } - - case parse_token_type_pipe: - case parse_token_type_redirection: - case parse_token_type_background: - return 1; - - case parse_token_type_end: - // Empty line - return 2; - - case parse_token_type_terminate: - // no more commands, just transition to empty - return 0; - break; - - default: - return NO_PRODUCTION; - } - } - -}; - -/* A job is a non-empty list of statements, separated by pipes. (Non-empty is useful for cases like if statements, where we require a command). To represent "non-empty", we require a statement, followed by a possibly empty job_continuation */ -struct job : public Symbol -{ - typedef Seq sole_production; - SYMBOL(symbol_job); -}; - -struct job_continuation : public Symbol -{ - typedef OR< - EMPTY, - Seq, statement, job_continuation> - > productions; - - SYMBOL(symbol_job_continuation); - - static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) - { - switch (token_type) - { - case parse_token_type_pipe: - // Pipe, continuation - return 1; - - default: - // Not a pipe, no job continuation - return 0; - } - - } -}; - -/* A statement is a normal command, or an if / while / and etc */ -struct statement : public Symbol -{ - typedef OR< - boolean_statement, - block_statement, - if_statement, - switch_statement, - decorated_statement - > productions; - - SYMBOL(symbol_statement); - - static int production(parse_token_type_t token_type, parse_keyword_t token_keyword) - { - switch (token_type) - { - case parse_token_type_string: - switch (token_keyword) - { - case parse_keyword_and: - case parse_keyword_or: - case parse_keyword_not: - return 0; - - case parse_keyword_for: - case parse_keyword_while: - case parse_keyword_function: - case parse_keyword_begin: - return 1; - - case parse_keyword_if: - return 2; - - case parse_keyword_else: - //symbol_stack_pop(); - return NO_PRODUCTION; - - case parse_keyword_switch: - return 3; - - case parse_keyword_end: - PARSER_DIE(); //todo - return NO_PRODUCTION; - - // 'in' is only special within a for_header - case parse_keyword_in: - case parse_keyword_none: - case parse_keyword_command: - case parse_keyword_builtin: - case parse_keyword_case: - return 4; - } - break; - - case parse_token_type_pipe: - case parse_token_type_redirection: - case parse_token_type_background: - case parse_token_type_terminate: - return NO_PRODUCTION; - //parse_error(L"statement", token); - - default: - return NO_PRODUCTION; - } - } - -}; - -struct if_statement : public Symbol -{ - typedef Seq, arguments_or_redirections_list> sole_production; - SYMBOL(symbol_if_statement); -}; - -struct if_clause : public Symbol -{ - typedef Seq, job, statement_terminator, job_list> sole_production; - SYMBOL(symbol_if_clause); -}; - -struct else_clause : public Symbol -{ - typedef OR< - EMPTY, - Seq, else_continuation> - > productions; - - SYMBOL(symbol_else_clause); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_else: - return 1; - default: - return 0; - } - } -}; - -struct else_continuation : public Symbol -{ - typedef OR< - Seq, - Seq - > productions; - - SYMBOL(symbol_else_continuation); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_if: - return 0; - default: - return 1; - } - } -}; - -struct switch_statement : public Symbol -{ - typedef Seq, - Token, - statement_terminator, - case_item_list, - Keyword - > sole_production; - - SYMBOL(symbol_switch_statement); -}; - -struct case_item_list : public Symbol -{ - typedef OR - < - EMPTY, - Seq, - Seq, case_item_list> - > productions; - - SYMBOL(symbol_case_item_list); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_case: return 1; - - default: - if (tok == parse_token_type_end) - { - /* empty line */ - return 2; - } - else - { - return 0; - } - - } - } -}; - -struct case_item : public Symbol -{ - typedef Seq, argument_list, statement_terminator, job_list> sole_production; - - SYMBOL(symbol_case_item); -}; - -struct argument_list_nonempty : public Symbol -{ - typedef Seq, argument_list> sole_production; - SYMBOL(symbol_argument_list_nonempty); -}; - -struct argument_list : public Symbol -{ - typedef OR productions; - - SYMBOL(symbol_argument_list); - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_string: - return 1; - default: - return 0; - } - } -}; - -struct block_statement : public Symbol -{ - typedef Seq, arguments_or_redirections_list> sole_production; - - SYMBOL(symbol_block_statement); -}; - -struct block_header : public Symbol -{ - typedef OR productions; - - SYMBOL(symbol_block_header); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - // todo - case parse_keyword_else: - return NO_PRODUCTION; - case parse_keyword_for: - return 0; - case parse_keyword_while: - return 1; - case parse_keyword_function: - return 2; - case parse_keyword_begin: - return 3; - default: - return NO_PRODUCTION; - } - } -}; - -struct for_header : public Symbol -{ - typedef Seq, Token, Keyword, arguments_or_redirections_list> sole_production; - - SYMBOL(symbol_for_header); -}; - -struct while_header : public Symbol -{ - typedef Seq, statement> sole_production; - - SYMBOL(symbol_while_header); -}; - -struct begin_header : public Symbol -{ - typedef Keyword sole_production; - SYMBOL(symbol_begin_header); -}; - -struct function_header : public Symbol -{ - typedef Seq< Keyword, Token, argument_list> sole_production; - SYMBOL(symbol_function_header); -}; - -/* A boolean statement is AND or OR or NOT */ -struct boolean_statement : public Symbol -{ - typedef OR< - Seq, statement>, - Seq, statement>, - Seq, statement> - > productions; - - SYMBOL(symbol_boolean_statement); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_and: - return 0; - case parse_keyword_or: - return 1; - case parse_keyword_not: - return 2; - default: - return NO_PRODUCTION; - } - } -}; - -/* A decorated_statement is a command with a list of arguments_or_redirections, possibly with "builtin" or "command" */ -struct decorated_statement : public Symbol -{ - - typedef OR< - Seq, plain_statement>, - Seq, plain_statement>, - plain_statement - > productions; - - SYMBOL(symbol_decorated_statement); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (key) - { - case parse_keyword_command: - return 0; - case parse_keyword_builtin: - return 1; - default: - return 2; - } - } -}; - -struct plain_statement : public Symbol -{ - - typedef Seq, arguments_or_redirections_list, optional_background> sole_production; - - SYMBOL(symbol_plain_statement); - -}; - -struct arguments_or_redirections_list : public Symbol -{ - typedef OR< - EMPTY, - Seq > - productions; - - SYMBOL(symbol_arguments_or_redirections_list); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_string: - case parse_token_type_redirection: - return 1; - default: - return 0; - } - } -}; - -struct argument_or_redirection : public Symbol -{ - typedef OR< - Token, - redirection - > productions; - - - SYMBOL(symbol_argument_or_redirection); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_string: - return 0; - case parse_token_type_redirection: - return 1; - default: - return NO_PRODUCTION; - } - } -}; - -struct redirection : public Symbol -{ - typedef Token production; - SYMBOL(parse_token_type_redirection); -}; - -struct statement_terminator : public Symbol -{ - typedef Token production; - SYMBOL(parse_token_type_end); -}; - -struct optional_background : public Symbol -{ - typedef OR< - EMPTY, - Token - > productions; - - SYMBOL(symbol_optional_background); - - static int production(parse_token_type_t tok, parse_keyword_t key) - { - switch (tok) - { - case parse_token_type_background: - return 1; - default: - return 0; - } - } -}; - -} - -#endif -- cgit v1.2.3 From 876b9d49b4ae89c02275ad7c3c2e26f27e38f298 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Thu, 8 Aug 2013 14:44:52 -0700 Subject: Added fish_tests target to fish.xcodeproj --- fish.xcodeproj/project.pbxproj | 215 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 708e65e3..c397f321 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -73,6 +73,48 @@ D07D266E15E33B86009E43F6 /* tools in Copy Files */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; }; D07D267215E34171009E43F6 /* config.fish in Copy Files */ = {isa = PBXBuildFile; fileRef = D0CBD580159EE48F0024809C /* config.fish */; }; D0879AC816BF9AAB00E98E56 /* fish_term_icon.icns in Resources */ = {isa = PBXBuildFile; fileRef = D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */; }; + D08A329417B4458D00F3A533 /* fish_tests.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D08A329317B4458D00F3A533 /* fish_tests.cpp */; }; + D08A329517B445C200F3A533 /* function.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854413B3ACEE0099B651 /* function.cpp */; }; + D08A329617B445FD00F3A533 /* builtin.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853513B3ACEE0099B651 /* builtin.cpp */; }; + D08A329717B4463B00F3A533 /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; }; + D08A329817B4463B00F3A533 /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; }; + D08A329917B4463B00F3A533 /* exec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853C13B3ACEE0099B651 /* exec.cpp */; }; + D08A329A17B4463B00F3A533 /* expand.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853D13B3ACEE0099B651 /* expand.cpp */; }; + D08A329B17B4463B00F3A533 /* highlight.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854713B3ACEE0099B651 /* highlight.cpp */; }; + D08A329C17B4463B00F3A533 /* history.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854813B3ACEE0099B651 /* history.cpp */; }; + D08A329D17B4463B00F3A533 /* kill.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854F13B3ACEE0099B651 /* kill.cpp */; }; + D08A329E17B4463B00F3A533 /* parser.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855413B3ACEE0099B651 /* parser.cpp */; }; + D08A329F17B4463B00F3A533 /* proc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855713B3ACEE0099B651 /* proc.cpp */; }; + D08A32A017B4463B00F3A533 /* reader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855813B3ACEE0099B651 /* reader.cpp */; }; + D08A32A117B4463B00F3A533 /* sanity.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855913B3ACEE0099B651 /* sanity.cpp */; }; + D08A32A217B4463B00F3A533 /* tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855D13B3ACEE0099B651 /* tokenizer.cpp */; }; + D08A32A317B4463B00F3A533 /* wgetopt.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855F13B3ACEE0099B651 /* wgetopt.cpp */; }; + D08A32A417B4463B00F3A533 /* wildcard.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856013B3ACEE0099B651 /* wildcard.cpp */; }; + D08A32A517B4463B00F3A533 /* wutil.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0856113B3ACEE0099B651 /* wutil.cpp */; }; + D08A32A617B4464300F3A533 /* input.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854A13B3ACEE0099B651 /* input.cpp */; }; + D08A32A717B446A300F3A533 /* autoload.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */; }; + D08A32A817B446A300F3A533 /* builtin_test.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */; }; + D08A32A917B446A300F3A533 /* color.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0B6B0FE14E88BA400AD6C10 /* color.cpp */; }; + D08A32AA17B446A300F3A533 /* common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853613B3ACEE0099B651 /* common.cpp */; }; + D08A32AB17B446A300F3A533 /* env_universal_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853813B3ACEE0099B651 /* env_universal_common.cpp */; }; + D08A32AC17B446A300F3A533 /* env_universal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853913B3ACEE0099B651 /* env_universal.cpp */; }; + D08A32AD17B446A300F3A533 /* event.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853B13B3ACEE0099B651 /* event.cpp */; }; + D08A32AE17B446A300F3A533 /* input_common.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854913B3ACEE0099B651 /* input_common.cpp */; }; + D08A32AF17B446A300F3A533 /* intern.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854B13B3ACEE0099B651 /* intern.cpp */; }; + D08A32B017B446A300F3A533 /* io.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854C13B3ACEE0099B651 /* io.cpp */; }; + D08A32B117B446A300F3A533 /* iothread.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0854D13B3ACEE0099B651 /* iothread.cpp */; }; + D08A32B217B446A300F3A533 /* output.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855113B3ACEE0099B651 /* output.cpp */; }; + D08A32B317B446A300F3A533 /* parse_util.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855213B3ACEE0099B651 /* parse_util.cpp */; }; + D08A32B417B446A300F3A533 /* parser_keywords.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855313B3ACEE0099B651 /* parser_keywords.cpp */; }; + D08A32B517B446A300F3A533 /* path.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855513B3ACEE0099B651 /* path.cpp */; }; + D08A32B617B446A300F3A533 /* postfork.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D09B1C1914FC7B5B00F91077 /* postfork.cpp */; }; + D08A32B717B446A300F3A533 /* screen.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855A13B3ACEE0099B651 /* screen.cpp */; }; + D08A32B817B446A300F3A533 /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; }; + D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; }; + D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; + D08A32BB17B446B100F3A533 /* parse_exec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F331765281F00BFAB82 /* parse_exec.cpp */; }; + D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; + D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; }; D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; @@ -295,6 +337,15 @@ name = "Copy Files"; runOnlyForDeploymentPostprocessing = 1; }; + D08A328B17B4455100F3A533 /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + ); + runOnlyForDeploymentPostprocessing = 1; + }; D0F019F015A977010034B3B1 /* CopyFiles */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; @@ -340,6 +391,8 @@ D07B247215BCC15700D4ADB4 /* add-shell */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = "add-shell"; path = "build_tools/osx_package_scripts/add-shell"; sourceTree = ""; }; D07B247515BCC4BE00D4ADB4 /* install.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = install.sh; path = osx/install.sh; sourceTree = ""; }; D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; name = fish_term_icon.icns; path = osx/fish_term_icon.icns; sourceTree = ""; }; + D08A328D17B4455100F3A533 /* fish_tests */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish_tests; sourceTree = BUILT_PRODUCTS_DIR; }; + D08A329317B4458D00F3A533 /* fish_tests.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = fish_tests.cpp; sourceTree = ""; }; D09B1C1914FC7B5B00F91077 /* postfork.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = postfork.cpp; sourceTree = ""; }; D09B1C1A14FC7B5B00F91077 /* postfork.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = postfork.h; sourceTree = ""; }; D0A0850313B3ACEE0099B651 /* builtin.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = builtin.h; sourceTree = ""; }; @@ -471,6 +524,15 @@ /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + D08A328A17B4455100F3A533 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */, + D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; D0D02AB915985EF9008E62BD /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -534,6 +596,13 @@ name = "Other Build Products"; sourceTree = ""; }; + D08A328E17B4455100F3A533 /* fish_tests */ = { + isa = PBXGroup; + children = ( + ); + path = fish_tests; + sourceTree = ""; + }; D0A084F013B3AC130099B651 = { isa = PBXGroup; children = ( @@ -543,6 +612,7 @@ D0D02A8E15983D5F008E62BD /* Libraries */, D0D02AAB15985C14008E62BD /* Resources */, D031890A15E36DB500D9CC39 /* Other Build Products */, + D08A328E17B4455100F3A533 /* fish_tests */, D0D2693215983562005D9B9C /* Products */, ); sourceTree = ""; @@ -672,6 +742,7 @@ D0A0856613B3ACEE0099B651 /* xdgmimemagic.cpp */, D0A0852F13B3ACEE0099B651 /* xdgmimeparent.h */, D0A0856713B3ACEE0099B651 /* xdgmimeparent.cpp */, + D08A329317B4458D00F3A533 /* fish_tests.cpp */, ); name = Sources; sourceTree = ""; @@ -713,6 +784,7 @@ D0D02ABC15985EF9008E62BD /* fishd */, D0D02AD01598642A008E62BD /* fish_indent */, D0D02AE415986537008E62BD /* fish_pager */, + D08A328D17B4455100F3A533 /* fish_tests */, ); name = Products; sourceTree = ""; @@ -745,6 +817,23 @@ /* End PBXLegacyTarget section */ /* Begin PBXNativeTarget section */ + D08A328C17B4455100F3A533 /* fish_tests */ = { + isa = PBXNativeTarget; + buildConfigurationList = D08A329217B4455100F3A533 /* Build configuration list for PBXNativeTarget "fish_tests" */; + buildPhases = ( + D08A328917B4455100F3A533 /* Sources */, + D08A328A17B4455100F3A533 /* Frameworks */, + D08A328B17B4455100F3A533 /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = fish_tests; + productName = fish_tests; + productReference = D08A328D17B4455100F3A533 /* fish_tests */; + productType = "com.apple.product-type.tool"; + }; D0D02A9915985A75008E62BD /* fish.app */ = { isa = PBXNativeTarget; buildConfigurationList = D0D02AA415985A75008E62BD /* Build configuration list for PBXNativeTarget "fish.app" */; @@ -854,6 +943,7 @@ D0D02ABB15985EF9008E62BD /* fishd */, D0D02ACF1598642A008E62BD /* fish_indent */, D0D02AE315986537008E62BD /* fish_pager */, + D08A328C17B4455100F3A533 /* fish_tests */, D0A564E6168CFDD800AF6161 /* man_pages */, D0A084F713B3AC130099B651 /* Makefile */, ); @@ -1034,6 +1124,53 @@ /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ + D08A328917B4455100F3A533 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */, + D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */, + D08A32BB17B446B100F3A533 /* parse_exec.cpp in Sources */, + D08A32A717B446A300F3A533 /* autoload.cpp in Sources */, + D08A32A817B446A300F3A533 /* builtin_test.cpp in Sources */, + D08A32A917B446A300F3A533 /* color.cpp in Sources */, + D08A32AA17B446A300F3A533 /* common.cpp in Sources */, + D08A32AB17B446A300F3A533 /* env_universal_common.cpp in Sources */, + D08A32AC17B446A300F3A533 /* env_universal.cpp in Sources */, + D08A32AD17B446A300F3A533 /* event.cpp in Sources */, + D08A32AE17B446A300F3A533 /* input_common.cpp in Sources */, + D08A32AF17B446A300F3A533 /* intern.cpp in Sources */, + D08A32B017B446A300F3A533 /* io.cpp in Sources */, + D08A32B117B446A300F3A533 /* iothread.cpp in Sources */, + D08A32B217B446A300F3A533 /* output.cpp in Sources */, + D08A32B317B446A300F3A533 /* parse_util.cpp in Sources */, + D08A32B417B446A300F3A533 /* parser_keywords.cpp in Sources */, + D08A32B517B446A300F3A533 /* path.cpp in Sources */, + D08A32B617B446A300F3A533 /* postfork.cpp in Sources */, + D08A32B717B446A300F3A533 /* screen.cpp in Sources */, + D08A32B817B446A300F3A533 /* signal.cpp in Sources */, + D08A32A617B4464300F3A533 /* input.cpp in Sources */, + D08A329717B4463B00F3A533 /* complete.cpp in Sources */, + D08A329817B4463B00F3A533 /* env.cpp in Sources */, + D08A329917B4463B00F3A533 /* exec.cpp in Sources */, + D08A329A17B4463B00F3A533 /* expand.cpp in Sources */, + D08A329B17B4463B00F3A533 /* highlight.cpp in Sources */, + D08A329C17B4463B00F3A533 /* history.cpp in Sources */, + D08A329D17B4463B00F3A533 /* kill.cpp in Sources */, + D08A329E17B4463B00F3A533 /* parser.cpp in Sources */, + D08A329F17B4463B00F3A533 /* proc.cpp in Sources */, + D08A32A017B4463B00F3A533 /* reader.cpp in Sources */, + D08A32A117B4463B00F3A533 /* sanity.cpp in Sources */, + D08A32A217B4463B00F3A533 /* tokenizer.cpp in Sources */, + D08A32A317B4463B00F3A533 /* wgetopt.cpp in Sources */, + D08A32A417B4463B00F3A533 /* wildcard.cpp in Sources */, + D08A32A517B4463B00F3A533 /* wutil.cpp in Sources */, + D08A329617B445FD00F3A533 /* builtin.cpp in Sources */, + D08A329417B4458D00F3A533 /* fish_tests.cpp in Sources */, + D08A329517B445C200F3A533 /* function.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; D0D02AB815985EF9008E62BD /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -1345,6 +1482,74 @@ }; name = Release; }; + D08A328F17B4455100F3A533 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_DYNAMIC_NO_PIC = NO; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_UNINITIALIZED_AUTOS = YES; + MACOSX_DEPLOYMENT_TARGET = 10.8; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + D08A329017B4455100F3A533 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES; + MACOSX_DEPLOYMENT_TARGET = 10.8; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; + D08A329117B4455100F3A533 /* Release_C++11 */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ARCHS = "$(ARCHS_STANDARD_64_BIT)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = YES; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_ENABLE_OBJC_EXCEPTIONS = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES; + MACOSX_DEPLOYMENT_TARGET = 10.8; + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = "Release_C++11"; + }; D0A084F813B3AC130099B651 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -1617,6 +1822,16 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + D08A329217B4455100F3A533 /* Build configuration list for PBXNativeTarget "fish_tests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + D08A328F17B4455100F3A533 /* Debug */, + D08A329017B4455100F3A533 /* Release */, + D08A329117B4455100F3A533 /* Release_C++11 */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; D0A084F513B3AC130099B651 /* Build configuration list for PBXProject "fish" */ = { isa = XCConfigurationList; buildConfigurations = ( -- cgit v1.2.3 From a51bd03a5c86d5532063610cc185f3d377daa4e1 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 8 Oct 2013 18:48:01 -0700 Subject: Remove parse_exec stuff --- Makefile.in | 2 +- builtin.cpp | 114 -------- fish.xcodeproj/project.pbxproj | 8 - parse_exec.cpp | 593 ----------------------------------------- parse_exec.h | 173 ------------ 5 files changed, 1 insertion(+), 889 deletions(-) delete mode 100644 parse_exec.cpp delete mode 100644 parse_exec.h (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/Makefile.in b/Makefile.in index 07dec023..23e24b38 100644 --- a/Makefile.in +++ b/Makefile.in @@ -100,7 +100,7 @@ FISH_OBJS := function.o builtin.o complete.o env.o exec.o expand.o \ env_universal.o env_universal_common.o input_common.o event.o \ signal.o io.o parse_util.o common.o screen.o path.o autoload.o \ parser_keywords.o iothread.o color.o postfork.o \ - builtin_test.o parse_tree.o parse_productions.o parse_exec.o + builtin_test.o parse_tree.o parse_productions.o FISH_INDENT_OBJS := fish_indent.o print_help.o common.o \ parser_keywords.o wutil.o tokenizer.o diff --git a/builtin.cpp b/builtin.cpp index 06c9a9b8..b84d78e7 100644 --- a/builtin.cpp +++ b/builtin.cpp @@ -65,7 +65,6 @@ #include "path.h" #include "history.h" #include "parse_tree.h" -#include "parse_exec.h" /** The default prompt for the read command @@ -3956,104 +3955,6 @@ static int builtin_history(parser_t &parser, wchar_t **argv) #pragma mark Simulator -struct parse_execution_simulator_t : public parse_execution_visitor_t -{ - wcstring_list_t result; - - wcstring &back() - { - assert(! result.empty()); - return result.back(); - } - - void append_src(node_offset_t idx) - { - wcstring tmp; - context->get_source(idx, &tmp); - back().append(tmp); - } - - void append(const wchar_t *s) - { - back().append(s); - } - - bool enter_job_list(void) - { - return true; - } - - bool enter_job(void) - { - result.resize(result.size() + 1); - return true; - } - - void visit_statement(void) - { - } - - virtual void visit_boolean_statement(void) - { - } - - virtual void enter_if_clause(const exec_if_clause_t &statement) - { - } - - virtual void exit_if_clause(const exec_if_clause_t &statement) - { - append_format(back(), L"\nIF successful jump to %lu", (unsigned long)statement.body); - } - - void visit_basic_statement(const exec_basic_statement_t &statement) - { - wcstring &line = this->back(); - if (! line.empty()) - { - line.append(L" "); - } - switch (statement.decoration) - { - case exec_basic_statement_t::decoration_builtin: - line.append(L" "); - break; - - case exec_basic_statement_t::decoration_command: - line.append(L" "); - break; - - default: - break; - } - - line.append(L"cmd:"); - this->append_src(statement.command_idx); - for (size_t i=0; i < statement.arguments().size(); i++) - { - const exec_argument_t &arg = statement.arguments().at(i); - append(L" "); - append(L"arg:"); - append_src(arg.parse_node_idx); - } - } - - void visit_function(const exec_function_header_t &function) - { - wcstring &line = this->back(); - line.append(L"define function: "); - wcstring tmp; - context->get_source(function.name_idx, &tmp); - line.append(tmp); - } - - void exit_job_list(void) - { - } -}; - - - int builtin_parse(parser_t &parser, wchar_t **argv) { struct sigaction act; @@ -4090,21 +3991,6 @@ int builtin_parse(parser_t &parser, wchar_t **argv) { const wcstring dump = parse_dump_tree(parse_tree, src); fprintf(stderr, "%ls", dump.c_str()); - if (0) - { - parse_execution_context_t ctx(parse_tree, src); - parse_execution_simulator_t sim; - sim.context = &ctx; - while (ctx.visit_next_node(&sim)) - { - } - stdout_buffer.append(L"Simulating execution:\n"); - for (size_t i=0; i < sim.result.size(); i++) - { - stdout_buffer.append(sim.result.at(i)); - stdout_buffer.push_back(L'\n'); - } - } } } return STATUS_BUILTIN_OK; diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 3b85e4bc..04d0c7cb 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -112,13 +112,11 @@ D08A32B817B446A300F3A533 /* signal.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0855C13B3ACEE0099B651 /* signal.cpp */; }; D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */; }; D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; - D08A32BB17B446B100F3A533 /* parse_exec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F331765281F00BFAB82 /* parse_exec.cpp */; }; D08A32BC17B4473B00F3A533 /* libncurses.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8C15983CFA008E62BD /* libncurses.dylib */; }; D08A32BD17B4474000F3A533 /* libiconv.dylib in Frameworks */ = {isa = PBXBuildFile; fileRef = D0D02A8A15983CDF008E62BD /* libiconv.dylib */; }; D0A564FE168D23D800AF6161 /* man in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0A56501168D258300AF6161 /* man in Copy Files */ = {isa = PBXBuildFile; fileRef = D0A564F1168D0BAB00AF6161 /* man */; }; D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F351765284C00BFAB82 /* parse_tree.cpp */; }; - D0C52F381765720600BFAB82 /* parse_exec.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0C52F331765281F00BFAB82 /* parse_exec.cpp */; }; D0CBD587159EF0E10024809C /* launch_fish.scpt in Resources */ = {isa = PBXBuildFile; fileRef = D0CBD586159EF0E10024809C /* launch_fish.scpt */; }; D0D02A67159837AD008E62BD /* complete.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853713B3ACEE0099B651 /* complete.cpp */; }; D0D02A69159837B2008E62BD /* env.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D0A0853A13B3ACEE0099B651 /* env.cpp */; }; @@ -497,8 +495,6 @@ D0B6B0FE14E88BA400AD6C10 /* color.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = color.cpp; sourceTree = ""; }; D0B6B0FF14E88BA400AD6C10 /* color.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = color.h; sourceTree = ""; }; D0C4FD9415A7D7EE00212EF1 /* config.fish */ = {isa = PBXFileReference; lastKnownFileType = text; name = config.fish; path = etc/config.fish; sourceTree = ""; }; - D0C52F331765281F00BFAB82 /* parse_exec.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_exec.cpp; sourceTree = ""; }; - D0C52F341765281F00BFAB82 /* parse_exec.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_exec.h; sourceTree = ""; }; D0C52F351765284C00BFAB82 /* parse_tree.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_tree.cpp; sourceTree = ""; }; D0C52F361765284C00BFAB82 /* parse_tree.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_tree.h; sourceTree = ""; }; D0C6FCC914CFA4B0004CE8AD /* autoload.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = autoload.cpp; sourceTree = ""; }; @@ -665,8 +661,6 @@ D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */, D0C52F361765284C00BFAB82 /* parse_tree.h */, D0C52F351765284C00BFAB82 /* parse_tree.cpp */, - D0C52F341765281F00BFAB82 /* parse_exec.h */, - D0C52F331765281F00BFAB82 /* parse_exec.cpp */, D0A0850D13B3ACEE0099B651 /* fallback.h */, D0A0853E13B3ACEE0099B651 /* fallback.cpp */, D0A0850E13B3ACEE0099B651 /* function.h */, @@ -1130,7 +1124,6 @@ files = ( D08A32B917B446B100F3A533 /* parse_productions.cpp in Sources */, D08A32BA17B446B100F3A533 /* parse_tree.cpp in Sources */, - D08A32BB17B446B100F3A533 /* parse_exec.cpp in Sources */, D08A32A717B446A300F3A533 /* autoload.cpp in Sources */, D08A32A817B446A300F3A533 /* builtin_test.cpp in Sources */, D08A32A917B446A300F3A533 /* color.cpp in Sources */, @@ -1255,7 +1248,6 @@ D0D02A7B15983928008E62BD /* env_universal_common.cpp in Sources */, D0D02A89159839DF008E62BD /* fish.cpp in Sources */, D0C52F371765284C00BFAB82 /* parse_tree.cpp in Sources */, - D0C52F381765720600BFAB82 /* parse_exec.cpp in Sources */, D0FE8EE8179FB760008C9F21 /* parse_productions.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/parse_exec.cpp b/parse_exec.cpp deleted file mode 100644 index c424ad95..00000000 --- a/parse_exec.cpp +++ /dev/null @@ -1,593 +0,0 @@ -#include "parse_exec.h" -#include - -struct exec_node_t -{ - node_offset_t parse_node_idx; - node_offset_t body_parse_node_idx; - bool visited; - - explicit exec_node_t(node_offset_t pni) : parse_node_idx(pni), body_parse_node_idx(NODE_OFFSET_INVALID), visited(false) - { - } - - explicit exec_node_t(node_offset_t pni, node_offset_t body_pni) : parse_node_idx(pni), body_parse_node_idx(body_pni), visited(false) - { - } -}; - -exec_basic_statement_t::exec_basic_statement_t() : command_idx(0), decoration(decoration_plain) -{ - -} - - -class parse_exec_t -{ - parse_node_tree_t parse_tree; - wcstring src; - - /* The stack of nodes as we execute them */ - std::vector exec_nodes; - - /* The stack of commands being built */ - std::vector assembling_statements; - - /* Current visitor (very transient) */ - struct parse_execution_visitor_t * visitor; - - const parse_node_t &get_child(const parse_node_t &parent, node_offset_t which) const - { - return parse_tree.at(parent.child_offset(which)); - } - - void pop_push_specific(node_offset_t idx1, node_offset_t idx2 = NODE_OFFSET_INVALID, node_offset_t idx3 = NODE_OFFSET_INVALID, node_offset_t idx4 = NODE_OFFSET_INVALID, node_offset_t idx5 = NODE_OFFSET_INVALID) - { - PARSE_ASSERT(! exec_nodes.empty()); - // Figure out the offset of the children - exec_node_t &top = exec_nodes.back(); - const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); - node_offset_t child_node_idx = parse_node.child_start; - - // Remove the top node - exec_nodes.pop_back(); - - // Append the given children, backwards - const node_offset_t idxs[] = {idx5, idx4, idx3, idx2, idx1}; - for (size_t q=0; q < sizeof idxs / sizeof *idxs; q++) - { - node_offset_t idx = idxs[q]; - if (idx != (node_offset_t)(-1)) - { - PARSE_ASSERT(idx < parse_node.child_count); - exec_nodes.push_back(exec_node_t(child_node_idx + idx)); - } - } - - } - - void push(node_offset_t global_idx) - { - exec_nodes.push_back(exec_node_t(global_idx)); - } - - void push(const exec_node_t &node) - { - exec_nodes.push_back(node); - } - - - void pop_push(node_offset_t child_idx, node_offset_t child_count = 1) - { - PARSE_ASSERT(! exec_nodes.empty()); - if (child_count == 0) - { - // No children, just remove the top node - exec_nodes.pop_back(); - } - else - { - // Figure out the offset of the children - exec_node_t &top = exec_nodes.back(); - const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); - PARSE_ASSERT(child_idx < parse_node.child_count); - node_offset_t child_node_idx = parse_node.child_start + child_idx; - - // Remove the top node - exec_nodes.pop_back(); - - // Append the given children, backwards - node_offset_t cursor = child_count; - while (cursor--) - { - exec_nodes.push_back(exec_node_t(child_node_idx + cursor)); - } - } - } - - void pop() - { - PARSE_ASSERT(! exec_nodes.empty()); - exec_nodes.pop_back(); - } - - void pop_push_all() - { - exec_node_t &top = exec_nodes.back(); - const parse_node_t &parse_node = parse_tree.at(top.parse_node_idx); - pop_push(0, parse_node.child_count); - } - - void assemble_1_argument_or_redirection(node_offset_t idx, exec_arguments_and_redirections_t *output) const - { - const parse_node_t &node = parse_tree.at(idx); - PARSE_ASSERT(output != NULL); - PARSE_ASSERT(node.type == symbol_argument_or_redirection); - PARSE_ASSERT(node.child_count == 1); - node_offset_t child_idx = node.child_offset(0); - const parse_node_t &child = parse_tree.at(child_idx); - switch (child.type) - { - case parse_token_type_string: - // Argument - { - exec_argument_t arg = exec_argument_t(); - arg.parse_node_idx = child_idx; - output->arguments.push_back(arg); - } - break; - - case parse_token_type_redirection: - // Redirection - { - exec_redirection_t redirect = exec_redirection_t(); - redirect.parse_node_idx = child_idx; - output->redirections.push_back(redirect); - } - break; - - default: - PARSER_DIE(); - break; - } - } - - void assemble_arguments_and_redirections(node_offset_t start_idx, exec_arguments_and_redirections_t *output) const - { - node_offset_t idx = start_idx; - for (;;) - { - const parse_node_t &node = parse_tree.at(idx); - PARSE_ASSERT(node.type == symbol_arguments_or_redirections_list); - PARSE_ASSERT(node.child_count == 0 || node.child_count == 2); - if (node.child_count == 0) - { - // No more children - break; - } - else - { - // Skip to next child - assemble_1_argument_or_redirection(node.child_offset(0), output); - idx = node.child_offset(1); - } - } - } - - void assemble_command_for_plain_statement(node_offset_t idx, parse_keyword_t decoration) - { - const parse_node_t &node = parse_tree.at(idx); - PARSE_ASSERT(node.type == symbol_plain_statement); - PARSE_ASSERT(node.child_count == 2); - exec_basic_statement_t statement; - statement.set_decoration(decoration); - statement.command_idx = node.child_offset(0); - assemble_arguments_and_redirections(node.child_offset(1), &statement.arguments_and_redirections); - visitor->visit_basic_statement(statement); - } - - void assemble_block_statement(node_offset_t parse_node_idx) - { - - const parse_node_t &node = parse_tree.at(parse_node_idx); - PARSE_ASSERT(node.type == symbol_block_statement); - PARSE_ASSERT(node.child_count == 5); - - // Fetch arguments and redirections. These ought to be evaluated before the job list - exec_block_statement_t statement; - assemble_arguments_and_redirections(node.child_offset(4), &statement.arguments_and_redirections); - - // Generic visit - visitor->enter_block_statement(statement); - - // Dig into the header to discover the type - const parse_node_t &header_parent = parse_tree.at(node.child_offset(0)); - PARSE_ASSERT(header_parent.type == symbol_block_header); - PARSE_ASSERT(header_parent.child_count == 1); - const node_offset_t header_idx = header_parent.child_offset(0); - - // Fetch body (job list) - node_offset_t body_idx = node.child_offset(2); - PARSE_ASSERT(parse_tree.at(body_idx).type == symbol_job_list); - - pop(); - push(exec_node_t(header_idx, body_idx)); - } - - /* which: 0 -> if, 1 -> else if, 2 -> else */ - void assemble_if_else_clause(exec_node_t &exec_node, const parse_node_t &node, int which) - { - if (which == 0) - { - PARSE_ASSERT(node.type == symbol_if_clause); - PARSE_ASSERT(node.child_count == 4); - } - else if (which == 2) - { - PARSE_ASSERT(node.type == symbol_else_continuation); - PARSE_ASSERT(node.child_count == 2); - } - - struct exec_if_clause_t clause; - if (which == 0) - { - clause.body = node.child_offset(3); - } - else - { - clause.body = node.child_offset(1); - } - if (! exec_node.visited) - { - visitor->enter_if_clause(clause); - exec_node.visited = true; - if (which == 0) - { - push(node.child_offset(1)); - } - } - else - { - visitor->exit_if_clause(clause); - pop(); - } - } - - void assemble_arguments(node_offset_t start_idx, exec_argument_list_t *output) const - { - node_offset_t idx = start_idx; - for (;;) - { - const parse_node_t &node = parse_tree.at(idx); - PARSE_ASSERT(node.type == symbol_argument_list); - if (node.type == symbol_argument_list) - { - // argument list, may be empty - PARSE_ASSERT(node.child_count == 0 || node.child_count == 1); - if (node.child_count == 0) - { - break; - } - else - { - idx = node.child_offset(0); - } - } - else - { - // nonempty argument list - PARSE_ASSERT(node.child_count == 2); - output->push_back(exec_argument_t(node.child_offset(0))); - idx = node.child_offset(1); - } - } - } - - void assemble_1_case_item(exec_switch_statement_t *statement, node_offset_t node_idx) - { - const parse_node_t &node = parse_tree.at(node_idx); - PARSE_ASSERT(node.type == symbol_case_item); - - // add a new case - size_t len = statement->cases.size(); - statement->cases.resize(len + 1); - exec_switch_case_t &new_case = statement->cases.back(); - - // assemble it - new_case.body = node.child_offset(3); - assemble_arguments(node.child_offset(1), &new_case.arguments); - - - } - - void assemble_case_item_list(exec_switch_statement_t *statement, node_offset_t node_idx) - { - const parse_node_t &node = parse_tree.at(node_idx); - PARSE_ASSERT(node.type == symbol_case_item_list); - PARSE_ASSERT(node.child_count == 0 || node.child_count == 2); - if (node.child_count == 2) - { - assemble_1_case_item(statement, node.child_offset(0)); - assemble_case_item_list(statement, node.child_offset(1)); - } - } - - void assemble_switch_statement(const exec_node_t &exec_node, const parse_node_t &parse_node) - { - PARSE_ASSERT(parse_node.type == symbol_switch_statement); - exec_switch_statement_t statement; - - statement.argument.parse_node_idx = parse_node.child_offset(1); - assemble_case_item_list(&statement, parse_node.child_offset(3)); - - visitor->visit_switch_statement(statement); - - // pop off the switch - pop(); - } - - void assemble_function_header(const exec_node_t &exec_node, const parse_node_t &header) - { - PARSE_ASSERT(header.type == symbol_function_header); - PARSE_ASSERT(&header == &parse_tree.at(exec_node.parse_node_idx)); - PARSE_ASSERT(exec_node.body_parse_node_idx != NODE_OFFSET_INVALID); - exec_function_header_t function_info; - function_info.name_idx = header.child_offset(1); - function_info.body_idx = exec_node.body_parse_node_idx; - assemble_arguments(header.child_offset(2), &function_info.arguments); - visitor->visit_function(function_info); - - // Always pop - pop(); - } - - - void enter_parse_node(size_t idx); - void run_top_node(void); - -public: - - void get_node_string(node_offset_t idx, wcstring *output) const - { - const parse_node_t &node = parse_tree.at(idx); - PARSE_ASSERT(node.source_start <= src.size()); - PARSE_ASSERT(node.source_start + node.source_length <= src.size()); - output->assign(src, node.source_start, node.source_length); - } - - bool visit_next_node(parse_execution_visitor_t *v); - - parse_exec_t(const parse_node_tree_t &tree, const wcstring &s) : parse_tree(tree), src(s), visitor(NULL) - { - if (! parse_tree.empty()) - { - exec_nodes.push_back(exec_node_t(0)); - } - } -}; - -void parse_exec_t::run_top_node() -{ - PARSE_ASSERT(! exec_nodes.empty()); - exec_node_t &exec_node = exec_nodes.back(); - const node_offset_t parse_node_idx = exec_node.parse_node_idx; - const parse_node_t &parse_node = parse_tree.at(exec_node.parse_node_idx); - bool log = true; - - if (log) - { - wcstring tmp; - tmp.append(exec_nodes.size(), L' '); - tmp.append(parse_node.describe()); - printf("%ls\n", tmp.c_str()); - } - - switch (parse_node.type) - { - case symbol_job_list: - PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 2); - if (parse_node.child_count == 0) - { - // No more jobs, done - visitor->exit_job_list(); - pop(); - } - else if (parse_tree.at(parse_node.child_start + 0).type == parse_token_type_end) - { - // Empty job, so just skip it - pop_push(1, 1); - } - else - { - // Normal job - visitor->enter_job_list(); - pop_push(0, 2); - } - break; - - case symbol_job: - { - PARSE_ASSERT(parse_node.child_count == 2); - visitor->enter_job(); - pop_push_all(); - break; - } - - case symbol_job_continuation: - PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 3); - if (parse_node.child_count == 0) - { - // All done with this job - visitor->exit_job(); - pop(); - } - else - { - // Skip the pipe - pop_push(1, 2); - } - break; - - case symbol_statement: - { - PARSE_ASSERT(parse_node.child_count == 1); - pop_push_all(); - break; - } - - case symbol_block_statement: - { - PARSE_ASSERT(parse_node.child_count == 5); - assemble_block_statement(parse_node_idx); - break; - } - - case symbol_block_header: - { - PARSE_ASSERT(parse_node.child_count == 1); - pop_push_all(); - break; - } - - case symbol_function_header: - { - PARSE_ASSERT(parse_node.child_count == 3); - assemble_function_header(exec_node, parse_node); - break; - } - - case symbol_if_statement: - { - PARSE_ASSERT(parse_node.child_count == 4); - pop_push(0, 2); - break; - } - - case symbol_if_clause: - { - PARSE_ASSERT(parse_node.child_count == 4); - assemble_if_else_clause(exec_node, parse_node, 0); - pop(); - break; - } - - case symbol_else_clause: - { - PARSE_ASSERT(parse_node.child_count == 0 || parse_node.child_count == 2); - if (parse_node.child_count == 0) - { - // No else - pop(); - } - else - { - // We have an else - pop_push(1); - } - break; - } - - case symbol_else_continuation: - { - // Figure out if this is an else if or a terminating else - PARSE_ASSERT(parse_node.child_count == 2); - const parse_node_t &first_child = get_child(parse_node, 1); - PARSE_ASSERT(first_child.type == symbol_if_clause || first_child.type == parse_token_type_end); - if (first_child.type == symbol_if_clause) - { - pop_push_all(); - } - else - { - // else - assemble_if_else_clause(exec_node, parse_node, 2); - pop(); - } - break; - } - - case symbol_switch_statement: - { - assemble_switch_statement(exec_node, parse_node); - break; - } - - case symbol_decorated_statement: - { - PARSE_ASSERT(parse_node.child_count == 1 || parse_node.child_count == 2); - - node_offset_t plain_statement_idx = parse_node.child_offset(parse_node.child_count - 1); - parse_keyword_t decoration = static_cast(parse_node.tag); - assemble_command_for_plain_statement(plain_statement_idx, decoration); - pop(); - break; - } - - // The following symbols should be handled by their parents, i.e. never pushed on our stack - case symbol_case_item_list: - case symbol_plain_statement: - case symbol_arguments_or_redirections_list: - case symbol_argument_or_redirection: - fprintf(stderr, "Unexpected token type %ls at index %ld. This should have been handled by the parent.\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx); - PARSER_DIE(); - break; - - case parse_token_type_end: - PARSE_ASSERT(parse_node.child_count == 0); - pop(); - break; - - default: - fprintf(stderr, "Unhandled token type %ls at index %ld\n", token_type_description(parse_node.type).c_str(), exec_node.parse_node_idx); - PARSER_DIE(); - break; - - } -} - -bool parse_exec_t::visit_next_node(parse_execution_visitor_t *v) -{ - PARSE_ASSERT(v != NULL); - PARSE_ASSERT(visitor == NULL); - if (exec_nodes.empty()) - { - return false; - } - - visitor = v; - run_top_node(); - visitor = NULL; - return true; -} - -void parse_exec_t::enter_parse_node(size_t idx) -{ - PARSE_ASSERT(idx < parse_tree.size()); - exec_node_t exec(idx); - exec_nodes.push_back(exec); -} - - -parse_execution_context_t::parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s) -{ - ctx = new parse_exec_t(n, s); -} - -parse_execution_context_t::~parse_execution_context_t() -{ - delete ctx; -} - -bool parse_execution_context_t::visit_next_node(parse_execution_visitor_t *visitor) -{ - return ctx->visit_next_node(visitor); -} - -void parse_execution_context_t::get_source(node_offset_t idx, wcstring *result) const -{ - return ctx->get_node_string(idx, result); -} - - - - diff --git a/parse_exec.h b/parse_exec.h deleted file mode 100644 index 1eea99ab..00000000 --- a/parse_exec.h +++ /dev/null @@ -1,173 +0,0 @@ -/**\file parse_exec.h - - Programmatic execution of a parse tree -*/ - -#ifndef FISH_PARSE_EXEC_H -#define FISH_PARSE_EXEC_H - -#include "parse_tree.h" - -struct parse_execution_visitor_t; -class parse_exec_t; -class parse_execution_context_t -{ - parse_exec_t *ctx; //owned - -public: - parse_execution_context_t(const parse_node_tree_t &n, const wcstring &s); - ~parse_execution_context_t(); - - bool visit_next_node(parse_execution_visitor_t *visitor); - - // Gets the source for a node at a given index - void get_source(node_offset_t idx, wcstring *result) const; -}; - - -struct exec_argument_t -{ - node_offset_t parse_node_idx; - exec_argument_t(node_offset_t p) : parse_node_idx(p) - { - } - exec_argument_t() - { - } -}; -typedef std::vector exec_argument_list_t; - -struct exec_redirection_t -{ - node_offset_t parse_node_idx; -}; -typedef std::vector exec_redirection_list_t; - -struct exec_arguments_and_redirections_t -{ - exec_argument_list_t arguments; - exec_redirection_list_t redirections; -}; - -struct exec_basic_statement_t -{ - // Node containing the command - node_offset_t command_idx; - - // Arguments - exec_arguments_and_redirections_t arguments_and_redirections; - - // Decoration - enum - { - decoration_plain, - decoration_command, - decoration_builtin - } decoration; - - exec_basic_statement_t(); - - void set_decoration(uint32_t k) - { - PARSE_ASSERT(k == parse_keyword_none || k == parse_keyword_command || k == parse_keyword_builtin); - switch (k) - { - case parse_keyword_none: - decoration = decoration_plain; - break; - case parse_keyword_command: - decoration = decoration_command; - break; - case parse_keyword_builtin: - decoration = decoration_builtin; - break; - default: - PARSER_DIE(); - break; - } - } - - const exec_argument_list_t &arguments() const - { - return arguments_and_redirections.arguments; - } - - const exec_redirection_list_t &redirections() const - { - return arguments_and_redirections.redirections; - } -}; - -struct exec_function_header_t -{ - // Node containing the function name - node_offset_t name_idx; - - // Node containing the function body - node_offset_t body_idx; - - // Arguments - exec_argument_list_t arguments; -}; - -struct exec_block_statement_t -{ - // Arguments - exec_arguments_and_redirections_t arguments_and_redirections; -}; - -struct exec_if_clause_t -{ - // Node containing the body of the if statement - node_offset_t body; -}; - -struct exec_switch_case_t -{ - exec_argument_list_t arguments; - node_offset_t body; -}; - -struct exec_switch_statement_t -{ - exec_argument_t argument; - std::vector cases; -}; - -struct parse_execution_visitor_t -{ - node_offset_t node_idx; - parse_execution_context_t *context; - - parse_execution_visitor_t() : node_idx(0), context(NULL) - { - } - - virtual bool enter_job_list(void) - { - return true; - } - virtual bool enter_job(void) - { - return true; - } - virtual void visit_statement(void) { } - virtual void visit_function(const exec_function_header_t &function) { } - virtual bool enter_block_statement(const exec_block_statement_t &statement) - { - return true; - } - - virtual void enter_if_clause(const exec_if_clause_t &statement) { } - virtual void exit_if_clause(const exec_if_clause_t &statement) { } - - virtual void visit_switch_statement(const exec_switch_statement_t &header) { } - - - virtual void visit_boolean_statement(void) { } - virtual void visit_basic_statement(const exec_basic_statement_t &statement) { } - virtual void exit_job(void) { } - virtual void exit_job_list(void) { } -}; - -#endif -- cgit v1.2.3 From 7a3f5afee7b6a6ab9f801ca3cd65c2c552554987 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Sun, 8 Dec 2013 21:54:06 -0800 Subject: Initial work towars improved error reporting. Tests currently fail. --- fish.xcodeproj/project.pbxproj | 2 + fish_tests.cpp | 55 ++++++- parse_constants.h | 320 +++++++++++++++++++++++++++++++++++++++++ parse_productions.cpp | 2 +- parse_tree.cpp | 126 ++++++++++++---- parse_tree.h | 96 +------------ parser.cpp | 16 +-- tokenizer.cpp | 8 -- wutil.cpp | 2 +- 9 files changed, 483 insertions(+), 144 deletions(-) create mode 100644 parse_constants.h (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index 2417de45..aa7f9c18 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -513,6 +513,7 @@ D0D02AE415986537008E62BD /* fish_pager */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish_pager; sourceTree = BUILT_PRODUCTS_DIR; }; D0D02AFA159871B2008E62BD /* osx_fish_launcher.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = osx_fish_launcher.m; path = osx/osx_fish_launcher.m; sourceTree = ""; }; D0D2693C159835CA005D9B9C /* fish */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = fish; sourceTree = BUILT_PRODUCTS_DIR; }; + D0D9B2B318555D92001AE279 /* parse_constants.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = parse_constants.h; sourceTree = ""; }; D0F3373A1506DE3C00ECEFC0 /* builtin_test.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = builtin_test.cpp; sourceTree = ""; }; D0F5E28415A7A32D00315DFF /* config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = config.h; sourceTree = ""; }; D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_productions.h; sourceTree = ""; }; @@ -659,6 +660,7 @@ D0A0853D13B3ACEE0099B651 /* expand.cpp */, D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */, D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */, + D0D9B2B318555D92001AE279 /* parse_constants.h */, D0C52F361765284C00BFAB82 /* parse_tree.h */, D0C52F351765284C00BFAB82 /* parse_tree.cpp */, D0A0850D13B3ACEE0099B651 /* fallback.h */, diff --git a/fish_tests.cpp b/fish_tests.cpp index 16f657fc..c43381b3 100644 --- a/fish_tests.cpp +++ b/fish_tests.cpp @@ -2333,7 +2333,7 @@ static void test_new_parser_ll2(void) } } -static void test_new_parser_ad_hoc(void) +static void test_new_parser_ad_hoc() { /* Very ad-hoc tests for issues encountered */ say(L"Testing new parser ad hoc tests"); @@ -2356,6 +2356,58 @@ static void test_new_parser_ad_hoc(void) } } +static void test_new_parser_errors(void) +{ + say(L"Testing new parser error reporting"); + const struct + { + const wchar_t *src; + parse_error_code_t code; + } + tests[] = + { + {L"echo (abc", parse_error_tokenizer}, + + {L"end", parse_error_unbalancing_end}, + {L"echo hi ; end", parse_error_unbalancing_end}, + + {L"else", parse_error_unbalancing_else}, + {L"if true ; end ; else", parse_error_unbalancing_else}, + + {L"case", parse_error_unbalancing_case}, + {L"if true ; case ; end", parse_error_unbalancing_case} + }; + + for (size_t i = 0; i < sizeof tests / sizeof *tests; i++) + { + const wcstring src = tests[i].src; + parse_error_code_t expected_code = tests[i].code; + + parse_error_list_t errors; + parse_node_tree_t parse_tree; + bool success = parse_t::parse(src, parse_flag_none, &parse_tree, &errors); + if (success) + { + err(L"Source '%ls' was expected to fail to parse, but succeeded", src.c_str()); + } + + if (errors.size() != 1) + { + err(L"Source '%ls' was expected to produce 1 error, but instead produced %lu errors", src.c_str(), errors.size()); + } + else if (errors.at(0).code != expected_code) + { + err(L"Source '%ls' was expected to produce error code %lu, but instead produced error code %lu", src.c_str(), expected_code, (unsigned long)errors.at(0).code); + for (size_t i=0; i < errors.size(); i++) + { + err(L"\t\t%ls", errors.at(i).describe(src).c_str()); + } + } + + } + +} + static void test_highlighting(void) { say(L"Testing syntax highlighting"); @@ -2574,6 +2626,7 @@ int main(int argc, char **argv) if (should_test_function("new_parser_fuzzing")) test_new_parser_fuzzing(); //fuzzing is expensive if (should_test_function("new_parser_correctness")) test_new_parser_correctness(); if (should_test_function("new_parser_ad_hoc")) test_new_parser_ad_hoc(); + if (should_test_function("new_parser_errors")) test_new_parser_errors(); if (should_test_function("escape")) test_unescape_sane(); if (should_test_function("escape")) test_escape_crazy(); if (should_test_function("format")) test_format(); diff --git a/parse_constants.h b/parse_constants.h new file mode 100644 index 00000000..d9f36212 --- /dev/null +++ b/parse_constants.h @@ -0,0 +1,320 @@ +/**\file parse_constants.h + + Constants used in the programmatic representation of fish code. +*/ + +#ifndef fish_parse_constants_h +#define fish_parse_constants_h + +#define PARSE_ASSERT(a) assert(a) +#define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0) + + +enum parse_token_type_t +{ + token_type_invalid, + + // Non-terminal tokens + symbol_job_list, + symbol_job, + symbol_job_continuation, + symbol_statement, + symbol_block_statement, + symbol_block_header, + symbol_for_header, + symbol_while_header, + symbol_begin_header, + symbol_function_header, + + symbol_if_statement, + symbol_if_clause, + symbol_else_clause, + symbol_else_continuation, + + symbol_switch_statement, + symbol_case_item_list, + symbol_case_item, + + symbol_boolean_statement, + symbol_decorated_statement, + symbol_plain_statement, + symbol_arguments_or_redirections_list, + symbol_argument_or_redirection, + + symbol_argument_list, + + symbol_argument, + symbol_redirection, + + symbol_optional_background, + + // Terminal types + parse_token_type_string, + parse_token_type_pipe, + parse_token_type_redirection, + parse_token_type_background, + parse_token_type_end, + parse_token_type_terminate, + + // Very special terminal types that don't appear in the production list + parse_special_type_parse_error, + parse_special_type_tokenizer_error, + parse_special_type_comment, + + FIRST_TERMINAL_TYPE = parse_token_type_string, + LAST_TERMINAL_TYPE = parse_token_type_terminate, + + LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate, + FIRST_PARSE_TOKEN_TYPE = parse_token_type_string +}; + +enum parse_keyword_t +{ + parse_keyword_none, + parse_keyword_if, + parse_keyword_else, + parse_keyword_for, + parse_keyword_in, + parse_keyword_while, + parse_keyword_begin, + parse_keyword_function, + parse_keyword_switch, + parse_keyword_case, + parse_keyword_end, + parse_keyword_and, + parse_keyword_or, + parse_keyword_not, + parse_keyword_command, + parse_keyword_builtin, + + LAST_KEYWORD = parse_keyword_builtin +}; + +/* Statement decorations. This matches the order of productions in decorated_statement */ +enum parse_statement_decoration_t +{ + parse_statement_decoration_none, + parse_statement_decoration_command, + parse_statement_decoration_builtin +}; + +/* Parse error code list */ +enum parse_error_code_t +{ + parse_error_none, + parse_error_generic, //unknown type + + parse_error_tokenizer, //tokenizer error + + parse_error_unbalancing_end, //end outside of block + parse_error_unbalancing_else, //else outside of if + parse_error_unbalancing_case, //case outside of switch +}; + + +/** + Error message for tokenizer error. The tokenizer message is + appended to this message. +*/ +#define TOK_ERR_MSG _( L"Tokenizer error: '%ls'") + +/** + Error message for short circuit command error. +*/ +#define COND_ERR_MSG _( L"An additional command is required" ) + +/** + Error message on a function that calls itself immediately +*/ +#define INFINITE_RECURSION_ERR_MSG _( L"The function calls itself immediately, which would result in an infinite loop.") + +/** + Error message on reaching maximum recursion depth +*/ +#define OVERFLOW_RECURSION_ERR_MSG _( L"Maximum recursion depth reached. Accidental infinite loop?") + +/** + Error message used when the end of a block can't be located +*/ +#define BLOCK_END_ERR_MSG _( L"Could not locate end of block. The 'end' command is missing, misspelled or a ';' is missing.") + +/** + Error message when a non-string token is found when expecting a command name +*/ +#define CMD_ERR_MSG _( L"Expected a command name, got token of type '%ls'") + +/** + Error message when a non-string token is found when expecting a command name +*/ +#define CMD_OR_ERR_MSG _( L"Expected a command name, got token of type '%ls'. Did you mean 'COMMAND; or COMMAND'? See the help section for the 'or' builtin command by typing 'help or'.") + +/** + Error message when a non-string token is found when expecting a command name +*/ +#define CMD_AND_ERR_MSG _( L"Expected a command name, got token of type '%ls'. Did you mean 'COMMAND; and COMMAND'? See the help section for the 'and' builtin command by typing 'help and'.") + +/** + Error message when encountering an illegal command name +*/ +#define ILLEGAL_CMD_ERR_MSG _( L"Illegal command name '%ls'") + +/** + Error message when encountering an illegal file descriptor +*/ +#define ILLEGAL_FD_ERR_MSG _( L"Illegal file descriptor '%ls'") + +/** + Error message for wildcards with no matches +*/ +#define WILDCARD_ERR_MSG _( L"No matches for wildcard '%ls'.") + +/** + Error when using case builtin outside of switch block +*/ +#define INVALID_CASE_ERR_MSG _( L"'case' builtin not inside of switch block") + +/** + Error when using loop control builtins (break or continue) outside of loop +*/ +#define INVALID_LOOP_ERR_MSG _( L"Loop control command while not inside of loop" ) + +/** + Error when using return builtin outside of function definition +*/ +#define INVALID_RETURN_ERR_MSG _( L"'return' builtin command outside of function definition" ) + +/** + Error when using else builtin outside of if block +*/ +#define INVALID_ELSE_ERR_MSG _( L"'%ls' builtin not inside of if block" ) + +/** + Error when using 'else if' past a naked 'else' +*/ +#define INVALID_ELSEIF_PAST_ELSE_ERR_MSG _( L"'%ls' used past terminating 'else'" ) + +/** + Error when using end builtin outside of block +*/ +#define INVALID_END_ERR_MSG _( L"'end' command outside of block") + +/** + Error message for Posix-style assignment: foo=bar +*/ +#define COMMAND_ASSIGN_ERR_MSG _( L"Unknown command '%ls'. Did you mean 'set %ls %ls'? See the help section on the set command by typing 'help set'.") + +/** + Error for invalid redirection token +*/ +#define REDIRECT_TOKEN_ERR_MSG _( L"Expected redirection specification, got token of type '%ls'") + +/** + Error when encountering redirection without a command +*/ +#define INVALID_REDIRECTION_ERR_MSG _( L"Encountered redirection when expecting a command name. Fish does not allow a redirection operation before a command.") + +/** + Error for evaluating null pointer +*/ +#define EVAL_NULL_ERR_MSG _( L"Tried to evaluate null pointer." ) + +/** + Error for evaluating in illegal scope +*/ +#define INVALID_SCOPE_ERR_MSG _( L"Tried to evaluate commands using invalid block type '%ls'" ) + + +/** + Error for wrong token type +*/ +#define UNEXPECTED_TOKEN_ERR_MSG _( L"Unexpected token of type '%ls'") + +/** + While block description +*/ +#define WHILE_BLOCK N_( L"'while' block" ) + +/** + For block description +*/ +#define FOR_BLOCK N_( L"'for' block" ) + +/** + Breakpoint block +*/ +#define BREAKPOINT_BLOCK N_( L"Block created by breakpoint" ) + + + +/** + If block description +*/ +#define IF_BLOCK N_( L"'if' conditional block" ) + + +/** + Function definition block description +*/ +#define FUNCTION_DEF_BLOCK N_( L"function definition block" ) + + +/** + Function invocation block description +*/ +#define FUNCTION_CALL_BLOCK N_( L"function invocation block" ) + +/** + Function invocation block description +*/ +#define FUNCTION_CALL_NO_SHADOW_BLOCK N_( L"function invocation block with no variable shadowing" ) + + +/** + Switch block description +*/ +#define SWITCH_BLOCK N_( L"'switch' block" ) + + +/** + Fake block description +*/ +#define FAKE_BLOCK N_( L"unexecutable block" ) + + +/** + Top block description +*/ +#define TOP_BLOCK N_( L"global root block" ) + + +/** + Command substitution block description +*/ +#define SUBST_BLOCK N_( L"command substitution block" ) + + +/** + Begin block description +*/ +#define BEGIN_BLOCK N_( L"'begin' unconditional block" ) + + +/** + Source block description +*/ +#define SOURCE_BLOCK N_( L"Block created by the . builtin" ) + +/** + Source block description +*/ +#define EVENT_BLOCK N_( L"event handler block" ) + + +/** + Unknown block description +*/ +#define UNKNOWN_BLOCK N_( L"unknown/invalid block" ) + + + +#endif diff --git a/parse_productions.cpp b/parse_productions.cpp index c3ab9c3a..9053c5da 100644 --- a/parse_productions.cpp +++ b/parse_productions.cpp @@ -46,7 +46,7 @@ RESOLVE(job_list) switch (token1.type) { case parse_token_type_string: - // 'end' is special + // some keywords are special switch (token1.keyword) { case parse_keyword_end: diff --git a/parse_tree.cpp b/parse_tree.cpp index ad0dd0ea..a249e45e 100644 --- a/parse_tree.cpp +++ b/parse_tree.cpp @@ -4,6 +4,11 @@ using namespace parse_productions; +static bool production_is_empty(const production_t *production) +{ + return (*production)[0] == token_type_invalid; +} + /** Returns a string description of this parse error */ wcstring parse_error_t::describe(const wcstring &src) const { @@ -18,7 +23,7 @@ wcstring parse_error_t::describe(const wcstring &src) const //fprintf(stderr, "newline: %lu, source_start %lu, source_length %lu\n", newline, source_start, source_length); if (newline != wcstring::npos) { - line_start = newline;// + 1; + line_start = newline + 1; } size_t line_end = src.find(L'\n', source_start + source_length); @@ -155,6 +160,8 @@ wcstring keyword_description(parse_keyword_t k) return L"function"; case parse_keyword_switch: return L"switch"; + case parse_keyword_case: + return L"case"; case parse_keyword_end: return L"end"; case parse_keyword_and: @@ -167,9 +174,8 @@ wcstring keyword_description(parse_keyword_t k) return L"command"; case parse_keyword_builtin: return L"builtin"; - default: - return format_string(L"Unknown keyword type %ld", static_cast(k)); } + return format_string(L"Unknown keyword type %ld", static_cast(k)); } /** Returns a string description of the given parse node */ @@ -348,7 +354,8 @@ class parse_ll_t bool top_node_handle_terminal_types(parse_token_t token); void parse_error(const wchar_t *expected, parse_token_t token); - void parse_error(parse_token_t token, const wchar_t *format, ...); + void parse_error(parse_token_t token, parse_error_code_t code, const wchar_t *format, ...); + void parse_error_unbalancing_token(parse_token_t token); void append_error_callout(wcstring &error_message, parse_token_t token); void dump_stack(void) const; @@ -450,6 +457,9 @@ class parse_ll_t /* Input */ void accept_tokens(parse_token_t token1, parse_token_t token2); + /* Report tokenizer errors */ + void report_tokenizer_error(parse_token_t token, const wchar_t *tok_error); + /* Indicate if we hit a fatal error */ bool has_fatal_error(void) const { @@ -558,7 +568,7 @@ void parse_ll_t::acquire_output(parse_node_tree_t *output, parse_error_list_t *e this->symbol_stack.clear(); } -void parse_ll_t::parse_error(parse_token_t token, const wchar_t *fmt, ...) +void parse_ll_t::parse_error(parse_token_t token, parse_error_code_t code, const wchar_t *fmt, ...) { this->fatal_errored = true; if (this->should_generate_error_messages) @@ -569,6 +579,7 @@ void parse_ll_t::parse_error(parse_token_t token, const wchar_t *fmt, ...) va_list va; va_start(va, fmt); err.text = vformat_string(fmt, va); + err.code = code; va_end(va); err.source_start = token.source_start; @@ -577,6 +588,42 @@ void parse_ll_t::parse_error(parse_token_t token, const wchar_t *fmt, ...) } } +// Unbalancing token. This includes 'else' or 'case' or 'end' outside of the appropriate block +// This essentially duplicates some logic from resolving the production for symbol_statement_list - yuck +void parse_ll_t::parse_error_unbalancing_token(parse_token_t token) +{ + this->fatal_errored = true; + if (this->should_generate_error_messages) + { + assert(token.type == parse_token_type_string); + assert(token.keyword == parse_keyword_end || token.keyword == parse_keyword_else || token.keyword == parse_keyword_case); + switch (token.keyword) + { + case parse_keyword_end: + this->parse_error(token, parse_error_unbalancing_end, L"'end' outside of a block"); + break; + + case parse_keyword_else: + this->parse_error(token, parse_error_unbalancing_else, L"'else' builtin not inside of if block"); + break; + + case parse_keyword_case: + this->parse_error(token, parse_error_unbalancing_case, L"'case' builtin not inside of if block"); + break; + + default: + fprintf(stderr, "Unexpected token %ls passed to %s\n", token.describe().c_str(), __FUNCTION__); + PARSER_DIE(); + break; + } + } +} + +void parse_ll_t::report_tokenizer_error(parse_token_t token, const wchar_t *tok_error) +{ + assert(tok_error != NULL); + this->parse_error(token, parse_error_tokenizer, L"%ls", tok_error); +} void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) { @@ -584,11 +631,7 @@ void parse_ll_t::parse_error(const wchar_t *expected, parse_token_t token) if (this->should_generate_error_messages) { wcstring desc = token_type_description(token.type); - parse_error_t error; - error.text = format_string(L"Expected a %ls, instead got a token of type %ls", expected, desc.c_str()); - error.source_start = token.source_start; - error.source_start = token.source_length; - errors.push_back(error); + this->parse_error(token, parse_error_generic, L"Expected a %ls, instead got a token of type %ls", expected, desc.c_str()); } } @@ -629,13 +672,6 @@ static bool type_is_terminal_type(parse_token_type_t type) bool parse_ll_t::top_node_handle_terminal_types(parse_token_t token) { - if (symbol_stack.empty()) - { - // This can come about with an unbalanced 'end' or 'else', which causes us to terminate the outermost job list. - this->fatal_errored = true; - return false; - } - PARSE_ASSERT(! symbol_stack.empty()); PARSE_ASSERT(token.type >= FIRST_PARSE_TOKEN_TYPE); bool handled = false; @@ -674,7 +710,30 @@ bool parse_ll_t::top_node_handle_terminal_types(parse_token_t token) else { // Failure - this->fatal_errored = true; + if (stack_top.type == parse_token_type_string && token.type == parse_token_type_string) + { + // Must be different keywords. We should unify this with the 'matched' computation above. + assert(stack_top.keyword != parse_keyword_none && stack_top.keyword != token.keyword); + const wcstring expected = keyword_description(stack_top.keyword); + wcstring actual; + if (token.keyword == parse_keyword_none) + { + // This is a random other string (not a keyword) + this->parse_error(token, parse_error_generic, L"Expected keyword '%ls'", expected.c_str()); + } + else + { + // Got a real keyword we can report + const wcstring actual = (token.keyword == parse_keyword_none ? token.describe() : keyword_description(token.keyword)); + this->parse_error(token, parse_error_generic, L"Expected keyword '%ls', instead got keyword '%ls'", expected.c_str(), actual.c_str()); + } + } + else + { + const wcstring expected = token_type_description(stack_top.type); + const wcstring actual = token_type_description(token.type); + this->parse_error(expected.c_str(), token); + } } // We handled the token, so pop the symbol stack @@ -734,25 +793,29 @@ void parse_ll_t::accept_tokens(parse_token_t token1, parse_token_t token2) { if (should_generate_error_messages) { - this->parse_error(token1, L"Unable to produce a '%ls' from input '%ls'", stack_elem.describe().c_str(), token1.describe().c_str()); + this->parse_error(token1, parse_error_generic, L"Unable to produce a '%ls' from input '%ls'", stack_elem.describe().c_str(), token1.describe().c_str()); } else { - this->parse_error(token1, NULL); + this->parse_error(token1, parse_error_generic, NULL); } // parse_error sets fatal_errored, which ends the loop } else { + // When a job_list encounters something like 'else', it returns an empty production to return control to the outer block. But if it's unbalanced, then we'll end up with an empty stack! So make sure that doesn't happen. This is the primary mechanism by which we detect e.g. unbalanced end. + if (symbol_stack.size() == 1 && production_is_empty(production)) + { + this->parse_error_unbalancing_token(token1); + break; + } + // Manipulate the symbol stack. // Note that stack_elem is invalidated by popping the stack. symbol_stack_pop_push_production(production); - - // If we end up with an empty stack, something bad happened, like an unbalanced end - if (symbol_stack.empty()) - { - this->parse_error(token1, L"All symbols removed from symbol stack. Likely unbalanced else or end?"); - } + + // Expect to not have an empty stack + assert(! symbol_stack.empty()); } } } @@ -842,13 +905,16 @@ bool parse_t::parse_internal(const wcstring &str, parse_tree_flags_t parse_flags this->parser->set_should_generate_error_messages(errors != NULL); /* Construct the tokenizer */ - tok_flags_t tok_options = TOK_SQUASH_ERRORS; + tok_flags_t tok_options = 0; if (parse_flags & parse_flag_include_comments) tok_options |= TOK_SHOW_COMMENTS; if (parse_flags & parse_flag_accept_incomplete_tokens) tok_options |= TOK_ACCEPT_UNFINISHED; + if (errors == NULL) + tok_options |= TOK_SQUASH_ERRORS; + tokenizer_t tok = tokenizer_t(str.c_str(), tok_options); /* We are an LL(2) parser. We pass two tokens at a time. New tokens come in at index 1. Seed our queue with an initial token at index 1. */ @@ -864,6 +930,12 @@ bool parse_t::parse_internal(const wcstring &str, parse_tree_flags_t parse_flags /* Pass these two tokens. We know that queue[0] is valid; queue[1] may be invalid. */ this->parser->accept_tokens(queue[0], queue[1]); + /* Handle tokenizer errors. This is a hack because really the parser should report this for itself; but it has no way of getting the tokenizer message */ + if (queue[1].type == parse_special_type_tokenizer_error) + { + this->parser->report_tokenizer_error(queue[1], tok_last(&tok)); + } + /* Handle errors */ if (this->parser->has_fatal_error()) { diff --git a/parse_tree.h b/parse_tree.h index d5b48331..6c8f20f7 100644 --- a/parse_tree.h +++ b/parse_tree.h @@ -12,12 +12,10 @@ #include "util.h" #include "common.h" #include "tokenizer.h" +#include "parse_constants.h" #include #include -#define PARSE_ASSERT(a) assert(a) -#define PARSER_DIE() do { fprintf(stderr, "Parser dying!\n"); exit_without_destructors(-1); } while (0) - class parse_node_t; class parse_node_tree_t; typedef size_t node_offset_t; @@ -27,6 +25,9 @@ struct parse_error_t { /** Text of the error */ wcstring text; + + /** Code for the error */ + enum parse_error_code_t code; /** Offset and length of the token in the source code that triggered this error */ size_t source_start; @@ -37,87 +38,6 @@ struct parse_error_t }; typedef std::vector parse_error_list_t; -enum parse_token_type_t -{ - token_type_invalid, - - // Non-terminal tokens - symbol_job_list, - symbol_job, - symbol_job_continuation, - symbol_statement, - symbol_block_statement, - symbol_block_header, - symbol_for_header, - symbol_while_header, - symbol_begin_header, - symbol_function_header, - - symbol_if_statement, - symbol_if_clause, - symbol_else_clause, - symbol_else_continuation, - - symbol_switch_statement, - symbol_case_item_list, - symbol_case_item, - - symbol_boolean_statement, - symbol_decorated_statement, - symbol_plain_statement, - symbol_arguments_or_redirections_list, - symbol_argument_or_redirection, - - symbol_argument_list, - - symbol_argument, - symbol_redirection, - - symbol_optional_background, - - // Terminal types - parse_token_type_string, - parse_token_type_pipe, - parse_token_type_redirection, - parse_token_type_background, - parse_token_type_end, - parse_token_type_terminate, - - // Very special terminal types that don't appear in the production list - parse_special_type_parse_error, - parse_special_type_tokenizer_error, - parse_special_type_comment, - - FIRST_TERMINAL_TYPE = parse_token_type_string, - LAST_TERMINAL_TYPE = parse_token_type_terminate, - - LAST_TOKEN_OR_SYMBOL = parse_token_type_terminate, - FIRST_PARSE_TOKEN_TYPE = parse_token_type_string -}; - -enum parse_keyword_t -{ - parse_keyword_none, - parse_keyword_if, - parse_keyword_else, - parse_keyword_for, - parse_keyword_in, - parse_keyword_while, - parse_keyword_begin, - parse_keyword_function, - parse_keyword_switch, - parse_keyword_case, - parse_keyword_end, - parse_keyword_and, - parse_keyword_or, - parse_keyword_not, - parse_keyword_command, - parse_keyword_builtin, - - LAST_KEYWORD = parse_keyword_builtin -}; - - /** A struct representing the token type that we use internally */ struct parse_token_t { @@ -233,14 +153,6 @@ public: } }; -/* Statement decorations. This matches the order of productions in decorated_statement */ -enum parse_statement_decoration_t -{ - parse_statement_decoration_none, - parse_statement_decoration_command, - parse_statement_decoration_builtin -}; - /* The parse tree itself */ class parse_node_tree_t : public std::vector diff --git a/parser.cpp b/parser.cpp index 02a4dffc..221b93da 100644 --- a/parser.cpp +++ b/parser.cpp @@ -86,11 +86,6 @@ The fish parser. Contains functions for parsing and evaluating code. */ #define BLOCK_END_ERR_MSG _( L"Could not locate end of block. The 'end' command is missing, misspelled or a ';' is missing.") -/** - Error message on reaching maximum number of block calls -*/ -#define BLOCK_ERR_MSG _( L"Maximum number of nested blocks reached.") - /** Error message when a non-string token is found when expecting a command name */ @@ -2572,9 +2567,9 @@ void parser_t::eval_job(tokenizer_t *tok) } -int parser_t::eval(const wcstring &cmdStr, const io_chain_t &io, enum block_type_t block_type) +int parser_t::eval(const wcstring &cmd_str, const io_chain_t &io, enum block_type_t block_type) { - const wchar_t * const cmd = cmdStr.c_str(); + const wchar_t * const cmd = cmd_str.c_str(); size_t forbid_count; int code; block_t *start_current_block = current_block; @@ -2597,13 +2592,6 @@ int parser_t::eval(const wcstring &cmdStr, const io_chain_t &io, enum block_type debug(4, L"eval: %ls", cmd); - if (!cmd) - { - debug(1, - EVAL_NULL_ERR_MSG); - bugreport(); - return 1; - } if ((block_type != TOP) && (block_type != SUBST)) diff --git a/tokenizer.cpp b/tokenizer.cpp index 113f8926..0705e620 100644 --- a/tokenizer.cpp +++ b/tokenizer.cpp @@ -96,16 +96,8 @@ int tok_get_error(tokenizer_t *tok) tokenizer_t::tokenizer_t(const wchar_t *b, tok_flags_t flags) : buff(NULL), orig_buff(NULL), last_type(TOK_NONE), last_pos(0), has_next(false), accept_unfinished(false), show_comments(false), last_quote(0), error(0), squash_errors(false), cached_lineno_offset(0), cached_lineno_count(0) { - - /* We can only generate error messages on the main thread due to wgettext() thread safety issues. */ - if (!(flags & TOK_SQUASH_ERRORS)) - { - ASSERT_IS_MAIN_THREAD(); - } - CHECK(b,); - this->accept_unfinished = !!(flags & TOK_ACCEPT_UNFINISHED); this->show_comments = !!(flags & TOK_SHOW_COMMENTS); this->squash_errors = !!(flags & TOK_SQUASH_ERRORS); diff --git a/wutil.cpp b/wutil.cpp index 3f70368e..ffb4f2b2 100644 --- a/wutil.cpp +++ b/wutil.cpp @@ -476,7 +476,7 @@ const wchar_t *wgettext(const wchar_t *in) { cstring mbs_in = wcs2string(key); char *out = fish_gettext(mbs_in.c_str()); - val = new wcstring(format_string(L"%s", out)); + val = new wcstring(format_string(L"%s", out)); //note that this writes into the map! } errno = err; return val->c_str(); -- cgit v1.2.3 From 5b1a53265233f837e272aa0936267e04c8848856 Mon Sep 17 00:00:00 2001 From: ridiculousfish Date: Tue, 24 Dec 2013 13:17:24 -0800 Subject: Factor execution aspects from parser_t to parse_execution_context_t . Still searching for best way to use new parser for execution. --- fish.xcodeproj/project.pbxproj | 8 +- parse_execution.cpp | 601 +++++++++++++++++++++++++++++++++++++++++ parse_execution.h | 76 ++++++ parse_productions.cpp | 4 +- parse_tree.h | 4 +- parser.cpp | 3 +- parser.h | 3 + 7 files changed, 693 insertions(+), 6 deletions(-) create mode 100644 parse_execution.cpp create mode 100644 parse_execution.h (limited to 'fish.xcodeproj/project.pbxproj') diff --git a/fish.xcodeproj/project.pbxproj b/fish.xcodeproj/project.pbxproj index aa7f9c18..ceb694ee 100644 --- a/fish.xcodeproj/project.pbxproj +++ b/fish.xcodeproj/project.pbxproj @@ -65,6 +65,7 @@ D033781115DC6D4C00A634BA /* completions in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02715D1FEA100B9DB63 /* completions */; }; D033781215DC6D5200A634BA /* functions in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02815D1FEA100B9DB63 /* functions */; }; D033781315DC6D5400A634BA /* tools in CopyFiles */ = {isa = PBXBuildFile; fileRef = D025C02915D1FEA100B9DB63 /* tools */; }; + D052D80B1868F7FC003ABCBD /* parse_execution.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D052D8091868F7FC003ABCBD /* parse_execution.cpp */; }; D07B247315BCC15700D4ADB4 /* add-shell in Resources */ = {isa = PBXBuildFile; fileRef = D07B247215BCC15700D4ADB4 /* add-shell */; }; D07B247615BCC4BE00D4ADB4 /* install.sh in Resources */ = {isa = PBXBuildFile; fileRef = D07B247515BCC4BE00D4ADB4 /* install.sh */; }; D07D266A15E33B86009E43F6 /* config.fish in CopyFiles */ = {isa = PBXBuildFile; fileRef = D0C4FD9415A7D7EE00212EF1 /* config.fish */; }; @@ -386,6 +387,8 @@ D025C02915D1FEA100B9DB63 /* tools */ = {isa = PBXFileReference; lastKnownFileType = folder; name = tools; path = share/tools; sourceTree = ""; }; D031890915E36D9800D9CC39 /* base */ = {isa = PBXFileReference; lastKnownFileType = text; path = base; sourceTree = BUILT_PRODUCTS_DIR; }; D03EE83814DF88B200FC7150 /* lru.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = lru.h; sourceTree = ""; }; + D052D8091868F7FC003ABCBD /* parse_execution.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = parse_execution.cpp; sourceTree = ""; }; + D052D80A1868F7FC003ABCBD /* parse_execution.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = parse_execution.h; sourceTree = ""; }; D07B247215BCC15700D4ADB4 /* add-shell */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = "add-shell"; path = "build_tools/osx_package_scripts/add-shell"; sourceTree = ""; }; D07B247515BCC4BE00D4ADB4 /* install.sh */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.script.sh; name = install.sh; path = osx/install.sh; sourceTree = ""; }; D0879AC616BF9A1A00E98E56 /* fish_term_icon.icns */ = {isa = PBXFileReference; lastKnownFileType = image.icns; name = fish_term_icon.icns; path = osx/fish_term_icon.icns; sourceTree = ""; }; @@ -658,11 +661,13 @@ D0A0853C13B3ACEE0099B651 /* exec.cpp */, D0A0850C13B3ACEE0099B651 /* expand.h */, D0A0853D13B3ACEE0099B651 /* expand.cpp */, + D0D9B2B318555D92001AE279 /* parse_constants.h */, D0FE8EE6179CA8A5008C9F21 /* parse_productions.h */, D0FE8EE7179FB75F008C9F21 /* parse_productions.cpp */, - D0D9B2B318555D92001AE279 /* parse_constants.h */, D0C52F361765284C00BFAB82 /* parse_tree.h */, D0C52F351765284C00BFAB82 /* parse_tree.cpp */, + D052D80A1868F7FC003ABCBD /* parse_execution.h */, + D052D8091868F7FC003ABCBD /* parse_execution.cpp */, D0A0850D13B3ACEE0099B651 /* fallback.h */, D0A0853E13B3ACEE0099B651 /* fallback.cpp */, D0A0850E13B3ACEE0099B651 /* function.h */, @@ -1222,6 +1227,7 @@ D0D02A83159839D5008E62BD /* iothread.cpp in Sources */, D0D02A84159839D5008E62BD /* parse_util.cpp in Sources */, D0D02A85159839D5008E62BD /* path.cpp in Sources */, + D052D80B1868F7FC003ABCBD /* parse_execution.cpp in Sources */, D0D02A86159839D5008E62BD /* postfork.cpp in Sources */, D0D02A87159839D5008E62BD /* screen.cpp in Sources */, D0D02A88159839D5008E62BD /* signal.cpp in Sources */, diff --git a/parse_execution.cpp b/parse_execution.cpp new file mode 100644 index 00000000..9bcce5be --- /dev/null +++ b/parse_execution.cpp @@ -0,0 +1,601 @@ +/**\file parse_execution.cpp + + Provides the "linkage" between a parse_node_tree_t and actual execution structures (job_t, etc.). + +*/ + +#include "parse_execution.h" +#include "complete.h" +#include "builtin.h" +#include "parser.h" +#include "expand.h" +#include "wutil.h" +#include "path.h" + + +parse_execution_context_t::parse_execution_context_t(const parse_node_tree_t &t, const wcstring s, parser_t *p) : tree(t), src(s), parser(p) +{ +} + +/* Utilities */ + +wcstring parse_execution_context_t::get_source(const parse_node_t &node) const +{ + return node.get_source(this->src); +} + +const parse_node_t *parse_execution_context_t::get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type) +{ + return this->tree.get_child(parent, which, expected_type); +} + +node_offset_t parse_execution_context_t::get_offset(const parse_node_t &node) const +{ + /* Pointer arithmetic, very hackish */ + const parse_node_t *addr = &node; + const parse_node_t *base = &this->tree.at(0); + assert(addr >= base); + node_offset_t offset = addr - base; + assert(offset < this->tree.size()); + return offset; +} + +/* Stack manipulation */ + +void parse_execution_context_t::stack_push(const parse_node_t *job_or_job_list, statement_completion_handler_t completion_handler, const parse_node_t *node) +{ + const struct parse_execution_stack_element_t elem = {job_or_job_list, completion_handler, node}; + job_stack.push_back(elem); +} + +process_t *parse_execution_context_t::create_for_process(job_t *job, const parse_node_t &header, const parse_node_t &statement) +{ + assert(header.type == symbol_for_header); + const wcstring for_variable = get_source(*get_child(header, 1, parse_token_type_string)); + const parse_node_t &arg_list = *get_child(header, 3, symbol_argument_list); + + for_block_t *fb = new for_block_t(for_variable); + fb->sequence = this->determine_arguments(arg_list, NULL); + fb->node_offset = this->get_offset(statement); + parser->push_block(fb); + return NULL; +} + +process_t *parse_execution_context_t::create_while_process(job_t *job, const parse_node_t &header, const parse_node_t &statement) +{ + assert(header.type == symbol_while_header); + while_block_t *wb = new while_block_t(); + wb->status = WHILE_TEST_FIRST; + wb->node_offset = this->get_offset(statement); + parser->push_block(wb); + return NULL; +} + +process_t *parse_execution_context_t::create_begin_process(job_t *job, const parse_node_t &header, const parse_node_t &statement) +{ + assert(header.type == symbol_begin_header); + scope_block_t *bb = new scope_block_t(BEGIN); + parser->push_block(bb); + return NULL; +} + +bool parse_execution_context_t::append_error(const parse_node_t &node, const wchar_t *fmt, ...) +{ + parse_error_t error; + error.source_start = node.source_start; + error.source_length = node.source_length; + error.code = parse_error_syntax; //hackish + + va_list va; + va_start(va, fmt); + error.text = vformat_string(fmt, va); + va_end(va); + + this->errors.push_back(error); + return true; +} + +process_t *parse_execution_context_t::create_plain_process(job_t *job, const parse_node_t &statement) +{ + /* Get the decoration */ + assert(statement.type == symbol_plain_statement); + + /* Get the command. We expect to always get it here. */ + wcstring cmd; + bool got_cmd = tree.command_for_plain_statement(statement, src, &cmd); + assert(got_cmd); + + /* Expand it as a command */ + bool expanded = expand_one(cmd, EXPAND_SKIP_CMDSUBST | EXPAND_SKIP_VARIABLES); + if (! expanded) + { + append_error(statement, ILLEGAL_CMD_ERR_MSG, cmd.c_str()); + return 0; + } + + /* The list of arguments. The command is the first argument. TODO: count hack */ + const parse_node_t *unmatched_wildcard = NULL; + wcstring_list_t argument_list = this->determine_arguments(statement, &unmatched_wildcard); + argument_list.insert(argument_list.begin(), cmd); + + /* We were not able to expand any wildcards. Here is the first one that failed */ + if (unmatched_wildcard != NULL) + { + job_set_flag(job, JOB_WILDCARD_ERROR, 1); + proc_set_last_status(STATUS_UNMATCHED_WILDCARD); + append_error(*unmatched_wildcard, WILDCARD_ERR_MSG, unmatched_wildcard->get_source(src).c_str()); + } + + /* The set of IO redirections that we construct for the process */ + const io_chain_t process_io_chain = this->determine_io_chain(statement); + + /* Determine the process type, which depends on the statement decoration (command, builtin, etc) */ + enum parse_statement_decoration_t decoration = tree.decoration_for_plain_statement(statement); + enum process_type_t process_type = EXTERNAL; + + /* exec hack */ + if (decoration != parse_statement_decoration_command && cmd == L"exec") + { + /* Either 'builtin exec' or just plain 'exec', and definitely not 'command exec'. Note we don't allow overriding exec with a function. */ + process_type = INTERNAL_EXEC; + } + else if (decoration == parse_statement_decoration_command) + { + /* Always a command */ + process_type = EXTERNAL; + } + else if (decoration == parse_statement_decoration_builtin) + { + /* What happens if this builtin is not valid? */ + process_type = INTERNAL_BUILTIN; + } + else if (function_exists(cmd)) + { + process_type = INTERNAL_FUNCTION; + } + else if (builtin_exists(cmd)) + { + process_type = INTERNAL_BUILTIN; + } + else + { + process_type = EXTERNAL; + } + + wcstring actual_cmd; + if (process_type == EXTERNAL) + { + /* Determine the actual command. Need to support implicit cd here */ + bool has_command = path_get_path(cmd, &actual_cmd); + + if (! has_command) + { + /* TODO: support fish_command_not_found, implicit cd, etc. here */ + } + + } + + /* Return the process */ + process_t *result = new process_t(); + result->type = process_type; + result->set_argv(argument_list); + result->set_io_chain(process_io_chain); + result->actual_cmd = actual_cmd; + return result; +} + +/* Determine the list of arguments, expanding stuff. If we have a wildcard and none could be expanded, return the unexpandable wildcard node by reference. */ +wcstring_list_t parse_execution_context_t::determine_arguments(const parse_node_t &parent, const parse_node_t **out_unmatched_wildcard_node) +{ + wcstring_list_t argument_list; + + /* Whether we failed to match any wildcards, and succeeded in matching any wildcards */ + bool unmatched_wildcard = false, matched_wildcard = false; + + /* First node that failed to expand as a wildcard (if any) */ + const parse_node_t *unmatched_wildcard_node = NULL; + + /* Get all argument nodes underneath the statement */ + const parse_node_tree_t::parse_node_list_t argument_nodes = tree.find_nodes(parent, symbol_argument); + argument_list.reserve(argument_nodes.size()); + for (size_t i=0; i < argument_nodes.size(); i++) + { + const parse_node_t &arg_node = *argument_nodes.at(i); + + /* Expect all arguments to have source */ + assert(arg_node.has_source()); + const wcstring arg_str = arg_node.get_source(src); + + /* Expand this string */ + std::vector arg_expanded; + int expand_ret = expand_string(arg_str, arg_expanded, 0); + switch (expand_ret) + { + case EXPAND_ERROR: + { + this->append_error(arg_node, + _(L"Could not expand string '%ls'"), + arg_str.c_str()); + break; + } + + case EXPAND_WILDCARD_NO_MATCH: + { + /* Store the node that failed to expand */ + unmatched_wildcard = true; + if (! unmatched_wildcard_node) + { + unmatched_wildcard_node = &arg_node; + } + break; + } + + case EXPAND_WILDCARD_MATCH: + { + matched_wildcard = true; + break; + } + + case EXPAND_OK: + { + break; + } + } + + /* Now copy over any expanded arguments */ + for (size_t i=0; i < arg_expanded.size(); i++) + { + argument_list.push_back(arg_expanded.at(i).completion); + } + } + + /* Return if we had a wildcard problem */ + if (unmatched_wildcard && ! matched_wildcard) + { + *out_unmatched_wildcard_node = unmatched_wildcard_node; + } + + return argument_list; +} + +io_chain_t parse_execution_context_t::determine_io_chain(const parse_node_t &statement) +{ + io_chain_t result; + + /* Get all redirection nodes underneath the statement */ + const parse_node_tree_t::parse_node_list_t redirect_nodes = tree.find_nodes(statement, symbol_redirection); + for (size_t i=0; i < redirect_nodes.size(); i++) + { + const parse_node_t &redirect_node = *redirect_nodes.at(i); + + int source_fd = -1; /* source fd */ + wcstring target; /* file path or target fd */ + enum token_type redirect_type = tree.type_for_redirection(redirect_node, src, &source_fd, &target); + + /* PCA: I can't justify this EXPAND_SKIP_VARIABLES flag. It was like this when I got here. */ + bool target_expanded = expand_one(target, no_exec ? EXPAND_SKIP_VARIABLES : 0); + if (! target_expanded || target.empty()) + { + /* Should improve this error message */ + this->append_error(redirect_node, + _(L"Invalid redirection target: %ls"), + target.c_str()); + } + + + /* Generate the actual IO redirection */ + shared_ptr new_io; + assert(redirect_type != TOK_NONE); + switch (redirect_type) + { + case TOK_REDIRECT_FD: + { + if (target == L"-") + { + new_io.reset(new io_close_t(source_fd)); + } + else + { + wchar_t *end = NULL; + errno = 0; + int old_fd = fish_wcstoi(target.c_str(), &end, 10); + if (old_fd < 0 || errno || *end) + { + this->append_error(redirect_node, + _(L"Requested redirection to something that is not a file descriptor %ls"), + target.c_str()); + } + else + { + new_io.reset(new io_fd_t(source_fd, old_fd)); + } + } + break; + } + + case TOK_REDIRECT_OUT: + case TOK_REDIRECT_APPEND: + case TOK_REDIRECT_IN: + case TOK_REDIRECT_NOCLOB: + { + int oflags = oflags_for_redirection_type(redirect_type); + io_file_t *new_io_file = new io_file_t(source_fd, target, oflags); + new_io.reset(new_io_file); + break; + } + + default: + { + // Should be unreachable + fprintf(stderr, "Unexpected redirection type %ld. aborting.\n", (long)redirect_type); + PARSER_DIE(); + break; + } + } + + /* Append the new_io if we got one */ + if (new_io.get() != NULL) + { + result.push_back(new_io); + } + } + return result; +} + +process_t *parse_execution_context_t::create_boolean_process(job_t *job, const parse_node_t &bool_statement) +{ + // Handle a boolean statement + bool skip_job = false; + assert(bool_statement.type == symbol_boolean_statement); + switch (bool_statement.production_idx) + { + // These magic numbers correspond to productions for boolean_statement + case 0: + // AND. Skip if the last job failed. + skip_job = (proc_get_last_status() != 0); + break; + + case 1: + // OR. Skip if the last job succeeded. + skip_job = (proc_get_last_status() == 0); + break; + + case 2: + // NOT. Negate it. + job_set_flag(job, JOB_NEGATE, !job_get_flag(job, JOB_NEGATE)); + break; + + default: + { + fprintf(stderr, "Unexpected production in boolean statement\n"); + PARSER_DIE(); + break; + } + } + + process_t *result = NULL; + if (! skip_job) + { + const parse_node_t &subject = *tree.get_child(bool_statement, 1, symbol_statement); + result = this->create_job_process(job, subject); + } + return result; +} + + +/* Returns a process_t allocated with new. It's the caller's responsibility to delete it (!) */ +process_t *parse_execution_context_t::create_job_process(job_t *job, const parse_node_t &statement_node) +{ + assert(statement_node.type == symbol_statement); + assert(statement_node.child_count == 1); + + // Get the "specific statement" which is boolean / block / if / switch / decorated + const parse_node_t &specific_statement = *get_child(statement_node, 0); + + process_t *result = NULL; + + switch (specific_statement.type) + { + case symbol_boolean_statement: + { + result = this->create_boolean_process(job, specific_statement); + break; + } + + case symbol_block_statement: + { + const parse_node_t &header = *get_child(specific_statement, 0, symbol_block_header); + const parse_node_t &specific_header = *get_child(header, 0); + switch (specific_header.type) + { + case symbol_for_header: + result = this->create_for_process(job, specific_header, specific_statement); + break; + + case symbol_while_header: + result = this->create_while_process(job, specific_header, specific_statement); + break; + + case symbol_function_header: + // No process is associated with creating a function + // TODO: create the darn function! + result = NULL; + break; + + case symbol_begin_header: + result = this->create_begin_process(job, specific_header, specific_statement); + break; + + default: + fprintf(stderr, "Unexpected header type\n"); + PARSER_DIE(); + break; + } + break; + } + + case symbol_decorated_statement: + { + const parse_node_t &plain_statement = tree.find_child(specific_statement, symbol_plain_statement); + result = this->create_plain_process(job, plain_statement); + break; + } + + default: + fprintf(stderr, "'%ls' not handled by new parser yet\n", specific_statement.describe().c_str()); + } + + return result; +} + + +void parse_execution_context_t::eval_job(job_t *j, const parse_node_t &job_node) +{ + assert(job_node.type == symbol_job); + + /* Track whether we had an error */ + bool process_errored = false; + + /* Tell the job what its command is */ + j->set_command(get_source(job_node)); + + /* We are going ot construct process_t structures for every statement in the job. Get the first statement. */ + const parse_node_t *statement_node = get_child(job_node, 0, symbol_statement); + assert(statement_node != NULL); + + /* Create the process (may fail!) */ + j->first_process = this->create_job_process(j, *statement_node); + if (j->first_process == NULL) + process_errored = true; + + /* Construct process_ts for job continuations (pipelines), by walking the list until we hit the terminal (empty) job continuationf */ + const parse_node_t *job_cont = get_child(job_node, 1, symbol_job_continuation); + process_t *last_process = j->first_process; + while (! process_errored && job_cont != NULL && job_cont->child_count > 0) + { + assert(job_cont->type == symbol_job_continuation); + + /* Get the statement node and make a process from it */ + const parse_node_t *statement_node = get_child(*job_cont, 1, symbol_statement); + assert(statement_node != NULL); + + /* Store the new process (and maybe with an error) */ + last_process->next = this->create_job_process(j, *statement_node); + if (last_process->next == NULL) + process_errored = true; + + /* Link the process and get the next continuation */ + last_process = last_process->next; + job_cont = get_child(*job_cont, 2, symbol_job_continuation); + } +} + +void parse_execution_context_t::eval_1_job(const parse_node_t &job_node) +{ + // Get terminal modes + struct termios tmodes = {}; + if (get_is_interactive()) + { + if (tcgetattr(STDIN_FILENO, &tmodes)) + { + // need real error handling here + wperror(L"tcgetattr"); + return; + } + } + + /* Profiling support */ + long long t1 = 0, t2 = 0, t3 = 0; + const bool do_profile = profile; + profile_item_t *profile_item = NULL; + if (do_profile) + { + profile_item = new profile_item_t(); + profile_item->skipped = 1; + profile_items.push_back(profile_item); + t1 = get_time(); + } + + job_t *j = parser->job_create(); + job_set_flag(j, JOB_FOREGROUND, 1); + job_set_flag(j, JOB_TERMINAL, job_get_flag(j, JOB_CONTROL)); + job_set_flag(j, JOB_TERMINAL, job_get_flag(j, JOB_CONTROL) \ + && (!is_subshell && !is_event)); + job_set_flag(j, JOB_SKIP_NOTIFICATION, is_subshell \ + || is_block \ + || is_event \ + || (!get_is_interactive())); + + parser->current_block()->job = j; + + this->eval_job(j, job_node); + +} + +void parse_execution_context_t::eval_next_stack_elem() +{ + // Pop the next thing to do + assert(! job_stack.empty()); + const parse_execution_stack_element_t elem = job_stack.back(); + job_stack.pop_back(); + + assert(elem.job_or_job_list->type == symbol_job || elem.job_or_job_list->type == symbol_job_list); + + if (elem.job_or_job_list->type == symbol_job) + { + const parse_node_t *job = elem.job_or_job_list; + this->eval_1_job(*job); + } + else + { + const parse_node_t *job_list = elem.job_or_job_list; + while (job_list != NULL) + { + assert(job_list->type == symbol_job_list); + + // These correspond to the three productions of job_list + // Try pulling out a job + const parse_node_t *job = NULL; + switch (job_list->production_idx) + { + case 0: // empty + job_list = NULL; + break; + + case 1: //job, job_list + job = get_child(*job_list, 0, symbol_job); + job_list = get_child(*job_list, 1, symbol_job_list); + break; + + case 2: //blank line, job_list + job = NULL; + job_list = get_child(*job_list, 1, symbol_job_list); + break; + + default: //if we get here, it means more productions have been added to job_list, which is bad + PARSER_DIE(); + } + + if (job != NULL) + { + this->eval_1_job(*job); + } + } + } + + /* Invoke any completion handler */ + if (elem.completion_handler) + { + assert(elem.node != NULL); + (this->*elem.completion_handler)(*elem.node); + } +} + +void parse_execution_context_t::eval_job_list(const parse_node_t &job_node) +{ + this->stack_push(&job_node, NULL, NULL); + while (! job_stack.empty()) + { + this->eval_next_stack_elem(); + } +} diff --git a/parse_execution.h b/parse_execution.h new file mode 100644 index 00000000..6495f944 --- /dev/null +++ b/parse_execution.h @@ -0,0 +1,76 @@ +/**\file parse_execution.h + + Provides the "linkage" between a parse_node_tree_t and actual execution structures (job_t, etc.). +*/ + +#ifndef FISH_PARSE_EXECUTION_H +#define FISH_PARSE_EXECUTION_H + +#include "config.h" +#include "util.h" +#include "parse_tree.h" +#include "proc.h" + +class job_t; +struct profile_item_t; + +class parse_execution_context_t +{ + private: + const parse_node_tree_t tree; + const wcstring src; + parser_t * const parser; + parse_error_list_t errors; + + std::vector profile_items; + + /* We maintain a stack of job lists to be executed, and something to do after the execution is finished. This is a pointer to member function that takes a node, a status, and the statement that was executed */ + typedef void (parse_execution_context_t::*statement_completion_handler_t)(const parse_node_t &node); + + struct parse_execution_stack_element_t + { + // These point into our tree, which is immutable + const parse_node_t *job_or_job_list; + statement_completion_handler_t completion_handler; + const parse_node_t *node; + }; + std::vector job_stack; + + void stack_push(const parse_node_t *job_or_job_list, statement_completion_handler_t completion_handler, const parse_node_t *node); + + /* No copying allowed */ + parse_execution_context_t(const parse_execution_context_t&); + parse_execution_context_t& operator=(const parse_execution_context_t&); + + /* Report an error. Always returns true. */ + bool append_error(const parse_node_t &node, const wchar_t *fmt, ...); + + wcstring get_source(const parse_node_t &node) const; + const parse_node_t *get_child(const parse_node_t &parent, node_offset_t which, parse_token_type_t expected_type = token_type_invalid); + + node_offset_t get_offset(const parse_node_t &node) const; + + process_t *create_job_process(job_t *job, const parse_node_t &statement_node); + process_t *create_boolean_process(job_t *job, const parse_node_t &bool_statement); + process_t *create_for_process(job_t *job, const parse_node_t &header, const parse_node_t &statement); + process_t *create_while_process(job_t *job, const parse_node_t &header, const parse_node_t &statement); + process_t *create_begin_process(job_t *job, const parse_node_t &header, const parse_node_t &statement); + process_t *create_plain_process(job_t *job, const parse_node_t &statement); + + wcstring_list_t determine_arguments(const parse_node_t &parent, const parse_node_t **out_unmatched_wildcard_node); + io_chain_t determine_io_chain(const parse_node_t &statement); + + void eval_1_job(const parse_node_t &job_node); + void eval_job(job_t *j, const parse_node_t &job_node); + + void eval_next_stack_elem(); + + public: + parse_execution_context_t(const parse_node_tree_t &t, const wcstring s, parser_t *p); + + void eval_job_list(const parse_node_t &job_node); + +}; + + +#endif diff --git a/parse_productions.cpp b/parse_productions.cpp index f79e945a..3325f50d 100644 --- a/parse_productions.cpp +++ b/parse_productions.cpp @@ -302,13 +302,13 @@ RESOLVE(block_header) PRODUCTIONS(for_header) = { - {KEYWORD(parse_keyword_for), parse_token_type_string, KEYWORD(parse_keyword_in), symbol_arguments_or_redirections_list} + {KEYWORD(parse_keyword_for), parse_token_type_string, KEYWORD(parse_keyword_in), symbol_argument_list} }; RESOLVE_ONLY(for_header) PRODUCTIONS(while_header) = { - {KEYWORD(parse_keyword_while), symbol_statement} + {KEYWORD(parse_keyword_while), symbol_job} }; RESOLVE_ONLY(while_header) diff --git a/parse_tree.h b/parse_tree.h index 87e3ecec..9a5d7c23 100644 --- a/parse_tree.h +++ b/parse_tree.h @@ -241,8 +241,8 @@ public: block_statement = block_header job_list end_command arguments_or_redirections_list block_header = for_header | while_header | function_header | begin_header - for_header = FOR var_name IN arguments_or_redirections_list - while_header = WHILE statement + for_header = FOR var_name IN argument_list + while_header = WHILE job begin_header = BEGIN function_header = FUNCTION function_name argument_list diff --git a/parser.cpp b/parser.cpp index 2a34f0dc..821d86c2 100644 --- a/parser.cpp +++ b/parser.cpp @@ -2772,7 +2772,7 @@ void parser_t::eval_job(const parse_node_t &job_node, const parser_context_t &ct /* Tell the job what its command is */ j->set_command(job_node.get_source(ctx.src)); - /* Construct process_t structures for every statement in the job */ + /* We are going to construct process_t structures for every statement in the job. Get the first statement. */ const parse_node_t *statement_node = ctx.tree.get_child(job_node, 0, symbol_statement); assert(statement_node != NULL); @@ -3616,6 +3616,7 @@ block_t::block_t(block_type_t t) : skip(), had_command(), tok_pos(), + node_offset(NODE_OFFSET_INVALID), loop_status(), job(), src_filename(), diff --git a/parser.h b/parser.h index fab73acf..cdb4b590 100644 --- a/parser.h +++ b/parser.h @@ -95,6 +95,8 @@ public: bool skip; /**< Whether execution of the commands in this block should be skipped */ bool had_command; /**< Set to non-zero once a command has been executed in this block */ int tok_pos; /**< The start index of the block */ + + node_offset_t node_offset; /* Offset of the node */ /** Status for the current loop block. Can be any of the values from the loop_status enum. */ int loop_status; @@ -277,6 +279,7 @@ struct parser_context_t class parser_t { + friend class parse_execution_context_t; private: enum parser_type_t parser_type; -- cgit v1.2.3