Improvements to new parser. All functions and completions now parse.

1 files changed, 4 insertions, 1 deletions

diff --git a/tokenizer.h b/tokenizer.h
index f2d6c0c0..0f3ff369 100644
--- a/tokenizer.h
+++ b/tokenizer.h
@@ -142,7 +142,10 @@ int tok_has_next(tokenizer_t *tok);
 /**
   Returns the position of the beginning of the current token in the original string
 */
-int tok_get_pos(tokenizer_t *tok);
+int tok_get_pos(const tokenizer_t *tok);
+
+/** Returns the extent of the current token */
+size_t tok_get_extent(const tokenizer_t *tok);
 
 /**
    Returns the original string to tokenizer