diff options
Diffstat (limited to 'vendor/github.com/tdewolff/parse/js')
-rw-r--r-- | vendor/github.com/tdewolff/parse/js/lex.go | 67 | ||||
-rw-r--r-- | vendor/github.com/tdewolff/parse/js/lex_test.go | 15 |
2 files changed, 51 insertions, 31 deletions
diff --git a/vendor/github.com/tdewolff/parse/js/lex.go b/vendor/github.com/tdewolff/parse/js/lex.go index 3ee73e7..ce4e1d5 100644 --- a/vendor/github.com/tdewolff/parse/js/lex.go +++ b/vendor/github.com/tdewolff/parse/js/lex.go @@ -23,7 +23,8 @@ const ( UnknownToken // extra token when no token can be matched WhitespaceToken // space \t \v \f LineTerminatorToken // \r \n \r\n - CommentToken + SingleLineCommentToken + MultiLineCommentToken // token for comments with line terminators (not just any /*block*/) IdentifierToken PunctuatorToken /* { } ( ) [ ] . ; , < > <= >= == != === !== + - * % ++ -- << >> >>> & | ^ ! ~ && || ? : = += -= *= %= <<= >>= >>>= &= |= ^= / /= >= */ @@ -68,8 +69,10 @@ func (tt TokenType) String() string { return "Whitespace" case LineTerminatorToken: return "LineTerminator" - case CommentToken: - return "Comment" + case SingleLineCommentToken: + return "SingleLineComment" + case MultiLineCommentToken: + return "MultiLineComment" case IdentifierToken: return "Identifier" case PunctuatorToken: @@ -174,15 +177,15 @@ func (l *Lexer) Next() (TokenType, []byte) { l.r.Move(1) tt = PunctuatorToken case '<', '>', '=', '!', '+', '-', '*', '%', '&', '|', '^': - if (c == '<' || (l.emptyLine && c == '-')) && l.consumeCommentToken() { - return CommentToken, l.r.Shift() + if l.consumeHTMLLikeCommentToken() { + return SingleLineCommentToken, l.r.Shift() } else if l.consumeLongPunctuatorToken() { l.state = ExprState tt = PunctuatorToken } case '/': - if l.consumeCommentToken() { - return CommentToken, l.r.Shift() + if tt = l.consumeCommentToken(); tt != UnknownToken { + return tt, l.r.Shift() } else if l.state == ExprState && l.consumeRegexpToken() { l.state = SubscriptState tt = RegexpToken @@ -374,46 +377,54 @@ func (l *Lexer) consumeSingleLineComment() { //////////////////////////////////////////////////////////////// -func (l *Lexer) consumeCommentToken() bool { +func (l *Lexer) consumeHTMLLikeCommentToken() bool { + c := l.r.Peek(0) + if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' { + // opening HTML-style single line comment + l.r.Move(4) + l.consumeSingleLineComment() + return true + } else if l.emptyLine && c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' { + // closing HTML-style single line comment + // (only if current line didn't contain any meaningful tokens) + l.r.Move(3) + l.consumeSingleLineComment() + return true + } + return false +} + +func (l *Lexer) consumeCommentToken() TokenType { c := l.r.Peek(0) if c == '/' { c = l.r.Peek(1) if c == '/' { - // single line + // single line comment l.r.Move(2) l.consumeSingleLineComment() + return SingleLineCommentToken } else if c == '*' { - // multi line + // block comment (potentially multiline) + tt := SingleLineCommentToken l.r.Move(2) for { c := l.r.Peek(0) if c == '*' && l.r.Peek(1) == '/' { l.r.Move(2) - return true + break } else if c == 0 { break } else if l.consumeLineTerminator() { + tt = MultiLineCommentToken l.emptyLine = true } else { l.r.Move(1) } } - } else { - return false + return tt } - } else if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' { - // opening HTML-style single line comment - l.r.Move(4) - l.consumeSingleLineComment() - } else if c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' { - // closing HTML-style single line comment - // (only if current line didn't contain any meaningful tokens) - l.r.Move(3) - l.consumeSingleLineComment() - } else { - return false } - return true + return UnknownToken } func (l *Lexer) consumeLongPunctuatorToken() bool { @@ -643,6 +654,12 @@ func (l *Lexer) consumeTemplateToken() bool { l.state = ExprState l.r.Move(2) return true + } else if c == '\\' { + l.r.Move(1) + if c := l.r.Peek(0); c != 0 { + l.r.Move(1) + } + continue } else if c == 0 { l.r.Rewind(mark) return false diff --git a/vendor/github.com/tdewolff/parse/js/lex_test.go b/vendor/github.com/tdewolff/parse/js/lex_test.go index b379321..1866087 100644 --- a/vendor/github.com/tdewolff/parse/js/lex_test.go +++ b/vendor/github.com/tdewolff/parse/js/lex_test.go @@ -20,7 +20,7 @@ func TestTokens(t *testing.T) { {"\n\r\r\n\u2028\u2029", TTs{LineTerminatorToken}}, {"5.2 .04 0x0F 5e99", TTs{NumericToken, NumericToken, NumericToken, NumericToken}}, {"a = 'string'", TTs{IdentifierToken, PunctuatorToken, StringToken}}, - {"/*comment*/ //comment", TTs{CommentToken, CommentToken}}, + {"/*comment*/ //comment", TTs{SingleLineCommentToken, SingleLineCommentToken}}, {"{ } ( ) [ ]", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}}, {". ; , < > <=", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}}, {">= == != === !==", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}}, @@ -31,12 +31,12 @@ func TestTokens(t *testing.T) { {">>= >>>= &= |= ^= =>", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken}}, {"a = /.*/g;", TTs{IdentifierToken, PunctuatorToken, RegexpToken, PunctuatorToken}}, - {"/*co\nm\u2028m/*ent*/ //co//mment\u2029//comment", TTs{CommentToken, CommentToken, LineTerminatorToken, CommentToken}}, + {"/*co\nm\u2028m/*ent*/ //co//mment\u2029//comment", TTs{MultiLineCommentToken, SingleLineCommentToken, LineTerminatorToken, SingleLineCommentToken}}, {"<!-", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken}}, - {"1<!--2\n", TTs{NumericToken, CommentToken, LineTerminatorToken}}, + {"1<!--2\n", TTs{NumericToken, SingleLineCommentToken, LineTerminatorToken}}, {"x=y-->10\n", TTs{IdentifierToken, PunctuatorToken, IdentifierToken, PunctuatorToken, PunctuatorToken, NumericToken, LineTerminatorToken}}, - {" /*comment*/ -->nothing\n", TTs{CommentToken, CommentToken, LineTerminatorToken}}, - {"1 /*comment\nmultiline*/ -->nothing\n", TTs{NumericToken, CommentToken, CommentToken, LineTerminatorToken}}, + {" /*comment*/ -->nothing\n", TTs{SingleLineCommentToken, SingleLineCommentToken, LineTerminatorToken}}, + {"1 /*comment\nmultiline*/ -->nothing\n", TTs{NumericToken, MultiLineCommentToken, SingleLineCommentToken, LineTerminatorToken}}, {"$ _\u200C \\u2000 \u200C", TTs{IdentifierToken, IdentifierToken, IdentifierToken, UnknownToken}}, {">>>=>>>>=", TTs{PunctuatorToken, PunctuatorToken, PunctuatorToken}}, {"1/", TTs{NumericToken, PunctuatorToken}}, @@ -63,7 +63,7 @@ func TestTokens(t *testing.T) { {"'\n '\u2028", TTs{UnknownToken, LineTerminatorToken, UnknownToken, LineTerminatorToken}}, {"'str\\\U00100000ing\\0'", TTs{StringToken}}, {"'strin\\00g'", TTs{StringToken}}, - {"/*comment", TTs{CommentToken}}, + {"/*comment", TTs{SingleLineCommentToken}}, {"a=/regexp", TTs{IdentifierToken, PunctuatorToken, RegexpToken}}, {"\\u002", TTs{UnknownToken, IdentifierToken}}, @@ -97,6 +97,9 @@ func TestTokens(t *testing.T) { {"function f(){}/1/g", TTs{IdentifierToken, IdentifierToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, PunctuatorToken, RegexpToken}}, {"this.return/1/g", TTs{IdentifierToken, PunctuatorToken, IdentifierToken, PunctuatorToken, NumericToken, PunctuatorToken, IdentifierToken}}, {"(a+b)/1/g", TTs{PunctuatorToken, IdentifierToken, PunctuatorToken, IdentifierToken, PunctuatorToken, PunctuatorToken, NumericToken, PunctuatorToken, IdentifierToken}}, + {"`\\``", TTs{TemplateToken}}, + {"`\\${ 1 }`", TTs{TemplateToken}}, + {"`\\\r\n`", TTs{TemplateToken}}, // go fuzz {"`", TTs{UnknownToken}}, |