diff options
Diffstat (limited to 'vendor/github.com/tdewolff/parse/js/lex.go')
| -rw-r--r-- | vendor/github.com/tdewolff/parse/js/lex.go | 67 |
1 files changed, 42 insertions, 25 deletions
diff --git a/vendor/github.com/tdewolff/parse/js/lex.go b/vendor/github.com/tdewolff/parse/js/lex.go index 3ee73e7..ce4e1d5 100644 --- a/vendor/github.com/tdewolff/parse/js/lex.go +++ b/vendor/github.com/tdewolff/parse/js/lex.go @@ -23,7 +23,8 @@ const ( UnknownToken // extra token when no token can be matched WhitespaceToken // space \t \v \f LineTerminatorToken // \r \n \r\n - CommentToken + SingleLineCommentToken + MultiLineCommentToken // token for comments with line terminators (not just any /*block*/) IdentifierToken PunctuatorToken /* { } ( ) [ ] . ; , < > <= >= == != === !== + - * % ++ -- << >> >>> & | ^ ! ~ && || ? : = += -= *= %= <<= >>= >>>= &= |= ^= / /= >= */ @@ -68,8 +69,10 @@ func (tt TokenType) String() string { return "Whitespace" case LineTerminatorToken: return "LineTerminator" - case CommentToken: - return "Comment" + case SingleLineCommentToken: + return "SingleLineComment" + case MultiLineCommentToken: + return "MultiLineComment" case IdentifierToken: return "Identifier" case PunctuatorToken: @@ -174,15 +177,15 @@ func (l *Lexer) Next() (TokenType, []byte) { l.r.Move(1) tt = PunctuatorToken case '<', '>', '=', '!', '+', '-', '*', '%', '&', '|', '^': - if (c == '<' || (l.emptyLine && c == '-')) && l.consumeCommentToken() { - return CommentToken, l.r.Shift() + if l.consumeHTMLLikeCommentToken() { + return SingleLineCommentToken, l.r.Shift() } else if l.consumeLongPunctuatorToken() { l.state = ExprState tt = PunctuatorToken } case '/': - if l.consumeCommentToken() { - return CommentToken, l.r.Shift() + if tt = l.consumeCommentToken(); tt != UnknownToken { + return tt, l.r.Shift() } else if l.state == ExprState && l.consumeRegexpToken() { l.state = SubscriptState tt = RegexpToken @@ -374,46 +377,54 @@ func (l *Lexer) consumeSingleLineComment() { //////////////////////////////////////////////////////////////// -func (l *Lexer) consumeCommentToken() bool { +func (l *Lexer) consumeHTMLLikeCommentToken() bool { + c := l.r.Peek(0) + if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' { + // opening HTML-style single line comment + l.r.Move(4) + l.consumeSingleLineComment() + return true + } else if l.emptyLine && c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' { + // closing HTML-style single line comment + // (only if current line didn't contain any meaningful tokens) + l.r.Move(3) + l.consumeSingleLineComment() + return true + } + return false +} + +func (l *Lexer) consumeCommentToken() TokenType { c := l.r.Peek(0) if c == '/' { c = l.r.Peek(1) if c == '/' { - // single line + // single line comment l.r.Move(2) l.consumeSingleLineComment() + return SingleLineCommentToken } else if c == '*' { - // multi line + // block comment (potentially multiline) + tt := SingleLineCommentToken l.r.Move(2) for { c := l.r.Peek(0) if c == '*' && l.r.Peek(1) == '/' { l.r.Move(2) - return true + break } else if c == 0 { break } else if l.consumeLineTerminator() { + tt = MultiLineCommentToken l.emptyLine = true } else { l.r.Move(1) } } - } else { - return false + return tt } - } else if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' { - // opening HTML-style single line comment - l.r.Move(4) - l.consumeSingleLineComment() - } else if c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' { - // closing HTML-style single line comment - // (only if current line didn't contain any meaningful tokens) - l.r.Move(3) - l.consumeSingleLineComment() - } else { - return false } - return true + return UnknownToken } func (l *Lexer) consumeLongPunctuatorToken() bool { @@ -643,6 +654,12 @@ func (l *Lexer) consumeTemplateToken() bool { l.state = ExprState l.r.Move(2) return true + } else if c == '\\' { + l.r.Move(1) + if c := l.r.Peek(0); c != 0 { + l.r.Move(1) + } + continue } else if c == 0 { l.r.Rewind(mark) return false |