1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
package css // import "github.com/tdewolff/parse/css"
import (
"bytes"
"fmt"
"io"
"testing"
"github.com/tdewolff/test"
)
type TTs []TokenType
func TestTokens(t *testing.T) {
var tokenTests = []struct {
css string
expected []TokenType
}{
{" ", TTs{}},
{"5.2 .4", TTs{NumberToken, NumberToken}},
{"color: red;", TTs{IdentToken, ColonToken, IdentToken, SemicolonToken}},
{"background: url(\"http://x\");", TTs{IdentToken, ColonToken, URLToken, SemicolonToken}},
{"background: URL(x.png);", TTs{IdentToken, ColonToken, URLToken, SemicolonToken}},
{"color: rgb(4, 0%, 5em);", TTs{IdentToken, ColonToken, FunctionToken, NumberToken, CommaToken, PercentageToken, CommaToken, DimensionToken, RightParenthesisToken, SemicolonToken}},
{"body { \"string\" }", TTs{IdentToken, LeftBraceToken, StringToken, RightBraceToken}},
{"body { \"str\\\"ing\" }", TTs{IdentToken, LeftBraceToken, StringToken, RightBraceToken}},
{".class { }", TTs{DelimToken, IdentToken, LeftBraceToken, RightBraceToken}},
{"#class { }", TTs{HashToken, LeftBraceToken, RightBraceToken}},
{"#class\\#withhash { }", TTs{HashToken, LeftBraceToken, RightBraceToken}},
{"@media print { }", TTs{AtKeywordToken, IdentToken, LeftBraceToken, RightBraceToken}},
{"/*comment*/", TTs{CommentToken}},
{"/*com* /ment*/", TTs{CommentToken}},
{"~= |= ^= $= *=", TTs{IncludeMatchToken, DashMatchToken, PrefixMatchToken, SuffixMatchToken, SubstringMatchToken}},
{"||", TTs{ColumnToken}},
{"<!-- -->", TTs{CDOToken, CDCToken}},
{"U+1234", TTs{UnicodeRangeToken}},
{"5.2 .4 4e-22", TTs{NumberToken, NumberToken, NumberToken}},
{"--custom-variable", TTs{CustomPropertyNameToken}},
// unexpected ending
{"ident", TTs{IdentToken}},
{"123.", TTs{NumberToken, DelimToken}},
{"\"string", TTs{StringToken}},
{"123/*comment", TTs{NumberToken, CommentToken}},
{"U+1-", TTs{IdentToken, NumberToken, DelimToken}},
// unicode
{"fooδbar", TTs{IdentToken}},
{"foo\\æ\\†", TTs{IdentToken}},
// {"foo\x00bar", TTs{IdentToken}},
{"'foo\u554abar'", TTs{StringToken}},
{"\\000026B", TTs{IdentToken}},
{"\\26 B", TTs{IdentToken}},
// hacks
{`\-\mo\z\-b\i\nd\in\g:\url(//business\i\nfo.co.uk\/labs\/xbl\/xbl\.xml\#xss);`, TTs{IdentToken, ColonToken, URLToken, SemicolonToken}},
{"width/**/:/**/ 40em;", TTs{IdentToken, CommentToken, ColonToken, CommentToken, DimensionToken, SemicolonToken}},
{":root *> #quince", TTs{ColonToken, IdentToken, DelimToken, DelimToken, HashToken}},
{"html[xmlns*=\"\"]:root", TTs{IdentToken, LeftBracketToken, IdentToken, SubstringMatchToken, StringToken, RightBracketToken, ColonToken, IdentToken}},
{"body:nth-of-type(1)", TTs{IdentToken, ColonToken, FunctionToken, NumberToken, RightParenthesisToken}},
{"color/*\\**/: blue\\9;", TTs{IdentToken, CommentToken, ColonToken, IdentToken, SemicolonToken}},
{"color: blue !ie;", TTs{IdentToken, ColonToken, IdentToken, DelimToken, IdentToken, SemicolonToken}},
// escapes, null and replacement character
{"c\\\x00olor: white;", TTs{IdentToken, ColonToken, IdentToken, SemicolonToken}},
{"null\\0", TTs{IdentToken}},
{"eof\\", TTs{IdentToken}},
{"\"a\x00b\"", TTs{StringToken}},
{"a\\\x00b", TTs{IdentToken}},
{"url(a\x00b)", TTs{BadURLToken}}, // null character cannot be unquoted
{"/*a\x00b*/", TTs{CommentToken}},
// coverage
{" \n\r\n\r\"\\\r\n\\\r\"", TTs{StringToken}},
{"U+?????? U+ABCD?? U+ABC-DEF", TTs{UnicodeRangeToken, UnicodeRangeToken, UnicodeRangeToken}},
{"U+? U+A?", TTs{IdentToken, DelimToken, DelimToken, IdentToken, DelimToken, IdentToken, DelimToken}},
{"-5.23 -moz", TTs{NumberToken, IdentToken}},
{"()", TTs{LeftParenthesisToken, RightParenthesisToken}},
{"url( //url )", TTs{URLToken}},
{"url( ", TTs{URLToken}},
{"url( //url", TTs{URLToken}},
{"url(\")a", TTs{URLToken}},
{"url(a'\\\n)a", TTs{BadURLToken, IdentToken}},
{"url(\"\n)a", TTs{BadURLToken, IdentToken}},
{"url(a h)a", TTs{BadURLToken, IdentToken}},
{"<!- | @4 ## /2", TTs{DelimToken, DelimToken, DelimToken, DelimToken, DelimToken, NumberToken, DelimToken, DelimToken, DelimToken, NumberToken}},
{"\"s\\\n\"", TTs{StringToken}},
{"\"a\\\"b\"", TTs{StringToken}},
{"\"s\n", TTs{BadStringToken}},
// small
{"\"abcd", TTs{StringToken}},
{"/*comment", TTs{CommentToken}},
{"U+A-B", TTs{UnicodeRangeToken}},
{"url((", TTs{BadURLToken}},
{"id\u554a", TTs{IdentToken}},
}
for _, tt := range tokenTests {
t.Run(tt.css, func(t *testing.T) {
l := NewLexer(bytes.NewBufferString(tt.css))
i := 0
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
test.T(t, i, len(tt.expected), "when error occurred we must be at the end")
break
} else if token == WhitespaceToken {
continue
}
test.That(t, i < len(tt.expected), "index", i, "must not exceed expected token types size", len(tt.expected))
if i < len(tt.expected) {
test.T(t, token, tt.expected[i], "token types must match")
}
i++
}
})
}
test.T(t, WhitespaceToken.String(), "Whitespace")
test.T(t, EmptyToken.String(), "Empty")
test.T(t, CustomPropertyValueToken.String(), "CustomPropertyValue")
test.T(t, TokenType(100).String(), "Invalid(100)")
test.T(t, NewLexer(bytes.NewBufferString("x")).consumeBracket(), ErrorToken, "consumeBracket on 'x' must return error")
}
////////////////////////////////////////////////////////////////
func ExampleNewLexer() {
l := NewLexer(bytes.NewBufferString("color: red;"))
out := ""
for {
tt, data := l.Next()
if tt == ErrorToken {
break
} else if tt == WhitespaceToken || tt == CommentToken {
continue
}
out += string(data)
}
fmt.Println(out)
// Output: color:red;
}
|
