1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
package buffer // import "github.com/tdewolff/parse/buffer"
import (
"bytes"
"io"
"testing"
"github.com/tdewolff/test"
)
func TestBufferPool(t *testing.T) {
z := &bufferPool{}
lorem := []byte("Lorem ipsum")
dolor := []byte("dolor sit amet")
consectetur := []byte("consectetur adipiscing elit")
// set lorem as first buffer and get new dolor buffer
b := z.swap(lorem, len(dolor))
test.That(t, len(b) == 0)
test.That(t, cap(b) == len(dolor))
b = append(b, dolor...)
// free first buffer so it will be reused
z.free(len(lorem))
b = z.swap(b, len(lorem))
b = b[:len(lorem)]
test.Bytes(t, b, lorem)
b = z.swap(b, len(consectetur))
b = append(b, consectetur...)
// free in advance to reuse the same buffer
z.free(len(dolor) + len(lorem) + len(consectetur))
test.That(t, z.head == 0)
b = z.swap(b, len(consectetur))
b = b[:len(consectetur)]
test.Bytes(t, b, consectetur)
// free in advance but request larger buffer
z.free(len(consectetur))
b = z.swap(b, len(consectetur)+1)
b = append(b, consectetur...)
b = append(b, '.')
test.That(t, cap(b) == len(consectetur)+1)
}
func TestStreamLexer(t *testing.T) {
s := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
z := NewStreamLexer(bytes.NewBufferString(s))
test.T(t, z.err, io.EOF, "buffer must be fully in memory")
test.T(t, z.Err(), nil, "buffer is at EOF but must not return EOF until we reach that")
test.That(t, z.Pos() == 0, "buffer must start at position 0")
test.That(t, z.Peek(0) == 'L', "first character must be 'L'")
test.That(t, z.Peek(1) == 'o', "second character must be 'o'")
z.Move(1)
test.That(t, z.Peek(0) == 'o', "must be 'o' at position 1")
test.That(t, z.Peek(1) == 'r', "must be 'r' at position 1")
z.Rewind(6)
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 6")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 7")
test.Bytes(t, z.Lexeme(), []byte("Lorem "), "buffered string must now read 'Lorem ' when at position 6")
test.Bytes(t, z.Shift(), []byte("Lorem "), "shift must return the buffered string")
test.That(t, z.ShiftLen() == len("Lorem "), "shifted length must equal last shift")
test.That(t, z.Pos() == 0, "after shifting position must be 0")
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 0 after shifting")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 1 after shifting")
test.T(t, z.Err(), nil, "error must be nil at this point")
z.Move(len(s) - len("Lorem ") - 1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer")
z.Skip()
test.That(t, z.Pos() == 0, "after skipping position must be 0")
z.Move(1)
test.T(t, z.Err(), io.EOF, "error must be EOF when past the buffer")
z.Move(-1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer, even when it has been past the buffer")
z.Free(0) // has already been tested
}
func TestStreamLexerShift(t *testing.T) {
s := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
z := NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 5)
z.Move(len("Lorem "))
test.Bytes(t, z.Shift(), []byte("Lorem "), "shift must return the buffered string")
test.That(t, z.ShiftLen() == len("Lorem "), "shifted length must equal last shift")
}
func TestStreamLexerSmall(t *testing.T) {
s := `abcdefghijklm`
z := NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 4)
test.That(t, z.Peek(8) == 'i', "first character must be 'i' at position 8")
z = NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 4)
test.That(t, z.Peek(12) == 'm', "first character must be 'm' at position 12")
z = NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 0)
test.That(t, z.Peek(4) == 'e', "first character must be 'e' at position 4")
z = NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 13)
test.That(t, z.Peek(13) == 0, "must yield error at position 13")
}
func TestStreamLexerSingle(t *testing.T) {
z := NewStreamLexer(test.NewInfiniteReader())
test.That(t, z.Peek(0) == '.')
test.That(t, z.Peek(1) == '.')
test.That(t, z.Peek(3) == '.', "required two successful reads")
}
func TestStreamLexerRunes(t *testing.T) {
z := NewStreamLexer(bytes.NewBufferString("aæ†\U00100000"))
r, n := z.PeekRune(0)
test.That(t, n == 1, "first character must be length 1")
test.That(t, r == 'a', "first character must be rune 'a'")
r, n = z.PeekRune(1)
test.That(t, n == 2, "second character must be length 2")
test.That(t, r == 'æ', "second character must be rune 'æ'")
r, n = z.PeekRune(3)
test.That(t, n == 3, "fourth character must be length 3")
test.That(t, r == '†', "fourth character must be rune '†'")
r, n = z.PeekRune(6)
test.That(t, n == 4, "seventh character must be length 4")
test.That(t, r == '\U00100000', "seventh character must be rune '\U00100000'")
}
func TestStreamLexerBadRune(t *testing.T) {
z := NewStreamLexer(bytes.NewBufferString("\xF0")) // expect four byte rune
r, n := z.PeekRune(0)
test.T(t, n, 4, "length")
test.T(t, r, rune(0), "rune")
}
func TestStreamLexerZeroLen(t *testing.T) {
z := NewStreamLexer(test.NewPlainReader(bytes.NewBufferString("")))
test.That(t, z.Peek(0) == 0, "first character must yield error")
}
func TestStreamLexerEmptyReader(t *testing.T) {
z := NewStreamLexer(test.NewEmptyReader())
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.T(t, z.Err(), io.EOF, "error must be EOF")
test.That(t, z.Peek(0) == 0, "second peek must also yield error")
}
|
