aboutsummaryrefslogtreecommitdiffhomepage
path: root/vendor/github.com/tdewolff/parse/buffer/streamlexer.go
diff options
context:
space:
mode:
Diffstat (limited to 'vendor/github.com/tdewolff/parse/buffer/streamlexer.go')
-rw-r--r--vendor/github.com/tdewolff/parse/buffer/streamlexer.go223
1 files changed, 0 insertions, 223 deletions
diff --git a/vendor/github.com/tdewolff/parse/buffer/streamlexer.go b/vendor/github.com/tdewolff/parse/buffer/streamlexer.go
deleted file mode 100644
index 3ed3282..0000000
--- a/vendor/github.com/tdewolff/parse/buffer/streamlexer.go
+++ /dev/null
@@ -1,223 +0,0 @@
-package buffer // import "github.com/tdewolff/parse/buffer"
-
-import (
- "io"
-)
-
-type block struct {
- buf []byte
- next int // index in pool plus one
- active bool
-}
-
-type bufferPool struct {
- pool []block
- head int // index in pool plus one
- tail int // index in pool plus one
-
- pos int // byte pos in tail
-}
-
-func (z *bufferPool) swap(oldBuf []byte, size int) []byte {
- // find new buffer that can be reused
- swap := -1
- for i := 0; i < len(z.pool); i++ {
- if !z.pool[i].active && size <= cap(z.pool[i].buf) {
- swap = i
- break
- }
- }
- if swap == -1 { // no free buffer found for reuse
- if z.tail == 0 && z.pos >= len(oldBuf) && size <= cap(oldBuf) { // but we can reuse the current buffer!
- z.pos -= len(oldBuf)
- return oldBuf[:0]
- }
- // allocate new
- z.pool = append(z.pool, block{make([]byte, 0, size), 0, true})
- swap = len(z.pool) - 1
- }
-
- newBuf := z.pool[swap].buf
-
- // put current buffer into pool
- z.pool[swap] = block{oldBuf, 0, true}
- if z.head != 0 {
- z.pool[z.head-1].next = swap + 1
- }
- z.head = swap + 1
- if z.tail == 0 {
- z.tail = swap + 1
- }
-
- return newBuf[:0]
-}
-
-func (z *bufferPool) free(n int) {
- z.pos += n
- // move the tail over to next buffers
- for z.tail != 0 && z.pos >= len(z.pool[z.tail-1].buf) {
- z.pos -= len(z.pool[z.tail-1].buf)
- newTail := z.pool[z.tail-1].next
- z.pool[z.tail-1].active = false // after this, any thread may pick up the inactive buffer, so it can't be used anymore
- z.tail = newTail
- }
- if z.tail == 0 {
- z.head = 0
- }
-}
-
-// StreamLexer is a buffered reader that allows peeking forward and shifting, taking an io.Reader.
-// It keeps data in-memory until Free, taking a byte length, is called to move beyond the data.
-type StreamLexer struct {
- r io.Reader
- err error
-
- pool bufferPool
-
- buf []byte
- start int // index in buf
- pos int // index in buf
- prevStart int
-
- free int
-}
-
-// NewStreamLexer returns a new StreamLexer for a given io.Reader with a 4kB estimated buffer size.
-// If the io.Reader implements Bytes, that buffer is used instead.
-func NewStreamLexer(r io.Reader) *StreamLexer {
- return NewStreamLexerSize(r, defaultBufSize)
-}
-
-// NewStreamLexerSize returns a new StreamLexer for a given io.Reader and estimated required buffer size.
-// If the io.Reader implements Bytes, that buffer is used instead.
-func NewStreamLexerSize(r io.Reader, size int) *StreamLexer {
- // if reader has the bytes in memory already, use that instead
- if buffer, ok := r.(interface {
- Bytes() []byte
- }); ok {
- return &StreamLexer{
- err: io.EOF,
- buf: buffer.Bytes(),
- }
- }
- return &StreamLexer{
- r: r,
- buf: make([]byte, 0, size),
- }
-}
-
-func (z *StreamLexer) read(pos int) byte {
- if z.err != nil {
- return 0
- }
-
- // free unused bytes
- z.pool.free(z.free)
- z.free = 0
-
- // get new buffer
- c := cap(z.buf)
- p := pos - z.start + 1
- if 2*p > c { // if the token is larger than half the buffer, increase buffer size
- c = 2*c + p
- }
- d := len(z.buf) - z.start
- buf := z.pool.swap(z.buf[:z.start], c)
- copy(buf[:d], z.buf[z.start:]) // copy the left-overs (unfinished token) from the old buffer
-
- // read in new data for the rest of the buffer
- var n int
- for pos-z.start >= d && z.err == nil {
- n, z.err = z.r.Read(buf[d:cap(buf)])
- d += n
- }
- pos -= z.start
- z.pos -= z.start
- z.start, z.buf = 0, buf[:d]
- if pos >= d {
- return 0
- }
- return z.buf[pos]
-}
-
-// Err returns the error returned from io.Reader. It may still return valid bytes for a while though.
-func (z *StreamLexer) Err() error {
- if z.err == io.EOF && z.pos < len(z.buf) {
- return nil
- }
- return z.err
-}
-
-// Free frees up bytes of length n from previously shifted tokens.
-// Each call to Shift should at one point be followed by a call to Free with a length returned by ShiftLen.
-func (z *StreamLexer) Free(n int) {
- z.free += n
-}
-
-// Peek returns the ith byte relative to the end position and possibly does an allocation.
-// Peek returns zero when an error has occurred, Err returns the error.
-// TODO: inline function
-func (z *StreamLexer) Peek(pos int) byte {
- pos += z.pos
- if uint(pos) < uint(len(z.buf)) { // uint for BCE
- return z.buf[pos]
- }
- return z.read(pos)
-}
-
-// PeekRune returns the rune and rune length of the ith byte relative to the end position.
-func (z *StreamLexer) PeekRune(pos int) (rune, int) {
- // from unicode/utf8
- c := z.Peek(pos)
- if c < 0xC0 {
- return rune(c), 1
- } else if c < 0xE0 {
- return rune(c&0x1F)<<6 | rune(z.Peek(pos+1)&0x3F), 2
- } else if c < 0xF0 {
- return rune(c&0x0F)<<12 | rune(z.Peek(pos+1)&0x3F)<<6 | rune(z.Peek(pos+2)&0x3F), 3
- }
- return rune(c&0x07)<<18 | rune(z.Peek(pos+1)&0x3F)<<12 | rune(z.Peek(pos+2)&0x3F)<<6 | rune(z.Peek(pos+3)&0x3F), 4
-}
-
-// Move advances the position.
-func (z *StreamLexer) Move(n int) {
- z.pos += n
-}
-
-// Pos returns a mark to which can be rewinded.
-func (z *StreamLexer) Pos() int {
- return z.pos - z.start
-}
-
-// Rewind rewinds the position to the given position.
-func (z *StreamLexer) Rewind(pos int) {
- z.pos = z.start + pos
-}
-
-// Lexeme returns the bytes of the current selection.
-func (z *StreamLexer) Lexeme() []byte {
- return z.buf[z.start:z.pos]
-}
-
-// Skip collapses the position to the end of the selection.
-func (z *StreamLexer) Skip() {
- z.start = z.pos
-}
-
-// Shift returns the bytes of the current selection and collapses the position to the end of the selection.
-// It also returns the number of bytes we moved since the last call to Shift. This can be used in calls to Free.
-func (z *StreamLexer) Shift() []byte {
- if z.pos > len(z.buf) { // make sure we peeked at least as much as we shift
- z.read(z.pos - 1)
- }
- b := z.buf[z.start:z.pos]
- z.start = z.pos
- return b
-}
-
-// ShiftLen returns the number of bytes moved since the last call to ShiftLen. This can be used in calls to Free because it takes into account multiple Shifts or Skips.
-func (z *StreamLexer) ShiftLen() int {
- n := z.start - z.prevStart
- z.prevStart = z.start
- return n
-}