Revert "Switch to the new SkSL lexer."

This reverts commit c576e93d174f3106e072a2f506bca3990b541265.

Reason for revert: ASAN failures

Original change's description:
> Switch to the new SkSL lexer.
> 
> This completely replaces flex with a new in-house lexical analyzer generator,
> which we have done for performance and memory usage reasons. Flex requires us
> to copy strings every time we need the text of a token, whereas this new lexer
> allows us to handle strings as a (non-null-terminated) pointer and length
> everywhere, eliminating most string copies.
> 
> Bug: skia:
> Change-Id: I2add26efc9e20cb699520e82abcf713af3968aca
> Reviewed-on: https://skia-review.googlesource.com/39780
> Reviewed-by: Brian Salomon <bsalomon@google.com>
> Commit-Queue: Ethan Nicholas <ethannicholas@google.com>

TBR=bsalomon@google.com,ethannicholas@google.com

Change-Id: If27b750a5f696d06a6bcffed12fe9f0598e084a6
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: skia:
Reviewed-on: https://skia-review.googlesource.com/44881
Reviewed-by: Ethan Nicholas <ethannicholas@google.com>
Commit-Queue: Ethan Nicholas <ethannicholas@google.com>

1 files changed, 186 insertions, 151 deletions

diff --git a/src/sksl/SkSLParser.cpp b/src/sksl/SkSLParser.cpp
index d8ecc618bc..f712b34e1a 100644
--- a/src/sksl/SkSLParser.cpp
+++ b/src/sksl/SkSLParser.cpp
@@ -7,6 +7,26 @@
 
 #include "stdio.h"
 #include "SkSLParser.h"
+#include "SkSLToken.h"
+
+#define register
+#include "disable_flex_warnings.h"
+#include "lex.sksl.c"
+static_assert(YY_FLEX_MAJOR_VERSION * 10000 + YY_FLEX_MINOR_VERSION * 100 +
+              YY_FLEX_SUBMINOR_VERSION >= 20601,
+              "we require Flex 2.6.1 or better for security reasons");
+#undef register
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+#include "lex.layout.h"
 #include "ast/SkSLASTBinaryExpression.h"
 #include "ast/SkSLASTBlock.h"
 #include "ast/SkSLASTBoolLiteral.h"
@@ -63,7 +83,7 @@ public:
 
     bool checkValid() {
         if (fParser->fDepth > MAX_PARSE_DEPTH) {
-            fParser->error(fParser->peek(), String("exceeded max parse depth"));
+            fParser->error(fParser->peek().fPosition, String("exceeded max parse depth"));
             return false;
         }
         return true;
@@ -73,12 +93,20 @@ private:
     Parser* fParser;
 };
 
-Parser::Parser(const char* text, size_t length, SymbolTable& types, ErrorReporter& errors)
-: fText(text)
-, fPushback(Token::INVALID, -1, -1)
+Parser::Parser(String text, SymbolTable& types, ErrorReporter& errors)
+: fPushback(Position(-1, -1), Token::INVALID_TOKEN, String())
 , fTypes(types)
 , fErrors(errors) {
-    fLexer.start(text, length);
+    sksllex_init(&fScanner);
+    layoutlex_init(&fLayoutScanner);
+    fBuffer = sksl_scan_string(text.c_str(), fScanner);
+    skslset_lineno(1, fScanner);
+}
+
+Parser::~Parser() {
+    sksl_delete_buffer(fBuffer, fScanner);
+    sksllex_destroy(fScanner);
+    layoutlex_destroy(fLayoutScanner);
 }
 
 /* (precision | directive | section | declaration)* END_OF_FILE */
@@ -120,39 +148,53 @@ std::vector<std::unique_ptr<ASTDeclaration>> Parser::file() {
     }
 }
 
-Token Parser::nextRawToken() {
-    if (fPushback.fKind != Token::INVALID) {
-        Token result = fPushback;
-        fPushback.fKind = Token::INVALID;
+Token Parser::nextRawToken(bool needText) {
+    if (fPushback.fKind != Token::INVALID_TOKEN) {
+        Token result(std::move(fPushback));
+        fPushback.fKind = Token::INVALID_TOKEN;
+        fPushback.fText.clear();
         return result;
     }
-    Token result = fLexer.next();
-    return result;
+    Token::Kind kind = (Token::Kind) sksllex(fScanner);
+    if (!needText) {
+        switch (kind) {
+            case Token::Kind::DIRECTIVE:     // fall through
+            case Token::Kind::IDENTIFIER:    // fall through
+            case Token::Kind::INT_LITERAL:   // fall through
+            case Token::Kind::FLOAT_LITERAL: // fall through
+            case Token::Kind::SECTION:
+                needText = true;
+            default:
+                break;
+        }
+    }
+    static String unavailable("<unavailable>");
+    return Token(Position(skslget_lineno(fScanner), -1), kind,
+                 needText ? String(skslget_text(fScanner)) : unavailable);
 }
 
 Token Parser::nextToken() {
-    Token token = this->nextRawToken();
-    while (token.fKind == Token::WHITESPACE || token.fKind == Token::LINE_COMMENT ||
-           token.fKind == Token::BLOCK_COMMENT) {
-        token = this->nextRawToken();
-    }
+    Token token;
+    do {
+        token = this->nextRawToken(false);
+    } while (token.fKind == Token::WHITESPACE);
     return token;
 }
 
 void Parser::pushback(Token t) {
-    ASSERT(fPushback.fKind == Token::INVALID);
+    ASSERT(fPushback.fKind == Token::INVALID_TOKEN);
     fPushback = std::move(t);
 }
 
 Token Parser::peek() {
-    if (fPushback.fKind == Token::INVALID) {
+    if (fPushback.fKind == Token::INVALID_TOKEN) {
         fPushback = this->nextToken();
     }
     return fPushback;
 }
 
 bool Parser::checkNext(Token::Kind kind, Token* result) {
-    if (fPushback.fKind != Token::INVALID && fPushback.fKind != kind) {
+    if (fPushback.fKind != Token::INVALID_TOKEN && fPushback.fKind != kind) {
         return false;
     }
     Token next = this->nextToken();
@@ -174,25 +216,25 @@ bool Parser::expect(Token::Kind kind, const char* expected, Token* result) {
         }
         return true;
     } else {
-        this->error(next, "expected " + String(expected) + ", but found '" +
-                    this->text(next) + "'");
+        if (next.fText.size()) {
+            this->error(next.fPosition, "expected " + String(expected) + ", but found '" +
+                                        next.fText + "'");
+        } else {
+            this->error(next.fPosition, "parse error, recompile in debug mode for details");
+        }
         return false;
     }
 }
 
-StringFragment Parser::text(Token token) {
-    return StringFragment(fText + token.fOffset, token.fLength);
-}
-
-void Parser::error(Token token, String msg) {
-    this->error(token.fOffset, msg);
+void Parser::error(Position p, const char* msg) {
+    this->error(p, String(msg));
 }
 
-void Parser::error(int offset, String msg) {
-    fErrors.error(offset, msg);
+void Parser::error(Position p, String msg) {
+    fErrors.error(p, msg);
 }
 
-bool Parser::isType(StringFragment name) {
+bool Parser::isType(const String& name) {
     return nullptr != fTypes[name];
 }
 
@@ -214,8 +256,8 @@ std::unique_ptr<ASTDeclaration> Parser::precision() {
             result = Modifiers::kHighp_Flag;
             break;
         default:
-            this->error(p, "expected 'lowp', 'mediump', or 'highp', but found '" +
-                           this->text(p) + "'");
+            this->error(p.fPosition, "expected 'lowp', 'mediump', or 'highp', but found '" +
+                                     p.fText + "'");
             return nullptr;
     }
     // FIXME handle the type
@@ -223,7 +265,7 @@ std::unique_ptr<ASTDeclaration> Parser::precision() {
         return nullptr;
     }
     this->expect(Token::SEMICOLON, "';'");
-    return std::unique_ptr<ASTDeclaration>(new ASTPrecision(p.fOffset, result));
+    return std::unique_ptr<ASTDeclaration>(new ASTPrecision(p.fPosition, result));
 }
 
 /* DIRECTIVE(#version) INT_LITERAL ("es" | "compatibility")? |
@@ -233,18 +275,16 @@ std::unique_ptr<ASTDeclaration> Parser::directive() {
     if (!this->expect(Token::DIRECTIVE, "a directive", &start)) {
         return nullptr;
     }
-    StringFragment text = this->text(start);
-    if (text == "#version") {
+    if (start.fText == "#version") {
         this->expect(Token::INT_LITERAL, "a version number");
         Token next = this->peek();
-        StringFragment nextText = this->text(next);
-        if (nextText == "es" || nextText == "compatibility") {
+        if (next.fText == "es" || next.fText == "compatibility") {
             this->nextToken();
         }
         // version is ignored for now; it will eventually become an error when we stop pretending
         // to be GLSL
         return nullptr;
-    } else if (text == "#extension") {
+    } else if (start.fText == "#extension") {
         Token name;
         if (!this->expect(Token::IDENTIFIER, "an identifier", &name)) {
             return nullptr;
@@ -256,10 +296,10 @@ std::unique_ptr<ASTDeclaration> Parser::directive() {
         if (!this->expect(Token::IDENTIFIER, "an identifier")) {
             return nullptr;
         }
-        return std::unique_ptr<ASTDeclaration>(new ASTExtension(start.fOffset,
-                                                                String(this->text(name))));
+        return std::unique_ptr<ASTDeclaration>(new ASTExtension(start.fPosition,
+                                                                std::move(name.fText)));
     } else {
-        this->error(start, "unsupported directive '" + this->text(start) + "'");
+        this->error(start.fPosition, "unsupported directive '" + start.fText + "'");
         return nullptr;
     }
 }
@@ -278,7 +318,7 @@ std::unique_ptr<ASTDeclaration> Parser::section() {
         if (!this->expect(Token::IDENTIFIER, "an identifier", &argToken)) {
             return nullptr;
         }
-        argument = this->text(argToken);
+        argument = argToken.fText;
         if (!this->expect(Token::RPAREN, "')'")) {
             return nullptr;
         }
@@ -289,7 +329,7 @@ std::unique_ptr<ASTDeclaration> Parser::section() {
     String text;
     int level = 1;
     for (;;) {
-        Token next = this->nextRawToken();
+        Token next = this->nextRawToken(true);
         switch (next.fKind) {
             case Token::LBRACE:
                 ++level;
@@ -298,7 +338,7 @@ std::unique_ptr<ASTDeclaration> Parser::section() {
                 --level;
                 break;
             case Token::END_OF_FILE:
-                this->error(start, "reached end of file while parsing section");
+                this->error(start.fPosition, "reached end of file while parsing section");
                 return nullptr;
             default:
                 break;
@@ -306,13 +346,10 @@ std::unique_ptr<ASTDeclaration> Parser::section() {
         if (!level) {
             break;
         }
-        text += this->text(next);
+        text += next.fText;
     }
-    StringFragment name = this->text(start);
-    ++name.fChars;
-    --name.fLength;
-    return std::unique_ptr<ASTDeclaration>(new ASTSection(start.fOffset,
-                                                          String(name),
+    return std::unique_ptr<ASTDeclaration>(new ASTSection(start.fPosition,
+                                                          String(start.fText.c_str() + 1),
                                                           argument,
                                                           text));
 }
@@ -322,7 +359,7 @@ std::unique_ptr<ASTDeclaration> Parser::section() {
 std::unique_ptr<ASTDeclaration> Parser::declaration() {
     Modifiers modifiers = this->modifiers();
     Token lookahead = this->peek();
-    if (lookahead.fKind == Token::IDENTIFIER && !this->isType(this->text(lookahead))) {
+    if (lookahead.fKind == Token::IDENTIFIER && !this->isType(lookahead.fText)) {
         // we have an identifier that's not a type, could be the start of an interface block
         return this->interfaceBlock(modifiers);
     }
@@ -366,14 +403,14 @@ std::unique_ptr<ASTDeclaration> Parser::declaration() {
                 return nullptr;
             }
         }
-        return std::unique_ptr<ASTDeclaration>(new ASTFunction(name.fOffset,
+        return std::unique_ptr<ASTDeclaration>(new ASTFunction(name.fPosition,
                                                                modifiers,
                                                                std::move(type),
-                                                               this->text(name),
+                                                               std::move(name.fText),
                                                                std::move(parameters),
                                                                std::move(body)));
     } else {
-        return this->varDeclarationEnd(modifiers, std::move(type), this->text(name));
+        return this->varDeclarationEnd(modifiers, std::move(type), std::move(name.fText));
     }
 }
 
@@ -388,7 +425,7 @@ std::unique_ptr<ASTVarDeclarations> Parser::varDeclarations() {
     if (!this->expect(Token::IDENTIFIER, "an identifier", &name)) {
         return nullptr;
     }
-    return this->varDeclarationEnd(modifiers, std::move(type), this->text(name));
+    return this->varDeclarationEnd(modifiers, std::move(type), std::move(name.fText));
 }
 
 /* STRUCT IDENTIFIER LBRACE varDeclaration* RBRACE */
@@ -413,7 +450,7 @@ std::unique_ptr<ASTType> Parser::structDeclaration() {
             auto type = (const Type*) fTypes[decl->fType->fName];
             for (int i = (int) var.fSizes.size() - 1; i >= 0; i--) {
                 if (!var.fSizes[i] || var.fSizes[i]->fKind != ASTExpression::kInt_Kind) {
-                    this->error(decl->fOffset, "array size in struct field must be a constant");
+                    this->error(decl->fPosition, "array size in struct field must be a constant");
                     return nullptr;
                 }
                 uint64_t columns = ((ASTIntLiteral&) *var.fSizes[i]).fValue;
@@ -423,16 +460,15 @@ std::unique_ptr<ASTType> Parser::structDeclaration() {
             }
             fields.push_back(Type::Field(decl->fModifiers, var.fName, type));
             if (var.fValue) {
-                this->error(decl->fOffset, "initializers are not permitted on struct fields");
+                this->error(decl->fPosition, "initializers are not permitted on struct fields");
             }
         }
     }
     if (!this->expect(Token::RBRACE, "'}'")) {
         return nullptr;
     }
-    fTypes.add(this->text(name), std::unique_ptr<Type>(new Type(name.fOffset, this->text(name),
-                                                                fields)));
-    return std::unique_ptr<ASTType>(new ASTType(name.fOffset, this->text(name),
+    fTypes.add(name.fText, std::unique_ptr<Type>(new Type(name.fPosition, name.fText, fields)));
+    return std::unique_ptr<ASTType>(new ASTType(name.fPosition, name.fText,
                                                 ASTType::kStruct_Kind, std::vector<int>()));
 }
 
@@ -446,11 +482,11 @@ std::unique_ptr<ASTVarDeclarations> Parser::structVarDeclaration(Modifiers modif
     if (this->checkNext(Token::IDENTIFIER, &name)) {
         std::unique_ptr<ASTVarDeclarations> result = this->varDeclarationEnd(modifiers,
                                                                              std::move(type),
-                                                                             this->text(name));
+                                                                             std::move(name.fText));
         if (result) {
             for (const auto& var : result->fVars) {
                 if (var.fValue) {
-                    this->error(var.fValue->fOffset,
+                    this->error(var.fValue->fPosition,
                                 "struct variables cannot be initialized");
                 }
             }
@@ -465,7 +501,7 @@ std::unique_ptr<ASTVarDeclarations> Parser::structVarDeclaration(Modifiers modif
    (LBRACKET expression? RBRACKET)* (EQ assignmentExpression)?)* SEMICOLON */
 std::unique_ptr<ASTVarDeclarations> Parser::varDeclarationEnd(Modifiers mods,
                                                               std::unique_ptr<ASTType> type,
-                                                              StringFragment name) {
+                                                              String name) {
     std::vector<ASTVarDeclaration> vars;
     std::vector<std::unique_ptr<ASTExpression>> currentVarSizes;
     while (this->checkNext(Token::LBRACKET)) {
@@ -489,7 +525,7 @@ std::unique_ptr<ASTVarDeclarations> Parser::varDeclarationEnd(Modifiers mods,
             return nullptr;
         }
     }
-    vars.emplace_back(name, std::move(currentVarSizes), std::move(value));
+    vars.emplace_back(std::move(name), std::move(currentVarSizes), std::move(value));
     while (this->checkNext(Token::COMMA)) {
         Token name;
         if (!this->expect(Token::IDENTIFIER, "an identifier", &name)) {
@@ -517,7 +553,7 @@ std::unique_ptr<ASTVarDeclarations> Parser::varDeclarationEnd(Modifiers mods,
                 return nullptr;
             }
         }
-        vars.emplace_back(this->text(name), std::move(currentVarSizes), std::move(value));
+        vars.emplace_back(std::move(name.fText), std::move(currentVarSizes), std::move(value));
     }
     if (!this->expect(Token::SEMICOLON, "';'")) {
         return nullptr;
@@ -544,13 +580,14 @@ std::unique_ptr<ASTParameter> Parser::parameter() {
         if (!this->expect(Token::INT_LITERAL, "a positive integer", &sizeToken)) {
             return nullptr;
         }
-        sizes.push_back(SkSL::stoi(this->text(sizeToken)));
+        sizes.push_back(SkSL::stoi(sizeToken.fText));
         if (!this->expect(Token::RBRACKET, "']'")) {
             return nullptr;
         }
     }
-    return std::unique_ptr<ASTParameter>(new ASTParameter(name.fOffset, modifiers, std::move(type),
-                                                          this->text(name), std::move(sizes)));
+    return std::unique_ptr<ASTParameter>(new ASTParameter(name.fPosition, modifiers,
+                                                          std::move(type), name.fText,
+                                                          std::move(sizes)));
 }
 
 /** (EQ INT_LITERAL)? */
@@ -560,7 +597,7 @@ int Parser::layoutInt() {
     }
     Token resultToken;
     if (this->expect(Token::INT_LITERAL, "a non-negative integer", &resultToken)) {
-        return SkSL::stoi(this->text(resultToken));
+        return SkSL::stoi(resultToken.fText);
     }
     return -1;
 }
@@ -570,13 +607,13 @@ String Parser::layoutCode() {
     if (!this->expect(Token::EQ, "'='")) {
         return "";
     }
-    Token start = this->nextRawToken();
+    Token start = this->nextRawToken(true);
     this->pushback(start);
     String code;
     int level = 1;
     bool done = false;
     while (!done) {
-        Token next = this->nextRawToken();
+        Token next = this->nextRawToken(true);
         switch (next.fKind) {
             case Token::LPAREN:
                 ++level;
@@ -590,7 +627,7 @@ String Parser::layoutCode() {
                 }
                 break;
             case Token::END_OF_FILE:
-                this->error(start, "reached end of file while parsing layout");
+                this->error(start.fPosition, "reached end of file while parsing layout");
                 return nullptr;
             default:
                 break;
@@ -602,7 +639,7 @@ String Parser::layoutCode() {
             this->pushback(std::move(next));
         }
         else {
-            code += this->text(next);
+            code += next.fText;
         }
     }
     return code;
@@ -614,10 +651,10 @@ Layout::Key Parser::layoutKey() {
         this->expect(Token::EQ, "'='");
         Token key;
         if (this->expect(Token::IDENTIFIER, "an identifer", &key)) {
-            if (this->text(key) == "identity") {
+            if (key.fText == "identity") {
                 return Layout::kIdentity_Key;
             } else {
-                this->error(key, "unsupported layout key");
+                this->error(key.fPosition, "unsupported layout key");
             }
         }
     }
@@ -651,84 +688,84 @@ Layout Parser::layout() {
         }
         for (;;) {
             Token t = this->nextToken();
-            String text = this->text(t);
-            fLayoutLexer.start(text.c_str(), text.size());
-            int token = fLayoutLexer.next().fKind;
-            if (token != LayoutToken::INVALID) {
+            YY_BUFFER_STATE buffer;
+            buffer = layout_scan_string(t.fText.c_str(), fLayoutScanner);
+            int token = layoutlex(fLayoutScanner);
+            layout_delete_buffer(buffer, fLayoutScanner);
+            if (token != Token::INVALID_TOKEN) {
                 switch (token) {
-                    case LayoutToken::LOCATION:
+                    case Token::LOCATION:
                         location = this->layoutInt();
                         break;
-                    case LayoutToken::OFFSET:
+                    case Token::OFFSET:
                         offset = this->layoutInt();
                         break;
-                    case LayoutToken::BINDING:
+                    case Token::BINDING:
                         binding = this->layoutInt();
                         break;
-                    case LayoutToken::INDEX:
+                    case Token::INDEX:
                         index = this->layoutInt();
                         break;
-                    case LayoutToken::SET:
+                    case Token::SET:
                         set = this->layoutInt();
                         break;
-                    case LayoutToken::BUILTIN:
+                    case Token::BUILTIN:
                         builtin = this->layoutInt();
                         break;
-                    case LayoutToken::INPUT_ATTACHMENT_INDEX:
+                    case Token::INPUT_ATTACHMENT_INDEX:
                         inputAttachmentIndex = this->layoutInt();
                         break;
-                    case LayoutToken::ORIGIN_UPPER_LEFT:
+                    case Token::ORIGIN_UPPER_LEFT:
                         originUpperLeft = true;
                         break;
-                    case LayoutToken::OVERRIDE_COVERAGE:
+                    case Token::OVERRIDE_COVERAGE:
                         overrideCoverage = true;
                         break;
-                    case LayoutToken::BLEND_SUPPORT_ALL_EQUATIONS:
+                    case Token::BLEND_SUPPORT_ALL_EQUATIONS:
                         blendSupportAllEquations = true;
                         break;
-                    case LayoutToken::PUSH_CONSTANT:
+                    case Token::PUSH_CONSTANT:
                         pushConstant = true;
                         break;
-                    case LayoutToken::POINTS:
+                    case Token::POINTS:
                         primitive = Layout::kPoints_Primitive;
                         break;
-                    case LayoutToken::LINES:
+                    case Token::LINES:
                         primitive = Layout::kLines_Primitive;
                         break;
-                    case LayoutToken::LINE_STRIP:
+                    case Token::LINE_STRIP:
                         primitive = Layout::kLineStrip_Primitive;
                         break;
-                    case LayoutToken::LINES_ADJACENCY:
+                    case Token::LINES_ADJACENCY:
                         primitive = Layout::kLinesAdjacency_Primitive;
                         break;
-                    case LayoutToken::TRIANGLES:
+                    case Token::TRIANGLES:
                         primitive = Layout::kTriangles_Primitive;
                         break;
-                    case LayoutToken::TRIANGLE_STRIP:
+                    case Token::TRIANGLE_STRIP:
                         primitive = Layout::kTriangleStrip_Primitive;
                         break;
-                    case LayoutToken::TRIANGLES_ADJACENCY:
+                    case Token::TRIANGLES_ADJACENCY:
                         primitive = Layout::kTrianglesAdjacency_Primitive;
                         break;
-                    case LayoutToken::MAX_VERTICES:
+                    case Token::MAX_VERTICES:
                         maxVertices = this->layoutInt();
                         break;
-                    case LayoutToken::INVOCATIONS:
+                    case Token::INVOCATIONS:
                         invocations = this->layoutInt();
                         break;
-                    case LayoutToken::WHEN:
+                    case Token::WHEN:
                         when = this->layoutCode();
                         break;
-                    case LayoutToken::KEY:
+                    case Token::KEY:
                         key = this->layoutKey();
                         break;
-                    default:
-                        ASSERT(false);
                 }
-            } else if (Layout::ReadFormat(this->text(t), &format)) {
+            } else if (Layout::ReadFormat(t.fText, &format)) {
                // AST::ReadFormat stored the result in 'format'.
             } else {
-                this->error(t, ("'" + this->text(t) + "' is not a valid layout qualifier").c_str());
+                this->error(t.fPosition, ("'" + t.fText +
+                                          "' is not a valid layout qualifier").c_str());
             }
             if (this->checkNext(Token::RPAREN)) {
                 break;
@@ -862,7 +899,7 @@ std::unique_ptr<ASTStatement> Parser::statement() {
             return this->block();
         case Token::SEMICOLON:
             this->nextToken();
-            return std::unique_ptr<ASTStatement>(new ASTBlock(start.fOffset,
+            return std::unique_ptr<ASTStatement>(new ASTBlock(start.fPosition,
                                                      std::vector<std::unique_ptr<ASTStatement>>()));
         case Token::CONST:   // fall through
         case Token::HIGHP:   // fall through
@@ -875,7 +912,7 @@ std::unique_ptr<ASTStatement> Parser::statement() {
             return std::unique_ptr<ASTStatement>(new ASTVarDeclarationStatement(std::move(decl)));
         }
         case Token::IDENTIFIER:
-            if (this->isType(this->text(start))) {
+            if (this->isType(start.fText)) {
                 auto decl = this->varDeclarations();
                 if (!decl) {
                     return nullptr;
@@ -895,8 +932,8 @@ std::unique_ptr<ASTType> Parser::type() {
     if (!this->expect(Token::IDENTIFIER, "a type", &type)) {
         return nullptr;
     }
-    if (!this->isType(this->text(type))) {
-        this->error(type, ("no type named '" + this->text(type) + "'").c_str());
+    if (!this->isType(type.fText)) {
+        this->error(type.fPosition, ("no type named '" + type.fText + "'").c_str());
         return nullptr;
     }
     std::vector<int> sizes;
@@ -913,7 +950,7 @@ std::unique_ptr<ASTType> Parser::type() {
         }
         this->expect(Token::RBRACKET, "']'");
     }
-    return std::unique_ptr<ASTType>(new ASTType(type.fOffset, this->text(type),
+    return std::unique_ptr<ASTType>(new ASTType(type.fPosition, std::move(type.fText),
                                                 ASTType::kIdentifier_Kind, sizes));
 }
 
@@ -927,7 +964,7 @@ std::unique_ptr<ASTDeclaration> Parser::interfaceBlock(Modifiers mods) {
         // we only get into interfaceBlock if we found a top-level identifier which was not a type.
         // 99% of the time, the user was not actually intending to create an interface block, so
         // it's better to report it as an unknown type
-        this->error(name, "no type named '" + this->text(name) + "'");
+        this->error(name.fPosition, "no type named '" + name.fText + "'");
         return nullptr;
     }
     this->nextToken();
@@ -941,9 +978,8 @@ std::unique_ptr<ASTDeclaration> Parser::interfaceBlock(Modifiers mods) {
     }
     this->nextToken();
     std::vector<std::unique_ptr<ASTExpression>> sizes;
-    StringFragment instanceName;
-    Token instanceNameToken;
-    if (this->checkNext(Token::IDENTIFIER, &instanceNameToken)) {
+    Token instanceName;
+    if (this->checkNext(Token::IDENTIFIER, &instanceName)) {
         while (this->checkNext(Token::LBRACKET)) {
             if (this->peek().fKind != Token::RBRACKET) {
                 std::unique_ptr<ASTExpression> size = this->expression();
@@ -956,13 +992,12 @@ std::unique_ptr<ASTDeclaration> Parser::interfaceBlock(Modifiers mods) {
             }
             this->expect(Token::RBRACKET, "']'");
         }
-        instanceName = this->text(instanceNameToken);
     }
     this->expect(Token::SEMICOLON, "';'");
-    return std::unique_ptr<ASTDeclaration>(new ASTInterfaceBlock(name.fOffset, mods,
-                                                                 this->text(name),
+    return std::unique_ptr<ASTDeclaration>(new ASTInterfaceBlock(name.fPosition, mods,
+                                                                 std::move(name.fText),
                                                                  std::move(decls),
-                                                                 instanceName,
+                                                                 std::move(instanceName.fText),
                                                                  std::move(sizes)));
 }
 
@@ -994,7 +1029,7 @@ std::unique_ptr<ASTIfStatement> Parser::ifStatement() {
             return nullptr;
         }
     }
-    return std::unique_ptr<ASTIfStatement>(new ASTIfStatement(start.fOffset,
+    return std::unique_ptr<ASTIfStatement>(new ASTIfStatement(start.fPosition,
                                                               isStatic,
                                                               std::move(test),
                                                               std::move(ifTrue),
@@ -1027,7 +1062,7 @@ std::unique_ptr<ASTDoStatement> Parser::doStatement() {
     if (!this->expect(Token::SEMICOLON, "';'")) {
         return nullptr;
     }
-    return std::unique_ptr<ASTDoStatement>(new ASTDoStatement(start.fOffset,
+    return std::unique_ptr<ASTDoStatement>(new ASTDoStatement(start.fPosition,
                                                               std::move(statement),
                                                               std::move(test)));
 }
@@ -1052,7 +1087,7 @@ std::unique_ptr<ASTWhileStatement> Parser::whileStatement() {
     if (!statement) {
         return nullptr;
     }
-    return std::unique_ptr<ASTWhileStatement>(new ASTWhileStatement(start.fOffset,
+    return std::unique_ptr<ASTWhileStatement>(new ASTWhileStatement(start.fPosition,
                                                                     std::move(test),
                                                                     std::move(statement)));
 }
@@ -1079,7 +1114,7 @@ std::unique_ptr<ASTSwitchCase> Parser::switchCase() {
         }
         statements.push_back(std::move(s));
     }
-    return std::unique_ptr<ASTSwitchCase>(new ASTSwitchCase(start.fOffset, std::move(value),
+    return std::unique_ptr<ASTSwitchCase>(new ASTSwitchCase(start.fPosition, std::move(value),
                                                             std::move(statements)));
 }
 
@@ -1127,13 +1162,13 @@ std::unique_ptr<ASTStatement> Parser::switchStatement() {
             }
             statements.push_back(std::move(s));
         }
-        cases.emplace_back(new ASTSwitchCase(defaultStart.fOffset, nullptr,
+        cases.emplace_back(new ASTSwitchCase(defaultStart.fPosition, nullptr,
                                              std::move(statements)));
     }
     if (!this->expect(Token::RBRACE, "'}'")) {
         return nullptr;
     }
-    return std::unique_ptr<ASTStatement>(new ASTSwitchStatement(start.fOffset,
+    return std::unique_ptr<ASTStatement>(new ASTSwitchStatement(start.fPosition,
                                                                 isStatic,
                                                                 std::move(value),
                                                                 std::move(cases)));
@@ -1165,7 +1200,7 @@ std::unique_ptr<ASTForStatement> Parser::forStatement() {
             break;
         }
         case Token::IDENTIFIER: {
-            if (this->isType(this->text(nextToken))) {
+            if (this->isType(nextToken.fText)) {
                 std::unique_ptr<ASTVarDeclarations> vd = this->varDeclarations();
                 if (!vd) {
                     return nullptr;
@@ -1202,7 +1237,7 @@ std::unique_ptr<ASTForStatement> Parser::forStatement() {
     if (!statement) {
         return nullptr;
     }
-    return std::unique_ptr<ASTForStatement>(new ASTForStatement(start.fOffset,
+    return std::unique_ptr<ASTForStatement>(new ASTForStatement(start.fPosition,
                                                                 std::move(initializer),
                                                                 std::move(test), std::move(next),
                                                                 std::move(statement)));
@@ -1224,7 +1259,7 @@ std::unique_ptr<ASTReturnStatement> Parser::returnStatement() {
     if (!this->expect(Token::SEMICOLON, "';'")) {
         return nullptr;
     }
-    return std::unique_ptr<ASTReturnStatement>(new ASTReturnStatement(start.fOffset,
+    return std::unique_ptr<ASTReturnStatement>(new ASTReturnStatement(start.fPosition,
                                                                       std::move(expression)));
 }
 
@@ -1237,7 +1272,7 @@ std::unique_ptr<ASTBreakStatement> Parser::breakStatement() {
     if (!this->expect(Token::SEMICOLON, "';'")) {
         return nullptr;
     }
-    return std::unique_ptr<ASTBreakStatement>(new ASTBreakStatement(start.fOffset));
+    return std::unique_ptr<ASTBreakStatement>(new ASTBreakStatement(start.fPosition));
 }
 
 /* CONTINUE SEMICOLON */
@@ -1249,7 +1284,7 @@ std::unique_ptr<ASTContinueStatement> Parser::continueStatement() {
     if (!this->expect(Token::SEMICOLON, "';'")) {
         return nullptr;
     }
-    return std::unique_ptr<ASTContinueStatement>(new ASTContinueStatement(start.fOffset));
+    return std::unique_ptr<ASTContinueStatement>(new ASTContinueStatement(start.fPosition));
 }
 
 /* DISCARD SEMICOLON */
@@ -1261,7 +1296,7 @@ std::unique_ptr<ASTDiscardStatement> Parser::discardStatement() {
     if (!this->expect(Token::SEMICOLON, "';'")) {
         return nullptr;
     }
-    return std::unique_ptr<ASTDiscardStatement>(new ASTDiscardStatement(start.fOffset));
+    return std::unique_ptr<ASTDiscardStatement>(new ASTDiscardStatement(start.fPosition));
 }
 
 /* LBRACE statement* RBRACE */
@@ -1279,10 +1314,10 @@ std::unique_ptr<ASTBlock> Parser::block() {
         switch (this->peek().fKind) {
             case Token::RBRACE:
                 this->nextToken();
-                return std::unique_ptr<ASTBlock>(new ASTBlock(start.fOffset,
+                return std::unique_ptr<ASTBlock>(new ASTBlock(start.fPosition,
                                                               std::move(statements)));
             case Token::END_OF_FILE:
-                this->error(this->peek(), "expected '}', but found end of file");
+                this->error(this->peek().fPosition, "expected '}', but found end of file");
                 return nullptr;
             default: {
                 std::unique_ptr<ASTStatement> statement = this->statement();
@@ -1679,7 +1714,7 @@ std::unique_ptr<ASTSuffix> Parser::suffix() {
     switch (next.fKind) {
         case Token::LBRACKET: {
             if (this->checkNext(Token::RBRACKET)) {
-                return std::unique_ptr<ASTSuffix>(new ASTIndexSuffix(next.fOffset));
+                return std::unique_ptr<ASTSuffix>(new ASTIndexSuffix(next.fPosition));
             }
             std::unique_ptr<ASTExpression> e = this->expression();
             if (!e) {
@@ -1689,10 +1724,10 @@ std::unique_ptr<ASTSuffix> Parser::suffix() {
             return std::unique_ptr<ASTSuffix>(new ASTIndexSuffix(std::move(e)));
         }
         case Token::DOT: {
-            int offset = this->peek().fOffset;
-            StringFragment text;
+            Position pos = this->peek().fPosition;
+            String text;
             if (this->identifier(&text)) {
-                return std::unique_ptr<ASTSuffix>(new ASTFieldSuffix(offset, std::move(text)));
+                return std::unique_ptr<ASTSuffix>(new ASTFieldSuffix(pos, std::move(text)));
             }
             return nullptr;
         }
@@ -1711,17 +1746,17 @@ std::unique_ptr<ASTSuffix> Parser::suffix() {
                 }
             }
             this->expect(Token::RPAREN, "')' to complete function parameters");
-            return std::unique_ptr<ASTSuffix>(new ASTCallSuffix(next.fOffset,
+            return std::unique_ptr<ASTSuffix>(new ASTCallSuffix(next.fPosition,
                                                                 std::move(parameters)));
         }
         case Token::PLUSPLUS:
-            return std::unique_ptr<ASTSuffix>(new ASTSuffix(next.fOffset,
+            return std::unique_ptr<ASTSuffix>(new ASTSuffix(next.fPosition,
                                                             ASTSuffix::kPostIncrement_Kind));
         case Token::MINUSMINUS:
-            return std::unique_ptr<ASTSuffix>(new ASTSuffix(next.fOffset,
+            return std::unique_ptr<ASTSuffix>(new ASTSuffix(next.fPosition,
                                                             ASTSuffix::kPostDecrement_Kind));
         default: {
-            this->error(next,  "expected expression suffix, but found '" + this->text(next) +
+            this->error(next.fPosition,  "expected expression suffix, but found '" + next.fText +
                                          "'\n");
             return nullptr;
         }
@@ -1734,23 +1769,23 @@ std::unique_ptr<ASTExpression> Parser::term() {
     Token t = this->peek();
     switch (t.fKind) {
         case Token::IDENTIFIER: {
-            StringFragment text;
+            String text;
             if (this->identifier(&text)) {
-                result.reset(new ASTIdentifier(t.fOffset, std::move(text)));
+                result.reset(new ASTIdentifier(t.fPosition, std::move(text)));
             }
             break;
         }
         case Token::INT_LITERAL: {
             int64_t i;
             if (this->intLiteral(&i)) {
-                result.reset(new ASTIntLiteral(t.fOffset, i));
+                result.reset(new ASTIntLiteral(t.fPosition, i));
             }
             break;
         }
         case Token::FLOAT_LITERAL: {
             double f;
             if (this->floatLiteral(&f)) {
-                result.reset(new ASTFloatLiteral(t.fOffset, f));
+                result.reset(new ASTFloatLiteral(t.fPosition, f));
             }
             break;
         }
@@ -1758,7 +1793,7 @@ std::unique_ptr<ASTExpression> Parser::term() {
         case Token::FALSE_LITERAL: {
             bool b;
             if (this->boolLiteral(&b)) {
-                result.reset(new ASTBoolLiteral(t.fOffset, b));
+                result.reset(new ASTBoolLiteral(t.fPosition, b));
             }
             break;
         }
@@ -1772,7 +1807,7 @@ std::unique_ptr<ASTExpression> Parser::term() {
         }
         default:
             this->nextToken();
-            this->error(t.fOffset,  "expected expression, but found '" + this->text(t) + "'\n");
+            this->error(t.fPosition,  "expected expression, but found '" + t.fText + "'\n");
             result = nullptr;
     }
     return result;
@@ -1782,7 +1817,7 @@ std::unique_ptr<ASTExpression> Parser::term() {
 bool Parser::intLiteral(int64_t* dest) {
     Token t;
     if (this->expect(Token::INT_LITERAL, "integer literal", &t)) {
-        *dest = SkSL::stol(this->text(t));
+        *dest = SkSL::stol(t.fText);
         return true;
     }
     return false;
@@ -1792,7 +1827,7 @@ bool Parser::intLiteral(int64_t* dest) {
 bool Parser::floatLiteral(double* dest) {
     Token t;
     if (this->expect(Token::FLOAT_LITERAL, "float literal", &t)) {
-        *dest = SkSL::stod(this->text(t));
+        *dest = SkSL::stod(t.fText);
         return true;
     }
     return false;
@@ -1809,16 +1844,16 @@ bool Parser::boolLiteral(bool* dest) {
             *dest = false;
             return true;
         default:
-            this->error(t, "expected 'true' or 'false', but found '" + this->text(t) + "'\n");
+            this->error(t.fPosition, "expected 'true' or 'false', but found '" + t.fText + "'\n");
             return false;
     }
 }
 
 /* IDENTIFIER */
-bool Parser::identifier(StringFragment* dest) {
+bool Parser::identifier(String* dest) {
     Token t;
     if (this->expect(Token::IDENTIFIER, "identifier", &t)) {
-        *dest = this->text(t);
+        *dest = std::move(t.fText);
         return true;
     }
     return false;