diff options
author | Laurent Le Brun <laurentlb@google.com> | 2015-10-21 13:13:35 +0000 |
---|---|---|
committer | Han-Wen Nienhuys <hanwen@google.com> | 2015-10-21 14:39:29 +0000 |
commit | 4d7fae3bfcc579676d7b2454ab8a38743aa4c635 (patch) | |
tree | 74983fbbd3eb9c330f958923d7ab695656de5d23 | |
parent | 5d9b8a0dcf89190edc190f439c383d7bdbae658f (diff) |
Lexer: Handle triple quoted raw strings (e.g. r"""abc""").
--
MOS_MIGRATED_REVID=105956734
-rw-r--r-- | src/main/java/com/google/devtools/build/lib/syntax/Lexer.java | 19 | ||||
-rw-r--r-- | src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java | 30 |
2 files changed, 40 insertions, 9 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java index 2f739493b3..f656f28002 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java @@ -312,7 +312,7 @@ public final class Lexer { * * @return the string-literal token. */ - private Token escapedStringLiteral(char quot) { + private Token escapedStringLiteral(char quot, boolean isRaw) { boolean inTriplequote = skipTripleQuote(quot); int oldPos = pos - 1; @@ -336,6 +336,14 @@ public final class Lexer { error("unterminated string literal at eof", oldPos, pos); return new Token(TokenKind.STRING, oldPos, pos, literal.toString()); } + if (isRaw) { + // Insert \ and the following character. + // As in Python, it means that a raw string can never end with a single \. + literal.append('\\'); + literal.append(buffer[pos]); + pos++; + break; + } c = buffer[pos]; pos++; switch (c) { @@ -428,7 +436,7 @@ public final class Lexer { // Don't even attempt to parse triple-quotes here. if (skipTripleQuote(quot)) { pos -= 2; - return escapedStringLiteral(quot); + return escapedStringLiteral(quot, isRaw); } // first quick optimistic scan for a simple non-escaped string @@ -446,11 +454,10 @@ public final class Lexer { // skip the next character pos++; break; - } else { - // oops, hit an escape, need to start over & build a new string buffer - pos = oldPos + 1; - return escapedStringLiteral(quot); } + // oops, hit an escape, need to start over & build a new string buffer + pos = oldPos + 1; + return escapedStringLiteral(quot, false); case '\'': case '"': if (c == quot) { diff --git a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java index c60067dbef..45e66bf060 100644 --- a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java +++ b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java @@ -247,7 +247,14 @@ public class LexerTest { assertEquals("STRING(\\$$) NEWLINE EOF", values(tokens("'\\$$'"))); assertEquals("STRING(ab) NEWLINE EOF", values(tokens("'a\\\nb'"))); // escape end of line + assertEquals("STRING(abcd) NEWLINE EOF", + values(tokens("\"ab\\ucd\""))); + assertEquals("/some/path.txt:1: escape sequence not implemented: \\u", + lastError.toString()); + } + @Test + public void testRawString() throws Exception { assertEquals("STRING(abcd) NEWLINE EOF", values(tokens("r'abcd'"))); assertEquals("STRING(abcd) NEWLINE EOF", @@ -261,9 +268,26 @@ public class LexerTest { assertEquals("STRING(ab) IDENTIFIER(r) NEWLINE EOF", values(tokens("r'ab'r"))); - assertEquals("STRING(abcd) NEWLINE EOF", - values(tokens("\"ab\\ucd\""))); - assertEquals("/some/path.txt:1: escape sequence not implemented: \\u", + // Unterminated raw string + values(tokens("r'\\'")); // r'\' + assertEquals("/some/path.txt:1: unterminated string literal at eof", + lastError.toString()); + } + + @Test + public void testTripleRawString() throws Exception { + // r'''a\ncd''' + assertEquals("STRING(ab\\ncd) NEWLINE EOF", + values(tokens("r'''ab\\ncd'''"))); + // r"""ab + // cd""" + assertEquals( + "STRING(ab\ncd) NEWLINE EOF", + values(tokens("\"\"\"ab\ncd\"\"\""))); + + // Unterminated raw string + values(tokens("r'''\\'''")); // r'''\''' + assertEquals("/some/path.txt:1: unterminated string literal at eof", lastError.toString()); } |