Reject files when the first line is indented.

A bug in the lexer ignored indentation on the first line of a file. This now
causes an error.

Also, remove the COMMENT token from the lexer. Comments are now accessed separately. This will allow further optimizations in the lexer. It also aligns the code a bit more with the Go implementation.

RELNOTES[INC]:
  Indentation on the first line of a file was previously ignored. This is now fixed.

PiperOrigin-RevId: 197889775

2 files changed, 29 insertions, 39 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java b/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
index a69563a73f..1ee0dff026 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/EvaluationTest.java
@@ -467,9 +467,9 @@ public class EvaluationTest extends EvaluationTestCase {
         .testStatement("[    ] * 10", MutableList.empty())
         .testStatement("[1, 2] * 0", MutableList.empty())
         .testStatement("[1, 2] * -4", MutableList.empty())
-        .testStatement(" 2 * [1, 2]", MutableList.of(env, 1, 2, 1, 2))
+        .testStatement("2 * [1, 2]", MutableList.of(env, 1, 2, 1, 2))
         .testStatement("10 * []", MutableList.empty())
-        .testStatement(" 0 * [1, 2]", MutableList.empty())
+        .testStatement("0 * [1, 2]", MutableList.empty())
         .testStatement("-4 * [1, 2]", MutableList.empty());
   }
 
@@ -484,9 +484,9 @@ public class EvaluationTest extends EvaluationTestCase {
         .testStatement("(    ) * 10", Tuple.empty())
         .testStatement("(1, 2) * 0", Tuple.empty())
         .testStatement("(1, 2) * -4", Tuple.empty())
-        .testStatement(" 2 * (1, 2)", Tuple.of(1, 2, 1, 2))
+        .testStatement("2 * (1, 2)", Tuple.of(1, 2, 1, 2))
         .testStatement("10 * ()", Tuple.empty())
-        .testStatement(" 0 * (1, 2)", Tuple.empty())
+        .testStatement("0 * (1, 2)", Tuple.empty())
         .testStatement("-4 * (1, 2)", Tuple.empty());
   }
 
diff --git a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
index 81212e6fdb..14c7152f7f 100644
--- a/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/syntax/LexerTest.java
@@ -146,33 +146,35 @@ public class LexerTest {
   public void testBasics1() throws Exception {
     assertThat(names(tokens("wiz) "))).isEqualTo("IDENTIFIER RPAREN NEWLINE EOF");
     assertThat(names(tokens("wiz )"))).isEqualTo("IDENTIFIER RPAREN NEWLINE EOF");
-    assertThat(names(tokens(" wiz)"))).isEqualTo("IDENTIFIER RPAREN NEWLINE EOF");
-    assertThat(names(tokens(" wiz ) "))).isEqualTo("IDENTIFIER RPAREN NEWLINE EOF");
+    assertThat(names(tokens(" wiz)")))
+        .isEqualTo("INDENT IDENTIFIER RPAREN NEWLINE OUTDENT NEWLINE EOF");
+    assertThat(names(tokens(" wiz ) ")))
+        .isEqualTo("INDENT IDENTIFIER RPAREN NEWLINE OUTDENT NEWLINE EOF");
     assertThat(names(tokens("wiz\t)"))).isEqualTo("IDENTIFIER RPAREN NEWLINE EOF");
   }
 
   @Test
   public void testBasics2() throws Exception {
     assertThat(names(tokens(")"))).isEqualTo("RPAREN NEWLINE EOF");
-    assertThat(names(tokens(" )"))).isEqualTo("RPAREN NEWLINE EOF");
-    assertThat(names(tokens(" ) "))).isEqualTo("RPAREN NEWLINE EOF");
+    assertThat(names(tokens(" )"))).isEqualTo("INDENT RPAREN NEWLINE OUTDENT NEWLINE EOF");
+    assertThat(names(tokens(" ) "))).isEqualTo("INDENT RPAREN NEWLINE OUTDENT NEWLINE EOF");
     assertThat(names(tokens(") "))).isEqualTo("RPAREN NEWLINE EOF");
   }
 
   @Test
   public void testBasics3() throws Exception {
-    assertThat(names(tokens("123#456\n789"))).isEqualTo("INT COMMENT NEWLINE INT NEWLINE EOF");
-    assertThat(names(tokens("123 #456\n789"))).isEqualTo("INT COMMENT NEWLINE INT NEWLINE EOF");
-    assertThat(names(tokens("123#456 \n789"))).isEqualTo("INT COMMENT NEWLINE INT NEWLINE EOF");
+    assertThat(names(tokens("123#456\n789"))).isEqualTo("INT NEWLINE INT NEWLINE EOF");
+    assertThat(names(tokens("123 #456\n789"))).isEqualTo("INT NEWLINE INT NEWLINE EOF");
+    assertThat(names(tokens("123#456 \n789"))).isEqualTo("INT NEWLINE INT NEWLINE EOF");
     assertThat(names(tokens("123#456\n 789")))
-        .isEqualTo("INT COMMENT NEWLINE INDENT INT NEWLINE OUTDENT NEWLINE EOF");
-    assertThat(names(tokens("123#456\n789 "))).isEqualTo("INT COMMENT NEWLINE INT NEWLINE EOF");
+        .isEqualTo("INT NEWLINE INDENT INT NEWLINE OUTDENT NEWLINE EOF");
+    assertThat(names(tokens("123#456\n789 "))).isEqualTo("INT NEWLINE INT NEWLINE EOF");
   }
 
   @Test
   public void testBasics4() throws Exception {
     assertThat(names(tokens(""))).isEqualTo("NEWLINE EOF");
-    assertThat(names(tokens("# foo"))).isEqualTo("COMMENT NEWLINE EOF");
+    assertThat(names(tokens("# foo"))).isEqualTo("NEWLINE EOF");
     assertThat(names(tokens("1 2 3 4"))).isEqualTo("INT INT INT INT NEWLINE EOF");
     assertThat(names(tokens("1.234"))).isEqualTo("INT DOT INT NEWLINE EOF");
     assertThat(names(tokens("foo(bar, wiz)")))
@@ -190,9 +192,8 @@ public class LexerTest {
   @Test
   public void testCrLf() throws Exception {
     assertThat(names(tokens("\r\n\r\n"))).isEqualTo("NEWLINE EOF");
-    assertThat(names(tokens("\r\n\r1\r\r\n"))).isEqualTo("NEWLINE INT NEWLINE EOF");
-    assertThat(names(tokens("# foo\r\n# bar\r\n")))
-        .isEqualTo("COMMENT NEWLINE COMMENT NEWLINE EOF");
+    assertThat(names(tokens("\r\n\r1\r\r\n"))).isEqualTo("INT NEWLINE EOF");
+    assertThat(names(tokens("# foo\r\n# bar\r\n"))).isEqualTo("NEWLINE EOF");
   }
 
   @Test
@@ -419,38 +420,27 @@ public class LexerTest {
   }
 
   @Test
+  public void testIndentationOnFirstLine() throws Exception {
+    assertThat(values(tokens("    1"))).isEqualTo("INDENT INT(1) NEWLINE OUTDENT NEWLINE EOF");
+    assertThat(values(tokens("\n\n    1"))).isEqualTo("INDENT INT(1) NEWLINE OUTDENT NEWLINE EOF");
+  }
+
+  @Test
   public void testBlankLineIndentation() throws Exception {
     // Blank lines and comment lines should not generate any newlines indents
     // (but note that every input ends with NEWLINE EOF).
-    assertThat(names(tokens("\n      #\n"))).isEqualTo("COMMENT NEWLINE EOF");
-    assertThat(names(tokens("      #"))).isEqualTo("COMMENT NEWLINE EOF");
-    assertThat(names(tokens("      #\n"))).isEqualTo("COMMENT NEWLINE EOF");
-    assertThat(names(tokens("      #comment\n"))).isEqualTo("COMMENT NEWLINE EOF");
+    assertThat(names(tokens("\n      #\n"))).isEqualTo("NEWLINE EOF");
+    assertThat(names(tokens("      #"))).isEqualTo("NEWLINE EOF");
+    assertThat(names(tokens("      #\n"))).isEqualTo("NEWLINE EOF");
+    assertThat(names(tokens("      #comment\n"))).isEqualTo("NEWLINE EOF");
     assertThat(names(tokens("def f(x):\n" + "  # comment\n" + "\n" + "  \n" + "  return x\n")))
         .isEqualTo(
             "DEF IDENTIFIER LPAREN IDENTIFIER RPAREN COLON NEWLINE "
-                + "COMMENT INDENT RETURN IDENTIFIER NEWLINE "
+                + "INDENT RETURN IDENTIFIER NEWLINE "
                 + "OUTDENT NEWLINE EOF");
   }
 
   @Test
-  public void testMultipleCommentLines() throws Exception {
-    assertThat(
-            names(
-                tokens(
-                    "# Copyright\n"
-                        + "#\n"
-                        + "# A comment line\n"
-                        + "# An adjoining line\n"
-                        + "def f(x):\n"
-                        + "  return x\n")))
-        .isEqualTo(
-            "COMMENT NEWLINE COMMENT COMMENT COMMENT "
-                + "DEF IDENTIFIER LPAREN IDENTIFIER RPAREN COLON NEWLINE "
-                + "INDENT RETURN IDENTIFIER NEWLINE OUTDENT NEWLINE EOF");
-  }
-
-  @Test
   public void testBackslash() throws Exception {
     assertThat(names(tokens("a\\\nb"))).isEqualTo("IDENTIFIER IDENTIFIER NEWLINE EOF");
     assertThat(names(tokens("a\\\r\nb"))).isEqualTo("IDENTIFIER IDENTIFIER NEWLINE EOF");