Add operator // for division.

In both Python 2 and Python 3, the operator // is used for int division.
With Python 3, operator / is for float division.

RELNOTES: None.
PiperOrigin-RevId: 156582262

5 files changed, 38 insertions, 17 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
index ea8280ac8a..0e03157a2d 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/BinaryOperatorExpression.java
@@ -134,6 +134,7 @@ public final class BinaryOperatorExpression extends Expression {
         return mult(lval, rval, env, location);
 
       case DIVIDE:
+      case FLOOR_DIVIDE:
         return divide(lval, rval, location);
 
       case PERCENT:
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
index d7ee319b5e..9257de77aa 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java
@@ -296,7 +296,7 @@ public final class Lexer {
    * delimiter (3 x quot), and advances 'pos' by two if so.
    */
   private boolean skipTripleQuote(char quot) {
-    if (pos + 1 < buffer.length && buffer[pos] == quot && buffer[pos + 1] == quot) {
+    if (lookaheadIs(0, quot) && lookaheadIs(1, quot)) {
       pos += 2;
       return true;
     } else {
@@ -340,7 +340,7 @@ public final class Lexer {
             // Insert \ and the following character.
             // As in Python, it means that a raw string can never end with a single \.
             literal.append('\\');
-            if (pos + 1 < buffer.length && buffer[pos] == '\r' && buffer[pos + 1] == '\n') {
+            if (lookaheadIs(0, '\r') && lookaheadIs(1, '\n')) {
               literal.append("\n");
               pos += 2;
             } else if (buffer[pos] == '\r' || buffer[pos] == '\n') {
@@ -356,7 +356,7 @@ public final class Lexer {
           pos++;
           switch (c) {
             case '\r':
-              if (pos < buffer.length && buffer[pos] == '\n') {
+              if (lookaheadIs(0, '\n')) {
                 pos += 1;
                 break;
               } else {
@@ -466,7 +466,7 @@ public final class Lexer {
           return t;
         case '\\':
           if (isRaw) {
-            if (pos + 1 < buffer.length && buffer[pos] == '\r' && buffer[pos + 1] == '\n') {
+            if (lookaheadIs(0, '\r') && lookaheadIs(1, '\n')) {
               // There was a CRLF after the newline. No shortcut possible, since it needs to be
               // transformed into a single LF.
               pos = oldPos + 1;
@@ -483,9 +483,10 @@ public final class Lexer {
         case '"':
           if (c == quot) {
             // close-quote, all done.
-            return new Token(TokenKind.STRING, oldPos, pos,
-                             bufferSlice(oldPos + 1, pos - 1));
+            return new Token(TokenKind.STRING, oldPos, pos, bufferSlice(oldPos + 1, pos - 1));
           }
+          break;
+        default: // fall out
       }
     }
 
@@ -667,6 +668,11 @@ public final class Lexer {
     }
   }
 
+  /** Test if the character at pos+p is c. */
+  private boolean lookaheadIs(int p, char c) {
+    return pos + p < buffer.length && buffer[pos + p] == c;
+  }
+
   /**
    * Performs tokenization of the character buffer of file contents provided to
    * the constructor.
@@ -747,7 +753,16 @@ public final class Lexer {
         break;
       }
       case '/': {
-        addToken(new Token(TokenKind.SLASH, pos - 1, pos));
+        if (lookaheadIs(0, '/') && lookaheadIs(1, '=')) {
+          addToken(new Token(TokenKind.SLASH_SLASH_EQUALS, pos - 1, pos + 2));
+          pos += 2;
+        } else if (lookaheadIs(0, '/')) {
+          addToken(new Token(TokenKind.SLASH_SLASH, pos - 1, pos + 1));
+          pos += 1;
+        } else {
+          // /= is handled by tokenizeTwoChars.
+          addToken(new Token(TokenKind.SLASH, pos - 1, pos));
+        }
         break;
       }
       case ';': {
@@ -770,9 +785,9 @@ public final class Lexer {
       }
       case '\\': {
         // Backslash character is valid only at the end of a line (or in a string)
-        if (pos + 1 < buffer.length && buffer[pos] == '\n') {
+        if (lookaheadIs(0, '\n')) {
           pos += 1;  // skip the end of line character
-        } else if (pos + 2 < buffer.length && buffer[pos] == '\r' && buffer[pos + 1] == '\n') {
+        } else if (lookaheadIs(0, '\r') && lookaheadIs(1, '\n')) {
           pos += 2;  // skip the CRLF at the end of line
         } else {
           addToken(new Token(TokenKind.ILLEGAL, pos - 1, pos, Character.toString(c)));
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
index 91bd49da26..b839865ad6 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Operator.java
@@ -21,6 +21,7 @@ public enum Operator {
   AND("and"),
   DIVIDE("/"),
   EQUALS_EQUALS("=="),
+  FLOOR_DIVIDE("//"),
   GREATER(">"),
   GREATER_EQUALS(">="),
   IN("in"),
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
index 23bb0cd0c6..1cb224d42e 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/Parser.java
@@ -38,7 +38,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-
 /**
  * Recursive descent parser for LL(2) BUILD language.
  * Loosely based on Python 2 grammar.
@@ -159,6 +158,7 @@ public class Parser {
           .put(TokenKind.OR, Operator.OR)
           .put(TokenKind.PERCENT, Operator.PERCENT)
           .put(TokenKind.SLASH, Operator.DIVIDE)
+          .put(TokenKind.SLASH_SLASH, Operator.FLOOR_DIVIDE)
           .put(TokenKind.PLUS, Operator.PLUS)
           .put(TokenKind.PIPE, Operator.PIPE)
           .put(TokenKind.STAR, Operator.MULT)
@@ -171,6 +171,7 @@ public class Parser {
           .put(TokenKind.MINUS_EQUALS, Operator.MINUS)
           .put(TokenKind.STAR_EQUALS, Operator.MULT)
           .put(TokenKind.SLASH_EQUALS, Operator.DIVIDE)
+          .put(TokenKind.SLASH_SLASH_EQUALS, Operator.FLOOR_DIVIDE)
           .put(TokenKind.PERCENT_EQUALS, Operator.PERCENT)
           .build();
 
@@ -185,7 +186,7 @@ public class Parser {
           Operator.GREATER, Operator.GREATER_EQUALS, Operator.IN, Operator.NOT_IN),
       EnumSet.of(Operator.PIPE),
       EnumSet.of(Operator.MINUS, Operator.PLUS),
-      EnumSet.of(Operator.DIVIDE, Operator.MULT, Operator.PERCENT));
+      EnumSet.of(Operator.DIVIDE, Operator.FLOOR_DIVIDE, Operator.MULT, Operator.PERCENT));
 
   private final Iterator<Token> tokens;
   private int errorsCount;
@@ -1198,7 +1199,8 @@ public class Parser {
     if (token.kind == TokenKind.EQUALS) {
       nextToken();
       Expression rvalue = parseExpression();
-      return setLocation(new AssignmentStatement(expression, rvalue), start, rvalue);
+      return setLocation(
+          new AssignmentStatement(/*lvalue=*/ expression, /*expression=*/ rvalue), start, rvalue);
     } else if (augmentedAssignmentMethods.containsKey(token.kind)) {
       Operator operator = augmentedAssignmentMethods.get(token.kind);
       nextToken();
diff --git a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
index 8000546fef..52acce13e9 100644
--- a/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
+++ b/src/main/java/com/google/devtools/build/lib/syntax/TokenKind.java
@@ -19,9 +19,9 @@ package com.google.devtools.build.lib.syntax;
  */
 public enum TokenKind {
 
-  ASSERT("assert"),
   AND("and"),
   AS("as"),
+  ASSERT("assert"),
   BREAK("break"),
   CLASS("class"),
   COLON(":"),
@@ -58,6 +58,7 @@ public enum TokenKind {
   LESS_EQUALS("<="),
   LPAREN("("),
   MINUS("-"),
+  MINUS_EQUALS("-="),
   NEWLINE("newline"),
   NONLOCAL("nonlocal"),
   NOT("not"),
@@ -67,13 +68,10 @@ public enum TokenKind {
   OUTDENT("outdent"),
   PASS("pass"),
   PERCENT("%"),
+  PERCENT_EQUALS("%="),
   PIPE("|"),
   PLUS("+"),
   PLUS_EQUALS("+="),
-  MINUS_EQUALS("-="),
-  STAR_EQUALS("*="),
-  SLASH_EQUALS("/="),
-  PERCENT_EQUALS("%="),
   RAISE("raise"),
   RBRACE("}"),
   RBRACKET("]"),
@@ -81,7 +79,11 @@ public enum TokenKind {
   RPAREN(")"),
   SEMI(";"),
   SLASH("/"),
+  SLASH_EQUALS("/="),
+  SLASH_SLASH("//"),
+  SLASH_SLASH_EQUALS("//="),
   STAR("*"),
+  STAR_EQUALS("*="),
   STAR_STAR("**"),
   STRING("string"),
   TRY("try"),