Create multiple ParamsFilePreProcessors to allow parsing files using the formats specified in ParameterFile.ParameterFileType. Also maintain the currently used parsing style of whitespace split arguments that allows single and double quoting and whitespace and quote escaping. This style of parsing is for a format not currently generated and will be removed once all consuming actions have been converted to using a supported format explicitly.

RELNOTES: None.
PiperOrigin-RevId: 169437362

5 files changed, 349 insertions, 114 deletions

diff --git a/src/main/java/com/google/devtools/common/options/LegacyParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/LegacyParamsFilePreProcessor.java
new file mode 100644
index 0000000000..9e8eeb0b17
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/LegacyParamsFilePreProcessor.java
@@ -0,0 +1,147 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * A {@link ParamsFilePreProcessor} that processes a parameter file using a custom format. This
+ * format assumes each parameter is separated by whitespace and allows arguments to use single and
+ * double quotes and quote and whitespace escaping.
+ */
+public class LegacyParamsFilePreProcessor extends ParamsFilePreProcessor {
+
+  public LegacyParamsFilePreProcessor(FileSystem fs) {
+    super(fs);
+  }
+
+  @Override
+  protected List<String> parse(Path paramsFile) throws IOException, OptionsParsingException {
+    try (Reader params = Files.newBufferedReader(paramsFile, StandardCharsets.UTF_8)) {
+      List<String> newArgs = new ArrayList<>();
+      StringBuilder arg = new StringBuilder();
+      CharIterator iterator = CharIterator.wrap(params);
+      while (iterator.hasNext()) {
+        char next = iterator.next();
+        if (Character.isWhitespace(next) && !iterator.isInQuote() && !iterator.isEscaped()) {
+          newArgs.add(unescape(arg.toString()));
+          arg = new StringBuilder();
+        } else {
+          arg.append(next);
+        }
+      }
+      // If there is an arg in the buffer, add it.
+      if (arg.length() > 0) {
+        newArgs.add(arg.toString());
+      }
+      // If we're still in a quote by the end of the file, throw an error.
+      if (iterator.isInQuote()) {
+        throw new OptionsParsingException(
+            String.format(ERROR_MESSAGE_FORMAT, paramsFile, iterator.getUnmatchedQuoteMessage()));
+      }
+      return newArgs;
+    }
+  }
+
+  private String unescape(String arg) {
+    if (arg.startsWith("'") && arg.endsWith("'")) {
+      String unescaped = arg.replace("'\\''", "'");
+      return unescaped.substring(1, unescaped.length() - 1);
+    }
+    return arg;
+  }
+
+  // Doesn't implement iterator to avoid autoboxing and to throw exceptions.
+  private static class CharIterator {
+
+    private final Reader reader;
+    private int readerPosition = 0;
+    private int singleQuoteStart = -1;
+    private int doubleQuoteStart = -1;
+    private boolean escaped = false;
+    private char lastChar = (char) -1;
+
+    public static CharIterator wrap(Reader reader) {
+      return new CharIterator(reader);
+    }
+
+    public CharIterator(Reader reader) {
+      this.reader = reader;
+    }
+
+    public boolean hasNext() throws IOException {
+      return peek() != -1;
+    }
+
+    private int peek() throws IOException {
+      reader.mark(1);
+      int next = reader.read();
+      reader.reset();
+      return next;
+    }
+
+    public boolean isInQuote() {
+      return singleQuoteStart != -1 || doubleQuoteStart != -1;
+    }
+
+    public boolean isEscaped() {
+      return escaped;
+    }
+
+    public String getUnmatchedQuoteMessage() {
+      StringBuilder message = new StringBuilder();
+      if (singleQuoteStart != -1) {
+        message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "'", singleQuoteStart));
+      }
+      if (doubleQuoteStart != -1) {
+        message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "\"", doubleQuoteStart));
+      }
+      return message.toString();
+    }
+
+    public char next() throws IOException {
+      if (!hasNext()) {
+        throw new NoSuchElementException();
+      }
+      char current = (char) reader.read();
+
+      // check for \r\n line endings. If found, drop the \r for normalized parsing.
+      if (current == '\r' && peek() == '\n') {
+        current = (char) reader.read();
+      }
+
+      // check to see if the current position is escaped
+      escaped = (lastChar == '\\');
+
+      if (!escaped && current == '\'') {
+        singleQuoteStart = singleQuoteStart == -1 ? readerPosition : -1;
+      }
+      if (!escaped && current == '"') {
+        doubleQuoteStart = doubleQuoteStart == -1 ? readerPosition : -1;
+      }
+
+      readerPosition++;
+      lastChar = current;
+      return current;
+    }
+  }
+}
diff --git a/src/main/java/com/google/devtools/common/options/OptionsParser.java b/src/main/java/com/google/devtools/common/options/OptionsParser.java
index 3e6b5ccba8..28c2206fbc 100644
--- a/src/main/java/com/google/devtools/common/options/OptionsParser.java
+++ b/src/main/java/com/google/devtools/common/options/OptionsParser.java
@@ -200,9 +200,16 @@ public class OptionsParser implements OptionsProvider {
     this.impl.setAllowSingleDashLongOptions(allowSingleDashLongOptions);
   }
 
-  /** Enables the Parser to handle params files loacted insinde the provided {@link FileSystem}. */
+  /** Enables the Parser to handle params files located inside the provided {@link FileSystem}. */
   public void enableParamsFileSupport(FileSystem fs) {
-    this.impl.setArgsPreProcessor(new ParamsFilePreProcessor(fs));
+    enableParamsFileSupport(new LegacyParamsFilePreProcessor(fs));
+  }
+
+  /**
+   * Enables the Parser to handle params files using the provided {@link ParamsFilePreProcessor}.
+   */
+  public void enableParamsFileSupport(ParamsFilePreProcessor preProcessor) {
+    this.impl.setArgsPreProcessor(preProcessor);
   }
 
   public void parseAndExitUponError(String[] args) {
diff --git a/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java
index 791265ae1b..87f87f6118 100644
--- a/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java
+++ b/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java
@@ -14,14 +14,9 @@
 package com.google.devtools.common.options;
 
 import java.io.IOException;
-import java.io.Reader;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystem;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.NoSuchElementException;
 
 /**
  * Defines an {@link ArgsPreProcessor} that will determine if the arguments list contains a "params"
@@ -31,7 +26,7 @@ import java.util.NoSuchElementException;
  * length. A params file argument is defined as a path starting with @. It will also be the only
  * entry in an argument list.
  */
-public class ParamsFilePreProcessor implements ArgsPreProcessor {
+public abstract class ParamsFilePreProcessor implements ArgsPreProcessor {
 
   static final String ERROR_MESSAGE_FORMAT = "Error reading params file: %s %s";
 
@@ -50,7 +45,7 @@ public class ParamsFilePreProcessor implements ArgsPreProcessor {
    * Parses the param file path and replaces the arguments list with the contents if one exists.
    *
    * @param args A list of arguments that may contain @&lt;path&gt; to a params file.
-   * @return A list of areguments suitable for parsing.
+   * @return A list of arguments suitable for parsing.
    * @throws OptionsParsingException if the path does not exist.
    */
   @Override
@@ -61,29 +56,8 @@ public class ParamsFilePreProcessor implements ArgsPreProcessor {
             String.format(TOO_MANY_ARGS_ERROR_MESSAGE_FORMAT, args), args.get(0));
       }
       Path path = fs.getPath(args.get(0).substring(1));
-      try (Reader params = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
-        List<String> newArgs = new ArrayList<>();
-        StringBuilder arg = new StringBuilder();
-        CharIterator iterator = CharIterator.wrap(params);
-        while (iterator.hasNext()) {
-          char next = iterator.next();
-          if (Character.isWhitespace(next) && !iterator.isInQuote() && !iterator.isEscaped()) {
-            newArgs.add(unescape(arg.toString()));
-            arg = new StringBuilder();
-          } else {
-            arg.append(next);
-          }
-        }
-        // If there is an arg in the buffer, add it.
-        if (arg.length() > 0) {
-          newArgs.add(arg.toString());
-        }
-        // If we're still in a quote by the end of the file, throw an error.
-        if (iterator.isInQuote()) {
-          throw new OptionsParsingException(
-              String.format(ERROR_MESSAGE_FORMAT, path, iterator.getUnmatchedQuoteMessage()));
-        }
-        return newArgs;
+      try {
+        return parse(path);
       } catch (RuntimeException | IOException e) {
         throw new OptionsParsingException(
             String.format(ERROR_MESSAGE_FORMAT, path, e.getMessage()), args.get(0), e);
@@ -92,86 +66,15 @@ public class ParamsFilePreProcessor implements ArgsPreProcessor {
     return args;
   }
 
-  private String unescape(String arg) {
-    if (arg.startsWith("'") && arg.endsWith("'")) {
-      String unescaped = arg.replace("'\\''", "'");
-      return unescaped.substring(1, unescaped.length() - 1);
-    }
-    return arg;
-  }
-
-  // Doesn't implement iterator to avoid autoboxing and to throw exceptions.
-  static class CharIterator {
-
-    private final Reader reader;
-    private int readerPosition = 0;
-    private int singleQuoteStart = -1;
-    private int doubleQuoteStart = -1;
-    private boolean escaped = false;
-    private char lastChar = (char) -1;
-
-    public static CharIterator wrap(Reader reader) {
-      return new CharIterator(reader);
-    }
-
-    public CharIterator(Reader reader) {
-      this.reader = reader;
-    }
-
-    public boolean hasNext() throws IOException {
-      return peek() != -1;
-    }
-
-    private int peek() throws IOException {
-      reader.mark(1);
-      int next = reader.read();
-      reader.reset();
-      return next;
-    }
-
-    public boolean isInQuote() {
-      return singleQuoteStart != -1 || doubleQuoteStart != -1;
-    }
-
-    public boolean isEscaped() {
-      return escaped;
-    }
-
-    public String getUnmatchedQuoteMessage() {
-      StringBuilder message = new StringBuilder();
-      if (singleQuoteStart != -1) {
-        message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "'", singleQuoteStart));
-      }
-      if (doubleQuoteStart != -1) {
-        message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "\"", doubleQuoteStart));
-      }
-      return message.toString();
-    }
-
-    public char next() throws IOException {
-      if (!hasNext()) {
-        throw new NoSuchElementException();
-      }
-      char current = (char) reader.read();
-
-      // check for \r\n line endings. If found, drop the \r for normalized parsing.
-      if (current == '\r' && peek() == '\n') {
-        current = (char) reader.read();
-      }
-
-      // check to see if the current position is escaped
-      escaped = (lastChar == '\\');
-
-      if (!escaped && current == '\'') {
-        singleQuoteStart = singleQuoteStart == -1 ? readerPosition : -1;
-      }
-      if (!escaped && current == '"') {
-        doubleQuoteStart = doubleQuoteStart == -1 ? readerPosition : -1;
-      }
-
-      readerPosition++;
-      lastChar = current;
-      return current;
-    }
-  }
+  /**
+   * Parses the paramsFile and returns a list of argument tokens to be further processed by the
+   * {@link OptionsParser}.
+   *
+   * @param paramsFile The path of the params file to parse.
+   * @return a list of argument tokens.
+   * @throws IOException if there is an error reading paramsFile.
+   * @throws OptionsParsingException if there is an error reading paramsFile.
+   */
+  protected abstract List<String> parse(Path paramsFile)
+      throws IOException, OptionsParsingException;
 }
diff --git a/src/main/java/com/google/devtools/common/options/ShellQuotedParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/ShellQuotedParamsFilePreProcessor.java
new file mode 100644
index 0000000000..525cb77227
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/ShellQuotedParamsFilePreProcessor.java
@@ -0,0 +1,138 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.io.PushbackReader;
+import java.io.Reader;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A {@link ParamsFilePreProcessor} that processes a parameter file using the {@code
+ * com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType.SHELL_QUOTED} format. This
+ * format assumes each parameter is separated by whitespace and is quoted using singe quotes
+ * ({@code '}) if it contains any special characters or is an empty string.
+ */
+public class ShellQuotedParamsFilePreProcessor extends ParamsFilePreProcessor {
+
+  public ShellQuotedParamsFilePreProcessor(FileSystem fs) {
+    super(fs);
+  }
+
+  @Override
+  protected List<String> parse(Path paramsFile) throws IOException {
+    List<String> args = new ArrayList<>();
+    try (ShellQuotedReader reader =
+        new ShellQuotedReader(Files.newBufferedReader(paramsFile, UTF_8))) {
+      String arg;
+      while ((arg = reader.readArg()) != null) {
+        args.add(arg);
+      }
+    }
+    return args;
+  }
+
+  private static class ShellQuotedReader implements AutoCloseable {
+
+    private final PushbackReader reader;
+    private int position = -1;
+
+    public ShellQuotedReader(Reader reader) {
+      this.reader = new PushbackReader(reader, 10);
+    }
+
+    private char read() throws IOException {
+      int value = reader.read();
+      position++;
+      return (char) value;
+    }
+
+    private void unread(char value) throws IOException {
+      reader.unread(value);
+      position--;
+    }
+
+    private boolean hasNext() throws IOException {
+      char value = read();
+      boolean hasNext = value != (char) -1;
+      unread(value);
+      return hasNext;
+    }
+
+    @Override
+    public void close() throws IOException {
+      reader.close();
+    }
+
+    public String readArg() throws IOException {
+      if (!hasNext()) {
+        return null;
+      }
+
+      StringBuilder arg = new StringBuilder();
+
+      int quoteStart = -1;
+      boolean quoted = false;
+      char current;
+
+      while ((current = read()) != (char) -1) {
+        if (quoted) {
+          if (current == '\'') {
+            StringBuilder escapedQuoteRemainder =
+                new StringBuilder().append(read()).append(read()).append(read());
+            if (escapedQuoteRemainder.toString().equals("\\''")) {
+              arg.append("'");
+            } else {
+              for (char c : escapedQuoteRemainder.reverse().toString().toCharArray()) {
+                unread(c);
+              }
+              quoted = false;
+              quoteStart = -1;
+            }
+          } else {
+            arg.append(current);
+          }
+        } else {
+          if (current == '\'') {
+            quoted = true;
+            quoteStart = position;
+          } else if (current == '\r') {
+            char next = read();
+            if (next == '\n') {
+              return arg.toString();
+            } else {
+              unread(next);
+              return arg.toString();
+            }
+          } else if (Character.isWhitespace(current)) {
+            return arg.toString();
+          } else {
+            arg.append(current);
+          }
+        }
+      }
+      if (quoted) {
+        throw new IOException(
+            String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "'", quoteStart));
+      }
+      return arg.toString();
+    }
+  }
+}
diff --git a/src/main/java/com/google/devtools/common/options/UnquotedParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/UnquotedParamsFilePreProcessor.java
new file mode 100644
index 0000000000..a754a68092
--- /dev/null
+++ b/src/main/java/com/google/devtools/common/options/UnquotedParamsFilePreProcessor.java
@@ -0,0 +1,40 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.common.options;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * A {@link ParamsFilePreProcessor} that processes a parameter file using the {@code
+ * com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType.UNQUOTED} format. This
+ * format assumes each parameter is on a separate line and does not perform any special handling on
+ * non-newline whitespace or special characters.
+ */
+public class UnquotedParamsFilePreProcessor extends ParamsFilePreProcessor {
+
+  public UnquotedParamsFilePreProcessor(FileSystem fs) {
+    super(fs);
+  }
+
+  @Override
+  protected List<String> parse(Path paramsFile) throws IOException {
+    return Files.readAllLines(paramsFile, UTF_8);
+  }
+}