diff options
Diffstat (limited to 'src/main/java/com/google/devtools')
5 files changed, 349 insertions, 114 deletions
diff --git a/src/main/java/com/google/devtools/common/options/LegacyParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/LegacyParamsFilePreProcessor.java new file mode 100644 index 0000000000..9e8eeb0b17 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/LegacyParamsFilePreProcessor.java @@ -0,0 +1,147 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import java.io.IOException; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * A {@link ParamsFilePreProcessor} that processes a parameter file using a custom format. This + * format assumes each parameter is separated by whitespace and allows arguments to use single and + * double quotes and quote and whitespace escaping. + */ +public class LegacyParamsFilePreProcessor extends ParamsFilePreProcessor { + + public LegacyParamsFilePreProcessor(FileSystem fs) { + super(fs); + } + + @Override + protected List<String> parse(Path paramsFile) throws IOException, OptionsParsingException { + try (Reader params = Files.newBufferedReader(paramsFile, StandardCharsets.UTF_8)) { + List<String> newArgs = new ArrayList<>(); + StringBuilder arg = new StringBuilder(); + CharIterator iterator = CharIterator.wrap(params); + while (iterator.hasNext()) { + char next = iterator.next(); + if (Character.isWhitespace(next) && !iterator.isInQuote() && !iterator.isEscaped()) { + newArgs.add(unescape(arg.toString())); + arg = new StringBuilder(); + } else { + arg.append(next); + } + } + // If there is an arg in the buffer, add it. + if (arg.length() > 0) { + newArgs.add(arg.toString()); + } + // If we're still in a quote by the end of the file, throw an error. + if (iterator.isInQuote()) { + throw new OptionsParsingException( + String.format(ERROR_MESSAGE_FORMAT, paramsFile, iterator.getUnmatchedQuoteMessage())); + } + return newArgs; + } + } + + private String unescape(String arg) { + if (arg.startsWith("'") && arg.endsWith("'")) { + String unescaped = arg.replace("'\\''", "'"); + return unescaped.substring(1, unescaped.length() - 1); + } + return arg; + } + + // Doesn't implement iterator to avoid autoboxing and to throw exceptions. + private static class CharIterator { + + private final Reader reader; + private int readerPosition = 0; + private int singleQuoteStart = -1; + private int doubleQuoteStart = -1; + private boolean escaped = false; + private char lastChar = (char) -1; + + public static CharIterator wrap(Reader reader) { + return new CharIterator(reader); + } + + public CharIterator(Reader reader) { + this.reader = reader; + } + + public boolean hasNext() throws IOException { + return peek() != -1; + } + + private int peek() throws IOException { + reader.mark(1); + int next = reader.read(); + reader.reset(); + return next; + } + + public boolean isInQuote() { + return singleQuoteStart != -1 || doubleQuoteStart != -1; + } + + public boolean isEscaped() { + return escaped; + } + + public String getUnmatchedQuoteMessage() { + StringBuilder message = new StringBuilder(); + if (singleQuoteStart != -1) { + message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "'", singleQuoteStart)); + } + if (doubleQuoteStart != -1) { + message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "\"", doubleQuoteStart)); + } + return message.toString(); + } + + public char next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + char current = (char) reader.read(); + + // check for \r\n line endings. If found, drop the \r for normalized parsing. + if (current == '\r' && peek() == '\n') { + current = (char) reader.read(); + } + + // check to see if the current position is escaped + escaped = (lastChar == '\\'); + + if (!escaped && current == '\'') { + singleQuoteStart = singleQuoteStart == -1 ? readerPosition : -1; + } + if (!escaped && current == '"') { + doubleQuoteStart = doubleQuoteStart == -1 ? readerPosition : -1; + } + + readerPosition++; + lastChar = current; + return current; + } + } +} diff --git a/src/main/java/com/google/devtools/common/options/OptionsParser.java b/src/main/java/com/google/devtools/common/options/OptionsParser.java index 3e6b5ccba8..28c2206fbc 100644 --- a/src/main/java/com/google/devtools/common/options/OptionsParser.java +++ b/src/main/java/com/google/devtools/common/options/OptionsParser.java @@ -200,9 +200,16 @@ public class OptionsParser implements OptionsProvider { this.impl.setAllowSingleDashLongOptions(allowSingleDashLongOptions); } - /** Enables the Parser to handle params files loacted insinde the provided {@link FileSystem}. */ + /** Enables the Parser to handle params files located inside the provided {@link FileSystem}. */ public void enableParamsFileSupport(FileSystem fs) { - this.impl.setArgsPreProcessor(new ParamsFilePreProcessor(fs)); + enableParamsFileSupport(new LegacyParamsFilePreProcessor(fs)); + } + + /** + * Enables the Parser to handle params files using the provided {@link ParamsFilePreProcessor}. + */ + public void enableParamsFileSupport(ParamsFilePreProcessor preProcessor) { + this.impl.setArgsPreProcessor(preProcessor); } public void parseAndExitUponError(String[] args) { diff --git a/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java index 791265ae1b..87f87f6118 100644 --- a/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java +++ b/src/main/java/com/google/devtools/common/options/ParamsFilePreProcessor.java @@ -14,14 +14,9 @@ package com.google.devtools.common.options; import java.io.IOException; -import java.io.Reader; -import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; -import java.util.NoSuchElementException; /** * Defines an {@link ArgsPreProcessor} that will determine if the arguments list contains a "params" @@ -31,7 +26,7 @@ import java.util.NoSuchElementException; * length. A params file argument is defined as a path starting with @. It will also be the only * entry in an argument list. */ -public class ParamsFilePreProcessor implements ArgsPreProcessor { +public abstract class ParamsFilePreProcessor implements ArgsPreProcessor { static final String ERROR_MESSAGE_FORMAT = "Error reading params file: %s %s"; @@ -50,7 +45,7 @@ public class ParamsFilePreProcessor implements ArgsPreProcessor { * Parses the param file path and replaces the arguments list with the contents if one exists. * * @param args A list of arguments that may contain @<path> to a params file. - * @return A list of areguments suitable for parsing. + * @return A list of arguments suitable for parsing. * @throws OptionsParsingException if the path does not exist. */ @Override @@ -61,29 +56,8 @@ public class ParamsFilePreProcessor implements ArgsPreProcessor { String.format(TOO_MANY_ARGS_ERROR_MESSAGE_FORMAT, args), args.get(0)); } Path path = fs.getPath(args.get(0).substring(1)); - try (Reader params = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { - List<String> newArgs = new ArrayList<>(); - StringBuilder arg = new StringBuilder(); - CharIterator iterator = CharIterator.wrap(params); - while (iterator.hasNext()) { - char next = iterator.next(); - if (Character.isWhitespace(next) && !iterator.isInQuote() && !iterator.isEscaped()) { - newArgs.add(unescape(arg.toString())); - arg = new StringBuilder(); - } else { - arg.append(next); - } - } - // If there is an arg in the buffer, add it. - if (arg.length() > 0) { - newArgs.add(arg.toString()); - } - // If we're still in a quote by the end of the file, throw an error. - if (iterator.isInQuote()) { - throw new OptionsParsingException( - String.format(ERROR_MESSAGE_FORMAT, path, iterator.getUnmatchedQuoteMessage())); - } - return newArgs; + try { + return parse(path); } catch (RuntimeException | IOException e) { throw new OptionsParsingException( String.format(ERROR_MESSAGE_FORMAT, path, e.getMessage()), args.get(0), e); @@ -92,86 +66,15 @@ public class ParamsFilePreProcessor implements ArgsPreProcessor { return args; } - private String unescape(String arg) { - if (arg.startsWith("'") && arg.endsWith("'")) { - String unescaped = arg.replace("'\\''", "'"); - return unescaped.substring(1, unescaped.length() - 1); - } - return arg; - } - - // Doesn't implement iterator to avoid autoboxing and to throw exceptions. - static class CharIterator { - - private final Reader reader; - private int readerPosition = 0; - private int singleQuoteStart = -1; - private int doubleQuoteStart = -1; - private boolean escaped = false; - private char lastChar = (char) -1; - - public static CharIterator wrap(Reader reader) { - return new CharIterator(reader); - } - - public CharIterator(Reader reader) { - this.reader = reader; - } - - public boolean hasNext() throws IOException { - return peek() != -1; - } - - private int peek() throws IOException { - reader.mark(1); - int next = reader.read(); - reader.reset(); - return next; - } - - public boolean isInQuote() { - return singleQuoteStart != -1 || doubleQuoteStart != -1; - } - - public boolean isEscaped() { - return escaped; - } - - public String getUnmatchedQuoteMessage() { - StringBuilder message = new StringBuilder(); - if (singleQuoteStart != -1) { - message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "'", singleQuoteStart)); - } - if (doubleQuoteStart != -1) { - message.append(String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "\"", doubleQuoteStart)); - } - return message.toString(); - } - - public char next() throws IOException { - if (!hasNext()) { - throw new NoSuchElementException(); - } - char current = (char) reader.read(); - - // check for \r\n line endings. If found, drop the \r for normalized parsing. - if (current == '\r' && peek() == '\n') { - current = (char) reader.read(); - } - - // check to see if the current position is escaped - escaped = (lastChar == '\\'); - - if (!escaped && current == '\'') { - singleQuoteStart = singleQuoteStart == -1 ? readerPosition : -1; - } - if (!escaped && current == '"') { - doubleQuoteStart = doubleQuoteStart == -1 ? readerPosition : -1; - } - - readerPosition++; - lastChar = current; - return current; - } - } + /** + * Parses the paramsFile and returns a list of argument tokens to be further processed by the + * {@link OptionsParser}. + * + * @param paramsFile The path of the params file to parse. + * @return a list of argument tokens. + * @throws IOException if there is an error reading paramsFile. + * @throws OptionsParsingException if there is an error reading paramsFile. + */ + protected abstract List<String> parse(Path paramsFile) + throws IOException, OptionsParsingException; } diff --git a/src/main/java/com/google/devtools/common/options/ShellQuotedParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/ShellQuotedParamsFilePreProcessor.java new file mode 100644 index 0000000000..525cb77227 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/ShellQuotedParamsFilePreProcessor.java @@ -0,0 +1,138 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.io.PushbackReader; +import java.io.Reader; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * A {@link ParamsFilePreProcessor} that processes a parameter file using the {@code + * com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType.SHELL_QUOTED} format. This + * format assumes each parameter is separated by whitespace and is quoted using singe quotes + * ({@code '}) if it contains any special characters or is an empty string. + */ +public class ShellQuotedParamsFilePreProcessor extends ParamsFilePreProcessor { + + public ShellQuotedParamsFilePreProcessor(FileSystem fs) { + super(fs); + } + + @Override + protected List<String> parse(Path paramsFile) throws IOException { + List<String> args = new ArrayList<>(); + try (ShellQuotedReader reader = + new ShellQuotedReader(Files.newBufferedReader(paramsFile, UTF_8))) { + String arg; + while ((arg = reader.readArg()) != null) { + args.add(arg); + } + } + return args; + } + + private static class ShellQuotedReader implements AutoCloseable { + + private final PushbackReader reader; + private int position = -1; + + public ShellQuotedReader(Reader reader) { + this.reader = new PushbackReader(reader, 10); + } + + private char read() throws IOException { + int value = reader.read(); + position++; + return (char) value; + } + + private void unread(char value) throws IOException { + reader.unread(value); + position--; + } + + private boolean hasNext() throws IOException { + char value = read(); + boolean hasNext = value != (char) -1; + unread(value); + return hasNext; + } + + @Override + public void close() throws IOException { + reader.close(); + } + + public String readArg() throws IOException { + if (!hasNext()) { + return null; + } + + StringBuilder arg = new StringBuilder(); + + int quoteStart = -1; + boolean quoted = false; + char current; + + while ((current = read()) != (char) -1) { + if (quoted) { + if (current == '\'') { + StringBuilder escapedQuoteRemainder = + new StringBuilder().append(read()).append(read()).append(read()); + if (escapedQuoteRemainder.toString().equals("\\''")) { + arg.append("'"); + } else { + for (char c : escapedQuoteRemainder.reverse().toString().toCharArray()) { + unread(c); + } + quoted = false; + quoteStart = -1; + } + } else { + arg.append(current); + } + } else { + if (current == '\'') { + quoted = true; + quoteStart = position; + } else if (current == '\r') { + char next = read(); + if (next == '\n') { + return arg.toString(); + } else { + unread(next); + return arg.toString(); + } + } else if (Character.isWhitespace(current)) { + return arg.toString(); + } else { + arg.append(current); + } + } + } + if (quoted) { + throw new IOException( + String.format(UNFINISHED_QUOTE_MESSAGE_FORMAT, "'", quoteStart)); + } + return arg.toString(); + } + } +} diff --git a/src/main/java/com/google/devtools/common/options/UnquotedParamsFilePreProcessor.java b/src/main/java/com/google/devtools/common/options/UnquotedParamsFilePreProcessor.java new file mode 100644 index 0000000000..a754a68092 --- /dev/null +++ b/src/main/java/com/google/devtools/common/options/UnquotedParamsFilePreProcessor.java @@ -0,0 +1,40 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.common.options; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * A {@link ParamsFilePreProcessor} that processes a parameter file using the {@code + * com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType.UNQUOTED} format. This + * format assumes each parameter is on a separate line and does not perform any special handling on + * non-newline whitespace or special characters. + */ +public class UnquotedParamsFilePreProcessor extends ParamsFilePreProcessor { + + public UnquotedParamsFilePreProcessor(FileSystem fs) { + super(fs); + } + + @Override + protected List<String> parse(Path paramsFile) throws IOException { + return Files.readAllLines(paramsFile, UTF_8); + } +} |