diff options
author | tomlu <tomlu@google.com> | 2017-09-19 15:48:14 +0200 |
---|---|---|
committer | László Csomor <laszlocsomor@google.com> | 2017-09-19 17:18:04 +0200 |
commit | 33a361fcc34b6b7c9dbec31209ec22f0ddfbb2ff (patch) | |
tree | 4df7c2f918f637579c5514188e7cc8b107facf2b /src/main/java/com/google | |
parent | 6434af053cfffa38909bc86afe69e33b573b6179 (diff) |
Implement support for Skylark param files.
PiperOrigin-RevId: 169234249
Diffstat (limited to 'src/main/java/com/google')
3 files changed, 144 insertions, 34 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java index 6e94070ff1..9091e82c2d 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/ParamFileInfo.java @@ -18,7 +18,6 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; import com.google.devtools.build.lib.util.Preconditions; -import com.google.errorprone.annotations.CompileTimeConstant; import java.nio.charset.Charset; import java.util.Objects; import javax.annotation.concurrent.Immutable; @@ -113,7 +112,7 @@ public final class ParamFileInfo { * <p>The format string must have a single "%s" that will be replaced by the execution path to * the param file. */ - public Builder setFlagFormatString(@CompileTimeConstant String flagFormatString) { + public Builder setFlagFormatString(String flagFormatString) { this.flagFormatString = flagFormatString; return this; } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java index 8a8ccc8ec1..c09eb68375 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkActionFactory.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; import com.google.devtools.build.lib.actions.Root; import com.google.devtools.build.lib.actions.RunfilesSupplier; import com.google.devtools.build.lib.actions.extra.SpawnInfo; @@ -25,8 +26,8 @@ import com.google.devtools.build.lib.analysis.FilesToRunProvider; import com.google.devtools.build.lib.analysis.PseudoAction; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.actions.CommandLine; -import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.ParamFileInfo; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction; import com.google.devtools.build.lib.analysis.actions.TemplateExpansionAction.Substitution; @@ -57,6 +58,7 @@ import com.google.devtools.build.lib.syntax.SkylarkSemanticsOptions; import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor; import com.google.devtools.build.lib.vfs.PathFragment; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; @@ -296,7 +298,10 @@ public class SkylarkActionFactory implements SkylarkValue { defaultValue = "[]", named = true, positional = false, - doc = "command line arguments of the action." + doc = + "command line arguments of the action. " + + "Must be a list of strings or actions.args() objects. " + + "See <a href=\"actions.html#args\">ctx.actions.args()</a>." ), @Param( name = "mnemonic", @@ -378,15 +383,8 @@ public class SkylarkActionFactory implements SkylarkValue { context.checkMutable("actions.run"); SpawnAction.Builder builder = new SpawnAction.Builder(); - if (arguments instanceof SkylarkList) { - SkylarkList skylarkList = ((SkylarkList) arguments); - @SuppressWarnings("unchecked") - List<String> argumentsContents = skylarkList.getContents(String.class, "arguments"); - builder.addCommandLine(CustomCommandLine.builder().addAll(argumentsContents).build()); - } else { - Args args = (Args) arguments; - builder.addCommandLine(args.build()); - } + SkylarkList argumentsList = ((SkylarkList) arguments); + buildCommandLine(builder, argumentsList); if (executableUnchecked instanceof Artifact) { Artifact executable = (Artifact) executableUnchecked; builder.addInput(executable); @@ -451,6 +449,7 @@ public class SkylarkActionFactory implements SkylarkValue { positional = false, doc = "command line arguments of the action. " + + "Must be a list of strings or actions.args() objects.<br>" + "Blaze passes the elements in this attribute as arguments to the command." + "The command can access these arguments as <code>$1</code>, <code>$2</code>, " + "etc. See <a href=\"actions.html#args\">ctx.actions.args()</a>." @@ -556,26 +555,9 @@ public class SkylarkActionFactory implements SkylarkValue { context.checkMutable("actions.run_shell"); // TODO(bazel-team): builder still makes unnecessary copies of inputs, outputs and args. + SkylarkList argumentList = (SkylarkList) arguments; SpawnAction.Builder builder = new SpawnAction.Builder(); - if (arguments instanceof SkylarkList) { - CustomCommandLine.Builder commandLine = CustomCommandLine.builder(); - SkylarkList argumentList = (SkylarkList) arguments; - if (argumentList.size() > 0) { - // When we use a shell command, add an empty argument before other arguments. - // e.g. bash -c "cmd" '' 'arg1' 'arg2' - // bash will use the empty argument as the value of $0 (which we don't care about). - // arg1 and arg2 will be $1 and $2, as a user expects. - commandLine.add(""); - } - - @SuppressWarnings("unchecked") - List<String> argumentsContents = argumentList.getContents(String.class, "arguments"); - commandLine.addAll(argumentsContents); - builder.addCommandLine(commandLine.build()); - } else { - Args args = (Args) arguments; - builder.addCommandLine(CommandLine.concat(ImmutableList.of(""), args.build())); - } + buildCommandLine(builder, argumentList); if (commandUnchecked instanceof String) { Map<String, String> executionInfo = @@ -609,6 +591,13 @@ public class SkylarkActionFactory implements SkylarkValue { "expected string or list of strings for command instead of " + EvalUtils.getDataTypeName(commandUnchecked)); } + if (argumentList.size() > 0) { + // When we use a shell command, add an empty argument before other arguments. + // e.g. bash -c "cmd" '' 'arg1' 'arg2' + // bash will use the empty argument as the value of $0 (which we don't care about). + // arg1 and arg2 will be $1 and $2, as a user expects. + builder.addExecutableArguments(""); + } registerSpawnAction( outputs, inputs, @@ -621,6 +610,31 @@ public class SkylarkActionFactory implements SkylarkValue { builder); } + private void buildCommandLine(SpawnAction.Builder builder, SkylarkList argumentsList) + throws EvalException { + List<String> stringArgs = new ArrayList<>(); + for (Object value : argumentsList) { + if (value instanceof String) { + stringArgs.add((String) value); + } else if (value instanceof Args) { + if (!stringArgs.isEmpty()) { + builder.addCommandLine(CommandLine.of(stringArgs)); + stringArgs = new ArrayList<>(); + } + Args args = (Args) value; + builder.addCommandLine(args.commandLine.build(), args.paramFileInfo); + } else { + throw new EvalException( + null, + "expected list of strings or ctx.actions.args() for arguments instead of " + + EvalUtils.getDataTypeName(value)); + } + } + if (!stringArgs.isEmpty()) { + builder.addCommandLine(CommandLine.of(stringArgs)); + } + } + /** * Setup for spawn actions common between {@link #run} and {@link #runShell}. * @@ -815,9 +829,11 @@ public class SkylarkActionFactory implements SkylarkValue { + "# ]" + "</pre>" ) - static class Args { + static class Args implements SkylarkValue { + private final SkylarkRuleContext context; private final SkylarkCustomCommandLine.Builder commandLine; + private ParamFileInfo paramFileInfo; @SkylarkSignature( name = "add", @@ -894,6 +910,9 @@ public class SkylarkActionFactory implements SkylarkValue { Object mapFn, Location loc) throws EvalException { + if (self.isImmutable()) { + throw new EvalException(null, "cannot modify frozen value"); + } if (value instanceof SkylarkNestedSet || value instanceof SkylarkList) { self.addVectorArg(value, format, beforeEach, joinWith, mapFn, loc); } else { @@ -960,7 +979,79 @@ public class SkylarkActionFactory implements SkylarkValue { } } - public Args(SkylarkSemanticsOptions skylarkSemantics, EventHandler eventHandler) { + @SkylarkCallable( + name = "use_param_file", + doc = + "Spills the args to a params file, replacing them with a pointer to the param file. " + + "Use when your args may be too large for the system's command length limits ", + parameters = { + @Param( + name = "param_file_arg", + type = String.class, + named = true, + doc = + "a format string with a single \"%s\". " + + "If the args are spilled to a params file then they are replaced " + + "with an argument consisting of this string formatted with" + + "the path of the params file." + ), + @Param( + name = "use_always", + type = Boolean.class, + named = true, + positional = false, + defaultValue = "False", + doc = + "whether to always spill the args to a params file. If false, " + + "bazel will decide whether the arguments need to be spilled " + + "based on your system and arg length." + ), + @Param( + name = "format", + type = String.class, + named = true, + positional = false, + defaultValue = "\"shell\"", + doc = + "the format of the param file. Must be one of:<br>" + + "\"shell\": All arguments are shell quoted and separated by whitespace<br>" + + "\"multiline\": All arguments are unquoted and separated by newline characters" + ) + } + ) + public void useParamsFile(String paramFileArg, Boolean useAlways, String format) + throws EvalException { + if (isImmutable()) { + throw new EvalException(null, "cannot modify frozen value"); + } + if (!paramFileArg.contains("%s")) { + throw new EvalException( + null, + "Invalid value for parameter \"param_file_arg\": Expected string with a single \"%s\""); + } + final ParameterFileType parameterFileType; + if (format.equals("shell")) { + parameterFileType = ParameterFileType.SHELL_QUOTED; + } else if (format.equals("multiline")) { + parameterFileType = ParameterFileType.UNQUOTED; + } else { + throw new EvalException( + null, + "Invalid value for parameter \"format\": Expected one of \"shell\", \"multiline\""); + } + + paramFileInfo = + ParamFileInfo.builder(parameterFileType) + .setUseAlways(useAlways) + .setFlagFormatString(paramFileArg) + .build(); + } + + public Args( + SkylarkRuleContext context, + SkylarkSemanticsOptions skylarkSemantics, + EventHandler eventHandler) { + this.context = context; this.commandLine = new SkylarkCustomCommandLine.Builder(skylarkSemantics, eventHandler); } @@ -968,11 +1059,30 @@ public class SkylarkActionFactory implements SkylarkValue { return commandLine.build(); } + @Override + public boolean isImmutable() { + return context.isImmutable(); + } + + @Override + public void repr(SkylarkPrinter printer) { + printer.append("context.args() object"); + } + static { SkylarkSignatureProcessor.configureSkylarkFunctions(Args.class); } } + @SkylarkCallable( + name = "args", + doc = "returns an Args object that can be used to build memory-efficient command lines." + ) + public Args args() { + return new Args( + context, skylarkSemanticsOptions, ruleContext.getAnalysisEnvironment().getEventHandler()); + } + @Override public boolean isImmutable() { return context.isImmutable(); diff --git a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java index a04fd1309c..5d7ced8940 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/skylark/SkylarkRuleImplementationFunctions.java @@ -125,6 +125,7 @@ public class SkylarkRuleImplementationFunctions { positional = false, doc = "command line arguments of the action." + + "Must be a list of strings or actions.args() objects." ), @Param( name = "mnemonic", |