// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.rules.java; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static java.nio.charset.StandardCharsets.ISO_8859_1; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionKeyContext; import com.google.devtools.build.lib.actions.ActionOwner; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.BaseSpawn; import com.google.devtools.build.lib.actions.CommandLine; import com.google.devtools.build.lib.actions.CommandLineExpansionException; import com.google.devtools.build.lib.actions.CommandLines; import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ParamFileInfo; import com.google.devtools.build.lib.actions.ParameterFile; import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; import com.google.devtools.build.lib.actions.ResourceSet; import com.google.devtools.build.lib.actions.RunfilesSupplier; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.ParameterFileWriteAction; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.rules.java.JavaPluginInfoProvider.JavaPluginInfo; import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.util.LazyString; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.ArrayList; import java.util.Collection; import java.util.List; import javax.annotation.Nullable; /** * Action for Java header compilation, to be used if --java_header_compilation is enabled. * *

The header compiler consumes the inputs of a java compilation, and produces an interface jar * that can be used as a compile-time jar by upstream targets. The header interface jar is * equivalent to the output of ijar, but unlike ijar the header compiler operates directly on Java * source files instead post-processing the class outputs of the compilation. Compiling the * interface jar from source moves javac off the build's critical path. * *

The implementation of the header compiler tool can be found under {@code * //src/java_tools/buildjar/java/com/google/devtools/build/java/turbine}. */ @AutoCodec public class JavaHeaderCompileAction extends SpawnAction { private static final String GUID = "952db158-2654-4ced-87e5-4646d50523cf"; private static final ResourceSet LOCAL_RESOURCES = ResourceSet.createWithRamCpuIo(/*memoryMb=*/ 750.0, /*cpuUsage=*/ 0.5, /*ioUsage=*/ 0.0); private final Iterable directInputs; @Nullable private final CommandLine directCommandLine; /** The command line for a direct classpath compilation, or {@code null} if disabled. */ @VisibleForTesting @Nullable public CommandLine directCommandLine() { return directCommandLine; } /** * Constructs an action to compile a set of Java source files to a header interface jar. * * @param owner the action owner, typically a java_* RuleConfiguredTarget * @param tools the set of files comprising the tool that creates the header interface jar * @param directInputs the set of direct input artifacts of the compile action * @param inputs the set of transitive input artifacts of the compile action * @param outputs the outputs of the action * @param primaryOutput the output jar * @param commandLines the transitive command line arguments for the java header compiler * @param directCommandLine the direct command line arguments for the java header compiler * @param commandLineLimits the command line limits * @param progressMessage the message printed during the progression of the build */ protected JavaHeaderCompileAction( ActionOwner owner, Iterable tools, Iterable directInputs, Iterable inputs, Iterable outputs, Artifact primaryOutput, CommandLines commandLines, CommandLine directCommandLine, CommandLineLimits commandLineLimits, CharSequence progressMessage, RunfilesSupplier runfilesSupplier) { super( owner, tools, inputs, outputs, primaryOutput, LOCAL_RESOURCES, commandLines, commandLineLimits, false, // TODO(#3320): This is missing the config's action environment. JavaCompileAction.UTF8_ACTION_ENVIRONMENT, /* executionInfo= */ ImmutableMap.of(), progressMessage, runfilesSupplier, "Turbine", /* executeUnconditionally= */ false, /* extraActionInfoSupplier= */ null); this.directInputs = checkNotNull(directInputs); this.directCommandLine = checkNotNull(directCommandLine); } @Override protected void computeKey(ActionKeyContext actionKeyContext, Fingerprint fp) { fp.addString(GUID); try { super.computeKey(actionKeyContext, fp); fp.addStrings(directCommandLine.arguments()); } catch (CommandLineExpansionException e) { throw new AssertionError("JavaHeaderCompileAction command line expansion cannot fail"); } } @Override protected List internalExecute(ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { Spawn spawn = getDirectSpawn(); SpawnActionContext context = actionExecutionContext.getContext(SpawnActionContext.class); try { return context.exec(spawn, actionExecutionContext); } catch (ExecException e) { // if the direct input spawn failed, try again with transitive inputs to produce better // better messages try { return context.exec(getSpawn(actionExecutionContext), actionExecutionContext); } catch (CommandLineExpansionException commandLineExpansionException) { throw new UserExecException(commandLineExpansionException); } // The compilation should never fail with direct deps but succeed with transitive inputs // unless it failed due to a strict deps error, in which case fall back to the transitive // classpath may allow it to succeed (Strict Java Deps errors are reported by javac, // not turbine). } } private final Spawn getDirectSpawn() { try { return new BaseSpawn( ImmutableList.copyOf(directCommandLine.arguments()), ImmutableMap.of() /*environment*/, ImmutableMap.of() /*executionInfo*/, this, LOCAL_RESOURCES) { @Override public Iterable getInputFiles() { return directInputs; } }; } catch (CommandLineExpansionException e) { throw new AssertionError("JavaHeaderCompileAction command line expansion cannot fail"); } } /** Builder class to construct Java header compilation actions. */ public static class Builder { private final RuleContext ruleContext; private Artifact outputJar; @Nullable private Artifact outputDepsProto; private ImmutableSet sourceFiles = ImmutableSet.of(); private final Collection sourceJars = new ArrayList<>(); private NestedSet classpathEntries = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); private ImmutableList bootclasspathEntries = ImmutableList.of(); @Nullable private Label targetLabel; @Nullable private String injectingRuleKind; private PathFragment tempDirectory; private BuildConfiguration.StrictDepsMode strictJavaDeps = BuildConfiguration.StrictDepsMode.OFF; private NestedSet directJars = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); private NestedSet compileTimeDependencyArtifacts = NestedSetBuilder.emptySet(Order.STABLE_ORDER); private ImmutableList javacOpts; private JavaPluginInfo plugins = JavaPluginInfo.empty(); private NestedSet additionalInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); private Artifact javacJar; private NestedSet toolsJars = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); public Builder(RuleContext ruleContext) { this.ruleContext = ruleContext; } /** Sets the output jdeps file. */ public Builder setOutputDepsProto(@Nullable Artifact outputDepsProto) { this.outputDepsProto = outputDepsProto; return this; } /** Sets the direct dependency artifacts. */ public Builder setDirectJars(NestedSet directJars) { checkNotNull(directJars, "directJars must not be null"); this.directJars = directJars; return this; } /** Sets the .jdeps artifacts for direct dependencies. */ public Builder setCompileTimeDependencyArtifacts(NestedSet dependencyArtifacts) { checkNotNull(dependencyArtifacts, "dependencyArtifacts must not be null"); this.compileTimeDependencyArtifacts = dependencyArtifacts; return this; } /** Sets Java compiler flags. */ public Builder setJavacOpts(ImmutableList javacOpts) { checkNotNull(javacOpts, "javacOpts must not be null"); this.javacOpts = javacOpts; return this; } /** Sets the output jar. */ public Builder setOutputJar(Artifact outputJar) { checkNotNull(outputJar, "outputJar must not be null"); this.outputJar = outputJar; return this; } /** Adds Java source files to compile. */ public Builder setSourceFiles(ImmutableSet sourceFiles) { checkNotNull(sourceFiles, "sourceFiles must not be null"); this.sourceFiles = sourceFiles; return this; } /** Adds a jar archive of Java sources to compile. */ public Builder addSourceJars(Collection sourceJars) { checkNotNull(sourceJars, "sourceJars must not be null"); this.sourceJars.addAll(sourceJars); return this; } /** Sets the compilation classpath entries. */ public Builder setClasspathEntries(NestedSet classpathEntries) { checkNotNull(classpathEntries, "classpathEntries must not be null"); this.classpathEntries = classpathEntries; return this; } /** Sets the compilation bootclasspath entries. */ public Builder setBootclasspathEntries(ImmutableList bootclasspathEntries) { checkNotNull(bootclasspathEntries, "bootclasspathEntries must not be null"); this.bootclasspathEntries = bootclasspathEntries; return this; } /** Sets the annotation processors classpath entries. */ public Builder setPlugins(JavaPluginInfo plugins) { checkNotNull(plugins, "plugins must not be null"); checkState(this.plugins.isEmpty()); this.plugins = plugins; return this; } /** Sets the label of the target being compiled. */ public Builder setTargetLabel(@Nullable Label targetLabel) { this.targetLabel = targetLabel; return this; } /** Sets the injecting rule kind of the target being compiled. */ public Builder setInjectingRuleKind(@Nullable String injectingRuleKind) { this.injectingRuleKind = injectingRuleKind; return this; } /** * Sets the path to a temporary directory, e.g. for extracting sourcejar entries to before * compilation. */ public Builder setTempDirectory(PathFragment tempDirectory) { checkNotNull(tempDirectory, "tempDirectory must not be null"); this.tempDirectory = tempDirectory; return this; } /** Sets the Strict Java Deps mode. */ public Builder setStrictJavaDeps(BuildConfiguration.StrictDepsMode strictJavaDeps) { checkNotNull(strictJavaDeps, "strictJavaDeps must not be null"); this.strictJavaDeps = strictJavaDeps; return this; } /** Sets the javabase inputs. */ public Builder setAdditionalInputs(NestedSet additionalInputs) { checkNotNull(additionalInputs, "additionalInputs must not be null"); this.additionalInputs = additionalInputs; return this; } /** Sets the javac jar. */ public Builder setJavacJar(Artifact javacJar) { checkNotNull(javacJar, "javacJar must not be null"); this.javacJar = javacJar; return this; } /** Sets the tools jars. */ public Builder setToolsJars(NestedSet toolsJars) { checkNotNull(toolsJars, "toolsJars must not be null"); this.toolsJars = toolsJars; return this; } /** Builds and registers the {@link JavaHeaderCompileAction} for a header compilation. */ public void build(JavaToolchainProvider javaToolchain, JavaRuntimeInfo hostJavabase) { checkNotNull(outputDepsProto, "outputDepsProto must not be null"); checkNotNull(sourceFiles, "sourceFiles must not be null"); checkNotNull(sourceJars, "sourceJars must not be null"); checkNotNull(classpathEntries, "classpathEntries must not be null"); checkNotNull(bootclasspathEntries, "bootclasspathEntries must not be null"); checkNotNull(tempDirectory, "tempDirectory must not be null"); checkNotNull(strictJavaDeps, "strictJavaDeps must not be null"); checkNotNull(directJars, "directJars must not be null"); checkNotNull( compileTimeDependencyArtifacts, "compileTimeDependencyArtifacts must not be null"); checkNotNull(javacOpts, "javacOpts must not be null"); // Invariant: if strictJavaDeps is OFF, then directJars and // dependencyArtifacts are ignored if (strictJavaDeps == BuildConfiguration.StrictDepsMode.OFF) { directJars = NestedSetBuilder.emptySet(Order.NAIVE_LINK_ORDER); compileTimeDependencyArtifacts = NestedSetBuilder.emptySet(Order.STABLE_ORDER); } // The compilation uses API-generating annotation processors and has to fall back to // javac-turbine. boolean requiresAnnotationProcessing = !plugins.isEmpty(); NestedSet tools = NestedSetBuilder.stableOrder() .add(javacJar) .addTransitive(javaToolchain.getHeaderCompiler().getFilesToRun()) .addTransitive(toolsJars) .build(); ImmutableList outputs = ImmutableList.of(outputJar, outputDepsProto); NestedSet baseInputs = NestedSetBuilder.stableOrder() .addTransitive(hostJavabase.javaBaseInputsMiddleman()) .addTransitive(additionalInputs) .addAll(bootclasspathEntries) .addAll(sourceJars) .addAll(sourceFiles) .addTransitive(tools) .build(); boolean noFallback = ruleContext.getFragment(JavaConfiguration.class).headerCompilationDisableJavacFallback(); // The action doesn't require annotation processing and either javac-turbine fallback is // disabled, or the action doesn't distinguish between direct and transitive deps, so // use a plain SpawnAction to invoke turbine. if ((noFallback || directJars.isEmpty()) && !requiresAnnotationProcessing) { SpawnAction.Builder builder = new SpawnAction.Builder(); NestedSet classpath; final ParamFileInfo paramFileInfo; if (!directJars.isEmpty() || classpathEntries.isEmpty()) { classpath = directJars; paramFileInfo = null; } else { classpath = classpathEntries; // Transitive classpath actions may exceed the command line length limit. paramFileInfo = ParamFileInfo.builder(ParameterFileType.UNQUOTED).setUseAlways(true).build(); } CustomCommandLine.Builder commandLine = baseCommandLine(CustomCommandLine.builder(), classpath); if (noFallback) { commandLine.add("--nojavac_fallback"); } Artifact headerCompiler = javaToolchain.getHeaderCompiler().getExecutable(); // The header compiler is either a jar file that needs to be executed using // `java -jar `, or an executable that can be run directly. if (!headerCompiler.getExtension().equals("jar")) { builder.setExecutable(headerCompiler); builder.addTool(javaToolchain.getHeaderCompiler()); } else { builder.setJarExecutable( hostJavabase.javaBinaryExecPath(), headerCompiler, javaToolchain.getJvmOptions()); } ruleContext.registerAction( builder .addTransitiveTools(tools) .addTransitiveInputs(baseInputs) .addTransitiveInputs(classpath) .addOutputs(outputs) .addCommandLine(commandLine.build(), paramFileInfo) .setMnemonic("Turbine") .setProgressMessage(getProgressMessage()) .build(ruleContext)); return; } CustomCommandLine.Builder commandLine = getBaseArgs(javaToolchain, hostJavabase); CommandLine paramFileCommandLine = transitiveCommandLine(); NestedSetBuilder transitiveInputs = NestedSetBuilder.stableOrder() .addTransitive(baseInputs) .addTransitive(classpathEntries) .addTransitive(plugins.processorClasspath()) .addTransitive(plugins.data()) .addTransitive(compileTimeDependencyArtifacts); final CommandLines commandLines; if (ruleContext.getConfiguration().deferParamFiles()) { commandLines = CommandLines.builder() .addCommandLine(commandLine.build()) .addCommandLine( paramFileCommandLine, ParamFileInfo.builder(ParameterFile.ParameterFileType.UNQUOTED) .setUseAlways(true) .setCharset(ISO_8859_1) .build()) .build(); } else { PathFragment paramFilePath = ParameterFile.derivePath(outputJar.getRootRelativePath()); Artifact paramsFile = ruleContext .getAnalysisEnvironment() .getDerivedArtifact(paramFilePath, outputJar.getRoot()); transitiveInputs.add(paramsFile); commandLine.addFormatted("@%s", paramsFile.getExecPath()); commandLines = CommandLines.of(commandLine.build()); ParameterFileWriteAction parameterFileWriteAction = new ParameterFileWriteAction( ruleContext.getActionOwner(), paramsFile, paramFileCommandLine, ParameterFile.ParameterFileType.UNQUOTED, ISO_8859_1); ruleContext.registerAction(parameterFileWriteAction); } if (requiresAnnotationProcessing) { // turbine doesn't support API-generating annotation processors, so skip the two-tiered // turbine/javac-turbine action and just use SpawnAction to invoke javac-turbine. ruleContext.registerAction( new SpawnAction( ruleContext.getActionOwner(), tools, transitiveInputs.build(), outputs, outputJar, LOCAL_RESOURCES, commandLines, ruleContext.getConfiguration().getCommandLineLimits(), false, // TODO(b/63280599): This is missing the config's action environment. JavaCompileAction.UTF8_ACTION_ENVIRONMENT, /* executionInfo= */ ImmutableMap.of(), getProgressMessageWithAnnotationProcessors(), javaToolchain.getHeaderCompiler().getRunfilesSupplier(), "JavacTurbine", /* executeUnconditionally= */ false, /* extraActionInfoSupplier= */ null)); return; } // The action doesn't require annotation processing, javac-turbine fallback is enabled, and // the target distinguishes between direct and transitive deps. Try a two-tiered spawn // the invokes turbine with direct deps, and falls back to javac-turbine on failures to // produce better diagnostics. (At the cost of slower failed actions and a larger // cache footprint.) // TODO(cushon): productionize --nojavac_fallback and remove this path checkState(!directJars.isEmpty()); NestedSet directInputs = NestedSetBuilder.fromNestedSet(baseInputs).addTransitive(directJars).build(); CustomCommandLine directCommandLine = baseCommandLine( getBaseArgs(javaToolchain, hostJavabase), directJars) .build(); ruleContext.registerAction( new JavaHeaderCompileAction( ruleContext.getActionOwner(), tools, directInputs, transitiveInputs.build(), outputs, outputJar, commandLines, directCommandLine, ruleContext.getConfiguration().getCommandLineLimits(), getProgressMessage(), javaToolchain.getHeaderCompiler().getRunfilesSupplier())); } private LazyString getProgressMessageWithAnnotationProcessors() { List shortNames = new ArrayList<>(); for (String name : plugins.processorClasses()) { shortNames.add(name.substring(name.lastIndexOf('.') + 1)); } String tail = " and running annotation processors (" + Joiner.on(", ").join(shortNames) + ")"; return getProgressMessage(tail); } private LazyString getProgressMessage() { return getProgressMessage(""); } private LazyString getProgressMessage(String tail) { Artifact outputJar = this.outputJar; int fileCount = sourceFiles.size() + sourceJars.size(); return new LazyString() { @Override public String toString() { return String.format( "Compiling Java headers %s (%d files)%s", outputJar.prettyPrint(), fileCount, tail); } }; } private CustomCommandLine.Builder getBaseArgs( JavaToolchainProvider javaToolchain, JavaRuntimeInfo hostJavabase) { Artifact headerCompiler = javaToolchain.getHeaderCompiler().getExecutable(); if (!headerCompiler.getExtension().equals("jar")) { return CustomCommandLine.builder().addExecPath(headerCompiler); } else { return CustomCommandLine.builder() .addPath(hostJavabase.javaBinaryExecPath()) .add("-Xverify:none") .addAll(javaToolchain.getJvmOptions()) .addExecPath("-jar", headerCompiler); } } /** * Adds the command line arguments shared by direct classpath and transitive classpath * invocations. */ private CustomCommandLine.Builder baseCommandLine( CustomCommandLine.Builder result, NestedSet classpathEntries) { result.addExecPath("--output", outputJar); if (outputDepsProto != null) { result.addExecPath("--output_deps", outputDepsProto); } result.add("--temp_dir").addPath(tempDirectory); result.addExecPaths("--bootclasspath", bootclasspathEntries); result.addExecPaths("--sources", sourceFiles); if (!sourceJars.isEmpty()) { result.addExecPaths("--source_jars", ImmutableList.copyOf(sourceJars)); } if (!javacOpts.isEmpty()) { result.addAll("--javacopts", javacOpts); // terminate --javacopts with `--` to support javac flags that start with `--` result.add("--"); } if (targetLabel != null) { result.add("--target_label"); if (targetLabel.getPackageIdentifier().getRepository().isDefault() || targetLabel.getPackageIdentifier().getRepository().isMain()) { result.addLabel(targetLabel); } else { // @-prefixed strings will be assumed to be params filenames and expanded, // so add an extra @ to escape it. result.addPrefixedLabel("@", targetLabel); } } if (injectingRuleKind != null) { result.add("--injecting_rule_kind", injectingRuleKind); } result.addExecPaths("--classpath", classpathEntries); return result; } /** Builds a transitive classpath command line. */ private CommandLine transitiveCommandLine() { CustomCommandLine.Builder result = CustomCommandLine.builder(); baseCommandLine(result, classpathEntries); result.addAll("--processors", plugins.processorClasses()); result.addExecPaths("--processorpath", plugins.processorClasspath()); if (strictJavaDeps != BuildConfiguration.StrictDepsMode.OFF) { result.addExecPaths("--direct_dependencies", directJars); if (!compileTimeDependencyArtifacts.isEmpty()) { result.addExecPaths("--deps_artifacts", compileTimeDependencyArtifacts); } } return result.build(); } } }