path: root/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoCompileActionBuilder.java

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.rules.proto;

import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Iterables.isEmpty;
import static com.google.devtools.build.lib.collect.nestedset.Order.STABLE_ORDER;
import static com.google.devtools.build.lib.rules.proto.ProtoCommon.areDepsStrict;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ParameterFile;
import com.google.devtools.build.lib.actions.ResourceSet;
import com.google.devtools.build.lib.analysis.FilesToRunProvider;
import com.google.devtools.build.lib.analysis.MakeVariableExpander;
import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.RuleContext;
import com.google.devtools.build.lib.analysis.actions.CustomCommandLine;
import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
import com.google.devtools.build.lib.analysis.actions.SpawnAction;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.util.LazyString;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;

/** Constructs actions to run the protocol compiler to generate sources from .proto files. */
public class ProtoCompileActionBuilder {
  private static final String MNEMONIC = "GenProto";
  private static final ResourceSet GENPROTO_RESOURCE_SET =
      ResourceSet.createWithRamCpuIo(100, .1, .0);
  private static final Action[] NO_ACTIONS = new Action[0];

  private RuleContext ruleContext;
  private SupportData supportData;
  private String language;
  private String langPrefix;
  private Iterable<Artifact> outputs;
  private String langParameter;
  private String langPluginName;
  private String langPluginParameter;
  private Supplier<String> langPluginParameterSupplier;
  private boolean hasServices;
  private Iterable<String> additionalCommandLineArguments;
  private Iterable<FilesToRunProvider> additionalTools;

  /** Build a proto compiler commandline argument for use in setXParameter methods. */
  public static String buildProtoArg(String arg, String value, Iterable<String> flags) {
    return String.format(
        "--%s=%s%s", arg, (isEmpty(flags) ? "" : Joiner.on(',').join(flags) + ":"), value);
  }

  public ProtoCompileActionBuilder setRuleContext(RuleContext ruleContext) {
    this.ruleContext = ruleContext;
    return this;
  }

  public ProtoCompileActionBuilder setSupportData(SupportData supportData) {
    this.supportData = supportData;
    return this;
  }

  public ProtoCompileActionBuilder setLanguage(String language) {
    this.language = language;
    return this;
  }

  public ProtoCompileActionBuilder setLangPrefix(String langPrefix) {
    this.langPrefix = langPrefix;
    return this;
  }

  public ProtoCompileActionBuilder allowServices(boolean hasServices) {
    this.hasServices = hasServices;
    return this;
  }

  public ProtoCompileActionBuilder setOutputs(Iterable<Artifact> outputs) {
    this.outputs = outputs;
    return this;
  }

  public ProtoCompileActionBuilder setLangParameter(String langParameter) {
    this.langParameter = langParameter;
    return this;
  }

  public ProtoCompileActionBuilder setLangPluginName(String langPluginName) {
    this.langPluginName = langPluginName;
    return this;
  }

  public ProtoCompileActionBuilder setLangPluginParameter(String langPluginParameter) {
    this.langPluginParameter = langPluginParameter;
    return this;
  }

  public ProtoCompileActionBuilder setLangPluginParameterSupplier(
      Supplier<String> langPluginParameterSupplier) {
    this.langPluginParameterSupplier = langPluginParameterSupplier;
    return this;
  }

  public ProtoCompileActionBuilder setAdditionalCommandLineArguments(
      Iterable<String> additionalCmdLine) {
    this.additionalCommandLineArguments = additionalCmdLine;
    return this;
  }

  public ProtoCompileActionBuilder setAdditionalTools(
      Iterable<FilesToRunProvider> additionalTools) {
    this.additionalTools = additionalTools;
    return this;
  }

  public ProtoCompileActionBuilder(
      RuleContext ruleContext,
      SupportData supportData,
      String language,
      String langPrefix,
      Iterable<Artifact> outputs) {
    this.ruleContext = ruleContext;
    this.supportData = supportData;
    this.language = language;
    this.langPrefix = langPrefix;
    this.outputs = outputs;
  }

  /** Static class to avoid keeping a reference to this builder after build() is called. */
  private static class LazyLangPluginFlag extends LazyString {
    private final String langPrefix;
    private final Supplier<String> langPluginParameter1;

    LazyLangPluginFlag(String langPrefix, Supplier<String> langPluginParameter1) {
      this.langPrefix = langPrefix;
      this.langPluginParameter1 = langPluginParameter1;
    }

    @Override
    public String toString() {
      return String.format("--%s_out=%s", langPrefix, langPluginParameter1.get());
    }
  }

  private static class LazyCommandLineExpansion extends LazyString {
    // E.g., --java_out=%s
    private final String template;
    private final Map<String, ? extends CharSequence> variableValues;

    private LazyCommandLineExpansion(
        String template, Map<String, ? extends CharSequence> variableValues) {
      this.template = template;
      this.variableValues = variableValues;
    }

    @Override
    public String toString() {
      try {
        return MakeVariableExpander.expand(
            template,
            new MakeVariableExpander.Context() {
              @Override
              public String lookupMakeVariable(String var)
                  throws MakeVariableExpander.ExpansionException {
                CharSequence value = variableValues.get(var);
                return value != null ? value.toString() : var;
              }
            });
      } catch (MakeVariableExpander.ExpansionException e) {
        // Squeelch. We don't throw this exception in the lookupMakeVariable implementation above,
        // and we can't report it here anyway, because this code will typically execute in the
        // Execution phase.
      }
      return template;
    }
  }

  public Action[] build() {
    checkState(
        langPluginParameter == null || langPluginParameterSupplier == null,
        "Only one of {langPluginParameter, langPluginParameterSupplier} should be set.");

    if (isEmpty(outputs)) {
      return NO_ACTIONS;
    }

    try {
      return createAction().build(ruleContext);
    } catch (MissingPrerequisiteException e) {
      return NO_ACTIONS;
    }
  }

  private SpawnAction.Builder createAction() throws MissingPrerequisiteException {
    SpawnAction.Builder result =
        new SpawnAction.Builder().addTransitiveInputs(supportData.getTransitiveImports());

    FilesToRunProvider langPluginTarget = getLangPluginTarget();
    if (langPluginTarget != null) {
      result.addTool(langPluginTarget);
    }

    FilesToRunProvider compilerTarget =
        ruleContext.getExecutablePrerequisite(":proto_compiler", RuleConfiguredTarget.Mode.HOST);
    if (compilerTarget == null) {
      throw new MissingPrerequisiteException();
    }

    if (this.additionalTools != null) {
      for (FilesToRunProvider tool : additionalTools) {
        result.addTool(tool);
      }
    }

    result
        .useParameterFile(ParameterFile.ParameterFileType.UNQUOTED)
        .addOutputs(outputs)
        .setResources(GENPROTO_RESOURCE_SET)
        .useDefaultShellEnvironment()
        .setExecutable(compilerTarget)
        .setCommandLine(createProtoCompilerCommandLine().build())
        .setProgressMessage("Generating " + language + " proto_library " + ruleContext.getLabel())
        .setMnemonic(MNEMONIC);

    return result;
  }

  @Nullable
  private FilesToRunProvider getLangPluginTarget() throws MissingPrerequisiteException {
    if (langPluginName == null) {
      return null;
    }
    FilesToRunProvider result =
        ruleContext.getExecutablePrerequisite(langPluginName, RuleConfiguredTarget.Mode.HOST);
    if (result == null) {
      throw new MissingPrerequisiteException();
    }
    return result;
  }

  /** Commandline generator for protoc invocations. */
  @VisibleForTesting
  CustomCommandLine.Builder createProtoCompilerCommandLine() throws MissingPrerequisiteException {
    CustomCommandLine.Builder result = CustomCommandLine.builder();

    if (langPluginName == null) {
      if (langParameter != null) {
        result.add(langParameter);
      }
    } else {
      FilesToRunProvider langPluginTarget = getLangPluginTarget();
      Supplier<String> langPluginParameter1 =
          langPluginParameter == null
              ? langPluginParameterSupplier
              : Suppliers.ofInstance(langPluginParameter);

      Preconditions.checkArgument(langParameter == null);
      Preconditions.checkArgument(langPluginParameter1 != null);
      // We pass a separate langPluginName as there are plugins that cannot be overridden
      // and thus we have to deal with "$xx_plugin" and "xx_plugin".
      result.add(
          String.format(
              "--plugin=protoc-gen-%s=%s",
              langPrefix, langPluginTarget.getExecutable().getExecPathString()));
      result.add(new LazyLangPluginFlag(langPrefix, langPluginParameter1));
    }

    result.add(ruleContext.getFragment(ProtoConfiguration.class).protocOpts());

    boolean areDepsStrict = areDepsStrict(ruleContext);

    // Add include maps
    result.add(
        new ProtoCommandLineArgv(
            areDepsStrict ? supportData.getProtosInDirectDeps() : null,
            supportData.getTransitiveImports()));

    if (areDepsStrict) {
      // Note: the %s in the line below is used by proto-compiler. That is, the string we create
      // here should have a literal %s in it.
      result.add(
          createStrictProtoDepsViolationErrorMessage(ruleContext.getLabel().getCanonicalForm()));
    }

    for (Artifact src : supportData.getDirectProtoSources()) {
      result.addPath(src.getRootRelativePath());
    }

    if (!hasServices) {
      result.add("--disallow_services");
    }

    if (additionalCommandLineArguments != null) {
      result.add(additionalCommandLineArguments);
    }

    return result;
  }

  /**
   * Static inner class since these objects live into the execution phase and so they must not keep
   * alive references to the surrounding analysis-phase objects.
   */
  @VisibleForTesting
  static class ProtoCommandLineArgv extends CustomCommandLine.CustomMultiArgv {
    @Nullable private final Iterable<Artifact> protosInDirectDependencies;
    private final Iterable<Artifact> transitiveImports;

    ProtoCommandLineArgv(
        @Nullable Iterable<Artifact> protosInDirectDependencies,
        Iterable<Artifact> transitiveImports) {
      this.protosInDirectDependencies = protosInDirectDependencies;
      this.transitiveImports = transitiveImports;
    }

    @Override
    public Iterable<String> argv() {
      ImmutableList.Builder<String> builder = ImmutableList.builder();
      for (Artifact artifact : transitiveImports) {
        builder.add(
            "-I" + getPathIgnoringRepository(artifact) + "=" + artifact.getExecPathString());
      }
      if (protosInDirectDependencies != null) {
        ArrayList<String> rootRelativePaths = new ArrayList<>();
        for (Artifact directDependency : protosInDirectDependencies) {
          rootRelativePaths.add(getPathIgnoringRepository(directDependency));
        }
        builder.add("--direct_dependencies=" + Joiner.on(":").join(rootRelativePaths));
      }
      return builder.build();
    }

    /**
     * Gets the artifact's path relative to the root, ignoring the external repository the artifact
     * is at. For example, <code>
     * //a:b.proto --> a/b.proto
     * {@literal @}foo//a:b.proto --> a/b.proto
     * </code>
     */
    private static String getPathIgnoringRepository(Artifact artifact) {
      return artifact
          .getRootRelativePath()
          .relativeTo(
              artifact
                  .getOwnerLabel()
                  .getPackageIdentifier()
                  .getRepository()
                  .getPathUnderExecRoot())
          .toString();
    }
  }

  /** Signifies that a prerequisite could not be satisfied. */
  private static class MissingPrerequisiteException extends Exception {}

  public static void writeDescriptorSet(
      RuleContext ruleContext,
      final CharSequence outReplacement,
      Collection<Artifact> protosToCompile,
      NestedSet<Artifact> transitiveSources,
      NestedSet<Artifact> protosInDirectDeps,
      Artifact output,
      boolean allowServices,
      NestedSet<Artifact> transitiveDescriptorSets) {
    if (protosToCompile.isEmpty()) {
      ruleContext.registerAction(
          FileWriteAction.createEmptyWithInputs(
              ruleContext.getActionOwner(), transitiveDescriptorSets, output));
      return;
    }

    SpawnAction.Builder actions =
        createActions(
            ruleContext,
            ImmutableList.of(createDescriptorSetToolchain(outReplacement)),
            protosToCompile,
            transitiveSources,
            protosInDirectDeps,
            ruleContext.getLabel().getCanonicalForm(),
            ImmutableList.of(output),
            "Descriptor Set",
            allowServices);
    if (actions == null) {
      return;
    }

    actions.setMnemonic("GenProtoDescriptorSet");
    actions.addTransitiveInputs(transitiveDescriptorSets);
    ruleContext.registerAction(actions.build(ruleContext));
  }

  private static ToolchainInvocation createDescriptorSetToolchain(CharSequence outReplacement) {
    return new ToolchainInvocation(
        "dontcare",
        ProtoLangToolchainProvider.create(
            "--descriptor_set_out=$(OUT)",
            null /* pluginExecutable */,
            null /* runtime */,
            NestedSetBuilder.<Artifact>emptySet(STABLE_ORDER) /* blacklistedProtos */),
        outReplacement);
  }

  /**
   * Registers actions to generate code from .proto files.
   *
   * <p>This method uses information from proto_lang_toolchain() rules. New rules should use this
   * method instead of the soup of methods above.
   *
   * @param toolchainInvocations See {@link #createCommandLineFromToolchains}.
   * @param ruleLabel See {@link #createCommandLineFromToolchains}.
   * @param outputs The artifacts that the resulting action must create.
   * @param flavorName e.g., "Java (Immutable)"
   * @param allowServices If false, the compilation will break if any .proto file has service
   */
  public static void registerActions(
      RuleContext ruleContext,
      List<ToolchainInvocation> toolchainInvocations,
      Iterable<Artifact> protosToCompile,
      NestedSet<Artifact> transitiveSources,
      NestedSet<Artifact> protosInDirectDeps,
      String ruleLabel,
      Iterable<Artifact> outputs,
      String flavorName,
      boolean allowServices) {
    SpawnAction.Builder actions =
        createActions(
            ruleContext,
            toolchainInvocations,
            protosToCompile,
            transitiveSources,
            protosInDirectDeps,
            ruleLabel,
            outputs,
            flavorName,
            allowServices);
    if (actions != null) {
      ruleContext.registerAction(actions.build(ruleContext));
    }
  }

  @Nullable
  private static SpawnAction.Builder createActions(
      RuleContext ruleContext,
      List<ToolchainInvocation> toolchainInvocations,
      Iterable<Artifact> protosToCompile,
      NestedSet<Artifact> transitiveSources,
      @Nullable NestedSet<Artifact> protosInDirectDeps,
      String ruleLabel,
      Iterable<Artifact> outputs,
      String flavorName,
      boolean allowServices) {

    if (isEmpty(outputs)) {
      return null;
    }

    SpawnAction.Builder result = new SpawnAction.Builder().addTransitiveInputs(transitiveSources);

    for (ToolchainInvocation invocation : toolchainInvocations) {
      ProtoLangToolchainProvider toolchain = invocation.toolchain;
      if (toolchain.pluginExecutable() != null) {
        result.addTool(toolchain.pluginExecutable());
      }
    }

    FilesToRunProvider compilerTarget =
        ruleContext.getExecutablePrerequisite(":proto_compiler", RuleConfiguredTarget.Mode.HOST);
    if (compilerTarget == null) {
      return null;
    }

    result
        .useParameterFile(ParameterFile.ParameterFileType.UNQUOTED)
        .addOutputs(outputs)
        .setResources(GENPROTO_RESOURCE_SET)
        .useDefaultShellEnvironment()
        .setExecutable(compilerTarget)
        .setCommandLine(
            createCommandLineFromToolchains(
                toolchainInvocations,
                protosToCompile,
                transitiveSources,
                areDepsStrict(ruleContext) ? protosInDirectDeps : null,
                ruleLabel,
                allowServices,
                ruleContext.getFragment(ProtoConfiguration.class).protocOpts()))
        .setProgressMessage("Generating " + flavorName + " proto_library " + ruleContext.getLabel())
        .setMnemonic(MNEMONIC);

    return result;
  }

  /**
   * Constructs command-line arguments to execute proto-compiler.
   *
   * <ul>
   *   <li>Each toolchain contributes a command-line, formatted from its commandLine() method.
   *   <li>$(OUT) is replaced with the outReplacement field of ToolchainInvocation.
   *   <li>$(PLUGIN_out) is replaced with PLUGIN_<key>_out where 'key' is the key of
   *       toolchainInvocations. The key thus allows multiple plugins in one command-line.
   *   <li>If a toolchain's {@code plugin()} is non-null, we point at it by emitting
   *       --plugin=protoc-gen-PLUGIN_<key>=<location of plugin>.
   * </ul>
   *
   * Note {@code toolchainInvocations} is ordered, and affects the order in which plugins are
   * called. As some plugins rely on output from other plugins, their order matters.
   *
   * @param toolchainInvocations See {@link #createCommandLineFromToolchains}.
   * @param ruleLabel Name of the proto_library for which we're compiling. This string is used to
   *     populate an error message format that's passed to proto-compiler.
   * @param allowServices If false, the compilation will break if any .proto file has
   */
  @VisibleForTesting
  static CustomCommandLine createCommandLineFromToolchains(
      List<ToolchainInvocation> toolchainInvocations,
      Iterable<Artifact> protosToCompile,
      NestedSet<Artifact> transitiveSources,
      @Nullable NestedSet<Artifact> protosInDirectDeps,
      String ruleLabel,
      boolean allowServices,
      ImmutableList<String> protocOpts) {
    CustomCommandLine.Builder cmdLine = CustomCommandLine.builder();

    // A set to check if there are multiple invocations with the same name.
    HashSet<String> invocationNames = new HashSet<>();

    for (ToolchainInvocation invocation : toolchainInvocations) {
      if (!invocationNames.add(invocation.name)) {
        throw new IllegalStateException(
            "Invocation name "
                + invocation.name
                + " appears more than once. "
                + "This could lead to incorrect proto-compiler behavior");
      }

      ProtoLangToolchainProvider toolchain = invocation.toolchain;

      cmdLine.add(
          new LazyCommandLineExpansion(
              toolchain.commandLine(),
              ImmutableMap.of(
                  "OUT",
                  invocation.outReplacement,
                  "PLUGIN_OUT",
                  String.format("PLUGIN_%s_out", invocation.name))));

      if (toolchain.pluginExecutable() != null) {
        cmdLine.add(
            String.format(
                "--plugin=protoc-gen-%s=%s",
                String.format("PLUGIN_%s", invocation.name),
                toolchain.pluginExecutable().getExecutable().getExecPathString()));
      }
    }

    cmdLine.add(protocOpts);

    // Add include maps
    cmdLine.add(new ProtoCommandLineArgv(protosInDirectDeps, transitiveSources));

    if (protosInDirectDeps != null) {
      cmdLine.add(createStrictProtoDepsViolationErrorMessage(ruleLabel));
    }

    for (Artifact src : protosToCompile) {
      cmdLine.addPath(src.getRootRelativePath());
    }

    if (!allowServices) {
      cmdLine.add("--disallow_services");
    }

    return cmdLine.build();
  }

  @SuppressWarnings("FormatString") // Errorprone complains that there's no '%s' in the format
  // string, but it's actually in MESSAGE.
  @VisibleForTesting
  public static String createStrictProtoDepsViolationErrorMessage(String ruleLabel) {
    return "--direct_dependencies_violation_msg="
        + String.format(StrictProtoDepsViolationMessage.MESSAGE, ruleLabel);
  }

  /**
   * Describes a toolchain and the value to replace for a $(OUT) that might appear in its
   * commandLine() (e.g., "bazel-out/foo.srcjar").
   */
  public static class ToolchainInvocation {
    final String name;
    public final ProtoLangToolchainProvider toolchain;
    final CharSequence outReplacement;

    public ToolchainInvocation(
        String name, ProtoLangToolchainProvider toolchain, CharSequence outReplacement) {
      checkState(!name.contains(" "), "Name %s should not contain spaces", name);
      this.name = name;
      this.toolchain = toolchain;
      this.outReplacement = outReplacement;
    }
  }
}