// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.rules.objc; import static com.google.devtools.build.lib.rules.objc.XcodeProductType.LIBRARY_STATIC; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.actions.CustomCommandLine; import com.google.devtools.build.lib.analysis.actions.FileWriteAction; import com.google.devtools.build.lib.analysis.actions.SpawnAction; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.rules.proto.ProtoSourcesProvider; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.PathFragment; import java.util.HashMap; import java.util.Set; /** * Support for generating Objective C proto static libraries that registers actions which generate * and compile the Objective C protos by using the open source protobuf library and compiler. * *

Each group represents one proto_library target depended on by objc_proto_library targets using * the portable_proto_filters attribute. This group contains all the necessary protos to satisfy its * internal dependencies. * *

Grouping has a first pass in which for each proto required to be built, we find the smallest * group containing it, and store that information in a map. We then reverse that map into a multi * map, in which the keys are the input protos and the values are the output protos to be * generated/compiled with the input group as dependencies. This minimizes the number of inputs * required for each generation/compilation action and the probability of regeneration when one of * the proto files change, improving cache hits. */ final class ProtobufSupport { private static final String BUNDLED_PROTOS_IDENTIFIER = "BundledProtos"; private static final String UNIQUE_DIRECTORY_NAME = "_generated_protos"; private final RuleContext ruleContext; private final BuildConfiguration buildConfiguration; private final ProtoAttributes attributes; // Each entry of this map represents a generation action and a compilation action. The input set // are dependencies of the output set. The output set is always a subset of, or the same set as, // the input set. For example, given a sample entry of the inputsToOutputsMap like: // // {A, B, C} => {B, C} // // this represents: // 1. A generation action in which the inputs are A, B and C, and the outputs are B.pbobjc.h, // B.pbobjc.m, C.pbobjc.h and C.pbobjc.m. // 2. A compilation action in which the inputs are A.pbobjc.h, B.pbobjc.h, C.pbobjc.h, // B.pbobjc.m and C.pbobjc.m, while the outputs are B.pbobjc.o and C.pbobjc.o. // // Given that each input set appears only once, by the nature of the structure, we can safely use // it as an identifier of the entry. private final ImmutableSetMultimap, Artifact> inputsToOutputsMap; /** * Creates a new proto support for the protobuf library. This support code bundles up all the * transitive protos within the groups in which they were defined. We use that information to * minimize the number of inputs per generation/compilation actions by only providing what is * really needed to the actions. * * @param ruleContext context this proto library is constructed in */ public ProtobufSupport(RuleContext ruleContext) throws RuleErrorException { this(ruleContext, null); } /** * Creates a new proto support for the protobuf library. This support code bundles up all the * transitive protos within the groups in which they were defined. We use that information to * minimize the number of inputs per generation/compilation actions by only providing what is * really needed to the actions. * * @param ruleContext context this proto library is constructed in * @param buildConfiguration the configuration from which to get prerequisites when building proto * targets in a split configuration */ public ProtobufSupport(RuleContext ruleContext, BuildConfiguration buildConfiguration) throws RuleErrorException { this.ruleContext = ruleContext; this.buildConfiguration = buildConfiguration; this.attributes = new ProtoAttributes(ruleContext); this.inputsToOutputsMap = getInputsToOutputsMap(); } /** * Registers the proto generation actions. These actions generate the ObjC/CPP code to be compiled * by this rule. */ public ProtobufSupport registerGenerationActions() { int actionId = 0; for (ImmutableSet inputProtos : inputsToOutputsMap.keySet()) { Iterable outputProtos = inputsToOutputsMap.get(inputProtos); registerGenerationAction(outputProtos, inputProtos, getUniqueBundledProtosSuffix(actionId)); actionId++; } if (!isLinkingTarget()) { registerModuleMapGenerationAction(); } return this; } private void registerModuleMapGenerationAction() { IntermediateArtifacts moduleMapIntermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); CompilationArtifacts.Builder moduleMapCompilationArtifacts = new CompilationArtifacts.Builder() .setIntermediateArtifacts(moduleMapIntermediateArtifacts) .setPchFile(Optional.absent()) .addAdditionalHdrs(getProtobufHeaders()) .addAdditionalHdrs( getGeneratedProtoOutputs(inputsToOutputsMap.values(), ".pbobjc.h")); new CompilationSupport( ruleContext, ObjcRuleClasses.intermediateArtifacts(ruleContext), new CompilationAttributes.Builder().build()) .registerGenerateModuleMapAction(Optional.of(moduleMapCompilationArtifacts.build())); } /** Registers the actions that will compile the generated code. */ public ProtobufSupport registerCompilationActions() { int actionId = 0; Iterable userHeaderSearchPaths = ImmutableList.of(getWorkspaceRelativeOutputDir()); for (ImmutableSet inputProtos : inputsToOutputsMap.keySet()) { ImmutableSet outputProtos = inputsToOutputsMap.get(inputProtos); IntermediateArtifacts intermediateArtifacts = getUniqueIntermediateArtifacts(actionId); CompilationArtifacts compilationArtifacts = getCompilationArtifacts(intermediateArtifacts, inputProtos, outputProtos); ObjcCommon common = getCommon(intermediateArtifacts, compilationArtifacts); new CompilationSupport( ruleContext, intermediateArtifacts, new CompilationAttributes.Builder().build()) .registerCompileAndArchiveActions(common, userHeaderSearchPaths); actionId++; } return this; } /** Adds the generated files to the set of files to be output when this rule is built. */ public ProtobufSupport addFilesToBuild(NestedSetBuilder filesToBuild) { for (ImmutableSet inputProtoFiles : inputsToOutputsMap.keySet()) { ImmutableSet outputProtoFiles = inputsToOutputsMap.get(inputProtoFiles); Iterable generatedSources = getGeneratedProtoOutputs(outputProtoFiles, ".pbobjc.m"); Iterable generatedHeaders = getGeneratedProtoOutputs(outputProtoFiles, ".pbobjc.h"); filesToBuild.addAll(generatedSources).addAll(generatedHeaders); } int actionId = 0; for (ImmutableSet inputProtos : inputsToOutputsMap.keySet()) { ImmutableSet outputProtos = inputsToOutputsMap.get(inputProtos); IntermediateArtifacts intermediateArtifacts = getUniqueIntermediateArtifacts(actionId); CompilationArtifacts compilationArtifacts = getCompilationArtifacts(intermediateArtifacts, inputProtos, outputProtos); ObjcCommon common = getCommon(intermediateArtifacts, compilationArtifacts); filesToBuild.addAll(common.getCompiledArchive().asSet()); actionId++; } return this; } /** * Returns the ObjcProvider for this target, or Optional.absent() if there were no protos to * generate. */ public Optional getObjcProvider() { if (inputsToOutputsMap.isEmpty()) { return Optional.absent(); } Iterable userHeaderSearchPaths = ImmutableList.of(getWorkspaceRelativeOutputDir()); IntermediateArtifacts moduleMapIntermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); ObjcCommon.Builder commonBuilder = new ObjcCommon.Builder(ruleContext) .setIntermediateArtifacts(moduleMapIntermediateArtifacts) .setHasModuleMap(); int actionId = 0; for (ImmutableSet inputProtos : inputsToOutputsMap.keySet()) { ImmutableSet outputProtos = inputsToOutputsMap.get(inputProtos); IntermediateArtifacts intermediateArtifacts = getUniqueIntermediateArtifacts(actionId); CompilationArtifacts compilationArtifacts = getCompilationArtifacts(intermediateArtifacts, inputProtos, outputProtos); ObjcCommon common = getCommon(intermediateArtifacts, compilationArtifacts); commonBuilder.addDepObjcProviders(ImmutableSet.of(common.getObjcProvider())); actionId++; } if (isLinkingTarget()) { commonBuilder.addUserHeaderSearchPaths(userHeaderSearchPaths); } else { commonBuilder.addDirectDependencyHeaderSearchPaths(userHeaderSearchPaths); } return Optional.of(commonBuilder.build().getObjcProvider()); } /** * Returns the XcodeProvider for this target or Optional.absent() if there were no protos to * generate. */ public Optional getXcodeProvider() throws RuleErrorException { if (inputsToOutputsMap.isEmpty()) { return Optional.absent(); } XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder(); IntermediateArtifacts intermediateArtifacts = ObjcRuleClasses.intermediateArtifacts(ruleContext); new XcodeSupport(ruleContext, intermediateArtifacts, getXcodeLabel(getBundledProtosSuffix())) .addXcodeSettings(xcodeProviderBuilder, getObjcProvider().get(), LIBRARY_STATIC); int actionId = 0; for (ImmutableSet inputProtos : inputsToOutputsMap.keySet()) { ImmutableSet outputProtos = inputsToOutputsMap.get(inputProtos); IntermediateArtifacts bundleIntermediateArtifacts = getUniqueIntermediateArtifacts(actionId); CompilationArtifacts compilationArtifacts = getCompilationArtifacts(bundleIntermediateArtifacts, inputProtos, outputProtos); ObjcCommon common = getCommon(bundleIntermediateArtifacts, compilationArtifacts); XcodeProvider bundleProvider = getBundleXcodeProvider( common, bundleIntermediateArtifacts, getUniqueBundledProtosSuffix(actionId)); if (isLinkingTarget()) { xcodeProviderBuilder.addPropagatedDependencies(ImmutableSet.of(bundleProvider)); } else { xcodeProviderBuilder.addPropagatedDependenciesWithStrictDependencyHeaders( ImmutableSet.of(bundleProvider)); } actionId++; } return Optional.of(xcodeProviderBuilder.build()); } private NestedSet getPortableProtoFilters() { Iterable objcProtoProviders = getObjcProtoProviders(); NestedSetBuilder portableProtoFilters = NestedSetBuilder.stableOrder(); for (ObjcProtoProvider objcProtoProvider : objcProtoProviders) { portableProtoFilters.addTransitive(objcProtoProvider.getPortableProtoFilters()); } portableProtoFilters.addAll(attributes.getPortableProtoFilters()); return portableProtoFilters.build(); } private NestedSet getProtobufHeaders() { Iterable objcProtoProviders = getObjcProtoProviders(); NestedSetBuilder protobufHeaders = NestedSetBuilder.stableOrder(); for (ObjcProtoProvider objcProtoProvider : objcProtoProviders) { protobufHeaders.addTransitive(objcProtoProvider.getProtobufHeaders()); } return protobufHeaders.build(); } private NestedSet getProtobufHeaderSearchPaths() { Iterable objcProtoProviders = getObjcProtoProviders(); NestedSetBuilder protobufHeaderSearchPaths = NestedSetBuilder.stableOrder(); for (ObjcProtoProvider objcProtoProvider : objcProtoProviders) { protobufHeaderSearchPaths.addTransitive(objcProtoProvider.getProtobufHeaderSearchPaths()); } return protobufHeaderSearchPaths.build(); } private ImmutableSetMultimap, Artifact> getInputsToOutputsMap() throws RuleErrorException { Iterable objcProtoProviders = getObjcProtoProviders(); Iterable protoProviders = getProtoSourcesProviders(); ImmutableList.Builder> protoSets = new ImmutableList.Builder>(); // Traverse all the dependencies ObjcProtoProviders and ProtoSourcesProviders to aggregate // all the transitive groups of proto. for (ObjcProtoProvider objcProtoProvider : objcProtoProviders) { protoSets.addAll(objcProtoProvider.getProtoGroups()); } for (ProtoSourcesProvider protoProvider : protoProviders) { protoSets.add(protoProvider.getTransitiveProtoSources()); } HashMap> protoToGroupMap = new HashMap<>(); // For each proto in each proto group, store the smallest group in which it is contained. This // group will be considered the smallest input group with which the proto can be generated. for (NestedSet nestedProtoSet : protoSets.build()) { ImmutableSet protoSet = ImmutableSet.copyOf(nestedProtoSet.toSet()); for (Artifact proto : protoSet) { // If the proto is well known, don't store it as we don't need to generate it; it comes // generated with the runtime library. if (attributes.isProtoWellKnown(proto)) { continue; } if (!protoToGroupMap.containsKey(proto)) { protoToGroupMap.put(proto, protoSet); } else { protoToGroupMap.put(proto, Sets.intersection(protoSet, protoToGroupMap.get(proto))); } } } // Now that we have the smallest proto inputs groups for each proto to be generated, we reverse // that map into a multimap to take advantage of the fact that multiple protos can be generated // with the same inputs, to avoid having multiple generation actions with the same inputs and // different ouputs. This only applies for the generation actions, as the compilation actions // compile one generated file at a time. // It's OK to use ImmutableSet as the key, since Artifact caches it's hashCode, and // ImmutableSet calculates it's hashCode in O(n). ImmutableSetMultimap.Builder, Artifact> inputsToOutputsMapBuilder = ImmutableSetMultimap.builder(); for (Artifact proto : protoToGroupMap.keySet()) { inputsToOutputsMapBuilder.put(ImmutableSet.copyOf(protoToGroupMap.get(proto)), proto); } return inputsToOutputsMapBuilder.build(); } private XcodeProvider getBundleXcodeProvider( ObjcCommon common, IntermediateArtifacts intermediateArtifacts, String labelSuffix) throws RuleErrorException { Iterable userHeaderSearchPaths = ImmutableList.of(getWorkspaceRelativeOutputDir()); XcodeProvider.Builder xcodeProviderBuilder = new XcodeProvider.Builder() .addUserHeaderSearchPaths(userHeaderSearchPaths) .setCompilationArtifacts(common.getCompilationArtifacts().get()); XcodeSupport xcodeSupport = new XcodeSupport(ruleContext, intermediateArtifacts, getXcodeLabel(labelSuffix)) .addXcodeSettings(xcodeProviderBuilder, common.getObjcProvider(), LIBRARY_STATIC); if (isLinkingTarget()) { xcodeProviderBuilder .addHeaders(getProtobufHeaders()) .addUserHeaderSearchPaths(getProtobufHeaderSearchPaths()); } else { xcodeSupport.addDependencies( xcodeProviderBuilder, new Attribute(ObjcRuleClasses.PROTO_LIB_ATTR, Mode.TARGET)); } return xcodeProviderBuilder.build(); } private String getBundledProtosSuffix() { return "_" + BUNDLED_PROTOS_IDENTIFIER; } private String getUniqueBundledProtosPrefix(int actionId) { return BUNDLED_PROTOS_IDENTIFIER + "_" + actionId; } private String getUniqueBundledProtosSuffix(int actionId) { return getBundledProtosSuffix() + "_" + actionId; } private Label getXcodeLabel(String suffix) throws RuleErrorException { Label xcodeLabel = null; try { xcodeLabel = ruleContext.getLabel().getLocalTargetLabel(ruleContext.getLabel().getName() + suffix); } catch (LabelSyntaxException e) { ruleContext.throwWithRuleError(e.getLocalizedMessage()); } return xcodeLabel; } private IntermediateArtifacts getUniqueIntermediateArtifacts(int actionId) { return new IntermediateArtifacts( ruleContext, getUniqueBundledProtosSuffix(actionId), getUniqueBundledProtosPrefix(actionId), ruleContext.getConfiguration()); } private ObjcCommon getCommon( IntermediateArtifacts intermediateArtifacts, CompilationArtifacts compilationArtifacts) { ObjcCommon.Builder commonBuilder = new ObjcCommon.Builder(ruleContext) .setIntermediateArtifacts(intermediateArtifacts) .setCompilationArtifacts(compilationArtifacts); if (isLinkingTarget()) { commonBuilder.addUserHeaderSearchPaths(getProtobufHeaderSearchPaths()); } else { commonBuilder.addDepObjcProviders( ruleContext.getPrerequisites( ObjcRuleClasses.PROTO_LIB_ATTR, Mode.TARGET, ObjcProvider.class)); } return commonBuilder.build(); } private CompilationArtifacts getCompilationArtifacts( IntermediateArtifacts intermediateArtifacts, Iterable inputProtoFiles, Iterable outputProtoFiles) { // Filter the well known protos from the set of headers. We don't generate the headers for them // as they are part of the runtime library. Iterable filteredInputProtos = attributes.filterWellKnownProtos(inputProtoFiles); CompilationArtifacts.Builder compilationArtifacts = new CompilationArtifacts.Builder() .setIntermediateArtifacts(intermediateArtifacts) .setPchFile(Optional.absent()) .addAdditionalHdrs(getGeneratedProtoOutputs(filteredInputProtos, ".pbobjc.h")) .addAdditionalHdrs(getProtobufHeaders()); if (isLinkingTarget()) { compilationArtifacts.addNonArcSrcs(getGeneratedProtoOutputs(outputProtoFiles, ".pbobjc.m")); } return compilationArtifacts.build(); } private void registerGenerationAction( Iterable outputProtos, Iterable inputProtos, String protoFileSuffix) { Artifact protoInputsFile = getProtoInputsFile(protoFileSuffix); ruleContext.registerAction( new FileWriteAction( ruleContext.getActionOwner(), protoInputsFile, getProtoInputsFileContents(outputProtos), false)); ruleContext.registerAction( new SpawnAction.Builder() .setMnemonic("GenObjcBundledProtos") .addInput(attributes.getProtoCompiler()) .addInputs(attributes.getProtoCompilerSupport()) .addTransitiveInputs(getPortableProtoFilters()) .addInput(protoInputsFile) .addInputs(inputProtos) .addOutputs(getGeneratedProtoOutputs(outputProtos, ".pbobjc.h")) .addOutputs(getGeneratedProtoOutputs(outputProtos, ".pbobjc.m")) .setExecutable(attributes.getProtoCompiler().getExecPath()) .setCommandLine(getGenerationCommandLine(protoInputsFile)) .build(ruleContext)); } private Artifact getProtoInputsFile(String suffix) { return ruleContext.getUniqueDirectoryArtifact( "_protos", "_proto_input_files" + suffix, ruleContext.getConfiguration().getGenfilesDirectory()); } private String getProtoInputsFileContents(Iterable outputProtos) { // Sort the file names to make the remote action key independent of the precise deps structure. // compile_protos.py will sort the input list anyway. Iterable sorted = Ordering.natural().immutableSortedCopy(outputProtos); return Artifact.joinRootRelativePaths("\n", sorted); } private CustomCommandLine getGenerationCommandLine(Artifact protoInputsFile) { return new CustomCommandLine.Builder() .add("--input-file-list") .add(protoInputsFile.getExecPathString()) .add("--output-dir") .add(getWorkspaceRelativeOutputDir().getSafePathString()) .add("--force") .add("--proto-root-dir") .add(getGenfilesPathString()) .add("--proto-root-dir") .add(".") .addBeforeEachExecPath("--config", getPortableProtoFilters()) .build(); } private String getGenfilesPathString() { if (buildConfiguration != null) { return buildConfiguration.getGenfilesDirectory().getExecPathString(); } return ruleContext.getConfiguration().getGenfilesDirectory().getExecPathString(); } private PathFragment getWorkspaceRelativeOutputDir() { // Generate sources in a package-and-rule-scoped directory; adds both the // package-and-rule-scoped directory and the header-containing-directory to the include path // of dependers. PathFragment rootRelativeOutputDir = ruleContext.getUniqueDirectory(UNIQUE_DIRECTORY_NAME); return new PathFragment( ruleContext.getBinOrGenfilesDirectory().getExecPath(), rootRelativeOutputDir); } private Iterable getGeneratedProtoOutputs( Iterable outputProtos, String extension) { ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (Artifact protoFile : outputProtos) { String protoFileName = FileSystemUtils.removeExtension(protoFile.getFilename()); String generatedOutputName = attributes.getGeneratedProtoFilename(protoFileName, true); PathFragment generatedFilePath = new PathFragment( protoFile.getRootRelativePath().getParentDirectory(), new PathFragment(generatedOutputName)); PathFragment outputFile = FileSystemUtils.appendExtension(generatedFilePath, extension); if (outputFile != null) { builder.add( ruleContext.getUniqueDirectoryArtifact( UNIQUE_DIRECTORY_NAME, outputFile, ruleContext.getBinOrGenfilesDirectory())); } } return builder.build(); } private Iterable getObjcProtoProviders() { if (buildConfiguration != null) { return ruleContext .getPrerequisitesByConfiguration("deps", Mode.SPLIT, ObjcProtoProvider.class) .get(buildConfiguration); } return ruleContext.getPrerequisites("deps", Mode.TARGET, ObjcProtoProvider.class); } private Iterable getProtoSourcesProviders() { if (buildConfiguration != null) { return ruleContext .getPrerequisitesByConfiguration("deps", Mode.SPLIT, ProtoSourcesProvider.class) .get(buildConfiguration); } return ruleContext.getPrerequisites("deps", Mode.TARGET, ProtoSourcesProvider.class); } private boolean isLinkingTarget() { return !ruleContext .attributes() .isAttributeValueExplicitlySpecified(ObjcProtoLibraryRule.PORTABLE_PROTO_FILTERS_ATTR); } }