From d08b27fa9701fecfdb69e1b0d1ac2459efc2129b Mon Sep 17 00:00:00 2001 From: Han-Wen Nienhuys Date: Wed, 25 Feb 2015 16:45:20 +0100 Subject: Update from Google. -- MOE_MIGRATED_REVID=85702957 --- .../devtools/build/lib/rules/cpp/CcBinary.java | 635 ++++++++ .../devtools/build/lib/rules/cpp/CcCommon.java | 678 ++++++++ .../build/lib/rules/cpp/CcCompilationOutputs.java | 207 +++ .../cpp/CcExecutionDynamicLibrariesProvider.java | 49 + .../devtools/build/lib/rules/cpp/CcLibrary.java | 395 +++++ .../build/lib/rules/cpp/CcLibraryHelper.java | 905 +++++++++++ .../devtools/build/lib/rules/cpp/CcLinkParams.java | 357 +++++ .../build/lib/rules/cpp/CcLinkParamsProvider.java | 50 + .../build/lib/rules/cpp/CcLinkParamsStore.java | 136 ++ .../build/lib/rules/cpp/CcLinkingOutputs.java | 243 +++ .../lib/rules/cpp/CcNativeLibraryProvider.java | 43 + .../rules/cpp/CcSpecificLinkParamsProvider.java | 48 + .../devtools/build/lib/rules/cpp/CcTest.java | 36 + .../devtools/build/lib/rules/cpp/CcToolchain.java | 249 +++ .../build/lib/rules/cpp/CcToolchainFeatures.java | 802 ++++++++++ .../build/lib/rules/cpp/CcToolchainProvider.java | 226 +++ .../build/lib/rules/cpp/CcToolchainRule.java | 71 + .../devtools/build/lib/rules/cpp/CppBuildInfo.java | 89 ++ .../build/lib/rules/cpp/CppCompilationContext.java | 918 +++++++++++ .../build/lib/rules/cpp/CppCompileAction.java | 1356 ++++++++++++++++ .../lib/rules/cpp/CppCompileActionBuilder.java | 439 +++++ .../lib/rules/cpp/CppCompileActionContext.java | 84 + .../build/lib/rules/cpp/CppConfiguration.java | 1691 ++++++++++++++++++++ .../lib/rules/cpp/CppConfigurationLoader.java | 174 ++ .../build/lib/rules/cpp/CppDebugFileProvider.java | 54 + .../lib/rules/cpp/CppDebugPackageProvider.java | 69 + .../devtools/build/lib/rules/cpp/CppFileTypes.java | 141 ++ .../devtools/build/lib/rules/cpp/CppHelper.java | 529 ++++++ .../build/lib/rules/cpp/CppLinkAction.java | 1074 +++++++++++++ .../build/lib/rules/cpp/CppLinkActionContext.java | 44 + .../devtools/build/lib/rules/cpp/CppModel.java | 707 ++++++++ .../devtools/build/lib/rules/cpp/CppModuleMap.java | 44 + .../build/lib/rules/cpp/CppModuleMapAction.java | 185 +++ .../devtools/build/lib/rules/cpp/CppOptions.java | 646 ++++++++ .../build/lib/rules/cpp/CppRuleClasses.java | 104 ++ .../build/lib/rules/cpp/CppRunfilesProvider.java | 85 + .../devtools/build/lib/rules/cpp/CppSemantics.java | 49 + .../cpp/CrosstoolConfigurationIdentifier.java | 132 ++ .../rules/cpp/CrosstoolConfigurationLoader.java | 327 ++++ .../rules/cpp/CrosstoolConfigurationOptions.java | 29 + .../rules/cpp/DiscoveredSourceInputsHelper.java | 139 ++ .../build/lib/rules/cpp/DwoArtifactsCollector.java | 120 ++ .../lib/rules/cpp/ExtractInclusionAction.java | 85 + .../build/lib/rules/cpp/FakeCppCompileAction.java | 212 +++ .../build/lib/rules/cpp/FdoStubAction.java | 70 + .../devtools/build/lib/rules/cpp/FdoSupport.java | 679 ++++++++ .../rules/cpp/HeaderTargetModuleMapProvider.java | 42 + .../cpp/ImplementedCcPublicLibrariesProvider.java | 42 + .../build/lib/rules/cpp/IncludeParser.java | 711 ++++++++ .../build/lib/rules/cpp/IncludeProblems.java | 51 + .../build/lib/rules/cpp/IncludeScannable.java | 90 ++ .../build/lib/rules/cpp/IncludeScanner.java | 177 ++ .../lib/rules/cpp/IncludeScanningContext.java | 44 + .../google/devtools/build/lib/rules/cpp/Link.java | 274 ++++ .../build/lib/rules/cpp/LinkCommandLine.java | 1121 +++++++++++++ .../devtools/build/lib/rules/cpp/LinkStrategy.java | 35 + .../devtools/build/lib/rules/cpp/LinkerInput.java | 51 + .../devtools/build/lib/rules/cpp/LinkerInputs.java | 353 ++++ .../devtools/build/lib/rules/cpp/LinkingMode.java | 46 + .../build/lib/rules/cpp/LipoContextProvider.java | 58 + .../build/lib/rules/cpp/LocalGccStrategy.java | 96 ++ .../build/lib/rules/cpp/LocalLinkStrategy.java | 62 + .../lib/rules/cpp/RemoteIncludeExtractor.java | 52 + .../build/lib/rules/cpp/SolibSymlinkAction.java | 234 +++ .../lib/rules/cpp/TransitiveLipoInfoProvider.java | 51 + .../lib/rules/cpp/WriteBuildInfoHeaderAction.java | 194 +++ 66 files changed, 19189 insertions(+) create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibraryHelper.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfigurationLoader.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugFileProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppDebugPackageProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppRunfilesProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeParser.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java (limited to 'src/main/java/com/google/devtools/build/lib/rules/cpp') diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java new file mode 100644 index 0000000000..6efcd9daeb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcBinary.java @@ -0,0 +1,635 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.RunfilesSupport; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.Util; +import com.google.devtools.build.lib.analysis.actions.FileWriteAction; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.TargetUtils; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A ConfiguredTarget for cc_binary rules. + */ +public abstract class CcBinary implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcBinary(CppSemantics semantics) { + this.semantics = semantics; + } + + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? + private static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + + /** + * The maximum number of inputs for any single .dwp generating action. For cases where + * this value is exceeded, the action is split up into "batches" that fall under the limit. + * See {@link #createDebugPackagerActions} for details. + */ + @VisibleForTesting + public static final int MAX_INPUTS_PER_DWP_ACTION = 100; + + /** + * Intermediate dwps are written to this subdirectory under the main dwp's output path. + */ + @VisibleForTesting + public static final String INTERMEDIATE_DWP_DIR = "_dwps"; + + private static Runfiles collectRunfiles(RuleContext context, + CcCommon common, + CcLinkingOutputs linkingOutputs, + CppCompilationContext cppCompilationContext, + LinkStaticness linkStaticness, + NestedSet filesToBuild, + Iterable fakeLinkerInputs, + boolean fake) { + Runfiles.Builder builder = new Runfiles.Builder(); + Function runfilesMapping = + CppRunfilesProvider.runfilesFunction(linkStaticness != LinkStaticness.DYNAMIC); + boolean linkshared = isLinkShared(context); + builder.addTransitiveArtifacts(filesToBuild); + // Add the shared libraries to the runfiles. This adds any shared libraries that are in the + // srcs of this target. + builder.addArtifacts(linkingOutputs.getLibrariesForRunfiles(true)); + builder.addRunfiles(context, RunfilesProvider.DEFAULT_RUNFILES); + builder.add(context, runfilesMapping); + CcToolchainProvider toolchain = CppHelper.getToolchain(context); + // Add the C++ runtime libraries if linking them dynamically. + if (linkStaticness == LinkStaticness.DYNAMIC) { + builder.addTransitiveArtifacts(toolchain.getDynamicRuntimeLinkInputs()); + } + // For cc_binary and cc_test rules, there is an implicit dependency on + // the malloc library package, which is specified by the "malloc" attribute. + // As the BUILD encyclopedia says, the "malloc" attribute should be ignored + // if linkshared=1. + if (!linkshared) { + TransitiveInfoCollection malloc = CppHelper.mallocForTarget(context); + builder.addTarget(malloc, RunfilesProvider.DEFAULT_RUNFILES); + builder.addTarget(malloc, runfilesMapping); + } + + if (fake) { + // Add the object files, libraries, and linker scripts that are used to + // link this executable. + builder.addSymlinksToArtifacts(Iterables.filter(fakeLinkerInputs, Artifact.MIDDLEMAN_FILTER)); + // The crosstool inputs for the link action are not sufficient; we also need the crosstool + // inputs for compilation. Node that these cannot be middlemen because Runfiles does not + // know how to expand them. + builder.addTransitiveArtifacts(toolchain.getCrosstool()); + builder.addTransitiveArtifacts(toolchain.getLibcLink()); + // Add the sources files that are used to compile the object files. + // We add the headers in the transitive closure and our own sources in the srcs + // attribute. We do not provide the auxiliary inputs, because they are only used when we + // do FDO compilation, and cc_fake_binary does not support FDO. + builder.addSymlinksToArtifacts( + Iterables.transform(common.getCAndCppSources(), Pair.firstFunction())); + builder.addSymlinksToArtifacts(cppCompilationContext.getDeclaredIncludeSrcs()); + } + return builder.build(); + } + + @Override + public ConfiguredTarget create(RuleContext context) { + return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ false); + } + + public static ConfiguredTarget init(CppSemantics semantics, RuleContext ruleContext, boolean fake, + boolean useTestOnlyFlags) { + ruleContext.checkSrcsSamePackage(true); + CcCommon common = new CcCommon(ruleContext); + CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + + LinkTargetType linkType = + isLinkShared(ruleContext) ? LinkTargetType.DYNAMIC_LIBRARY : LinkTargetType.EXECUTABLE; + + CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics) + .setLinkType(linkType) + .setHeadersCheckingMode(common.determineHeadersCheckingMode()) + .addCopts(common.getCopts()) + .setNoCopts(common.getNoCopts()) + .addLinkopts(common.getLinkopts()) + .addDefines(common.getDefines()) + .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs()) + .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs()) + .addSources(common.getCAndCppSources()) + .addPrivateHeaders(FileType.filter( + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(), + CppFileTypes.CPP_HEADER)) + .addObjectFiles(common.getObjectFilesFromSrcs(false)) + .addPicObjectFiles(common.getObjectFilesFromSrcs(true)) + .addPicIndependentObjectFiles(common.getLinkerScripts()) + .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET)) + .addDeps(ImmutableList.of(CppHelper.mallocForTarget(ruleContext))) + .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK)) + .addSystemIncludeDirs(common.getSystemIncludeDirs()) + .addIncludeDirs(common.getIncludeDirs()) + .addLooseIncludeDirs(common.getLooseIncludeDirs()) + .setFake(fake); + + CcLibraryHelper.Info info = helper.build(); + CppCompilationContext cppCompilationContext = info.getCppCompilationContext(); + CcCompilationOutputs ccCompilationOutputs = info.getCcCompilationOutputs(); + + // if cc_binary includes "linkshared=1", then gcc will be invoked with + // linkopt "-shared", which causes the result of linking to be a shared + // library. In this case, the name of the executable target should end + // in ".so". + PathFragment executableName = Util.getWorkspaceRelativePath( + ruleContext.getTarget(), "", OsUtils.executableExtension()); + CppLinkAction.Builder linkActionBuilder = determineLinkerArguments( + ruleContext, common, cppConfiguration, ccCompilationOutputs, + cppCompilationContext.getCompilationPrerequisites(), fake, executableName); + linkActionBuilder.setUseTestOnlyFlags(useTestOnlyFlags); + linkActionBuilder.addNonLibraryInputs(ccCompilationOutputs.getHeaderTokenFiles()); + + CcToolchainProvider ccToolchain = CppHelper.getToolchain(ruleContext); + LinkStaticness linkStaticness = getLinkStaticness(ruleContext, common, cppConfiguration); + if (linkStaticness == LinkStaticness.DYNAMIC) { + linkActionBuilder.setRuntimeInputs( + ccToolchain.getDynamicRuntimeLinkMiddleman(), + ccToolchain.getDynamicRuntimeLinkInputs()); + } else { + linkActionBuilder.setRuntimeInputs( + ccToolchain.getStaticRuntimeLinkMiddleman(), + ccToolchain.getStaticRuntimeLinkInputs()); + // Only force a static link of libgcc if static runtime linking is enabled (which + // can't be true if runtimeInputs is empty). + // TODO(bazel-team): Move this to CcToolchain. + if (!ccToolchain.getStaticRuntimeLinkInputs().isEmpty()) { + linkActionBuilder.addLinkopt("-static-libgcc"); + } + } + + linkActionBuilder.setLinkType(linkType); + linkActionBuilder.setLinkStaticness(linkStaticness); + linkActionBuilder.setFake(fake); + + // store immutable context now, recreate builder later + CppLinkAction.Context linkContext = new CppLinkAction.Context(linkActionBuilder); + + CppLinkAction linkAction = linkActionBuilder.build(); + ruleContext.registerAction(linkAction); + LibraryToLink outputLibrary = linkAction.getOutputLibrary(); + Iterable fakeLinkerInputs = + fake ? linkAction.getInputs() : ImmutableList.of(); + Artifact executable = outputLibrary.getArtifact(); + CcLinkingOutputs.Builder linkingOutputsBuilder = new CcLinkingOutputs.Builder(); + if (isLinkShared(ruleContext)) { + if (CppFileTypes.SHARED_LIBRARY.matches(executableName)) { + linkingOutputsBuilder.addDynamicLibrary(outputLibrary); + linkingOutputsBuilder.addExecutionDynamicLibrary(outputLibrary); + } else { + ruleContext.attributeError("linkshared", "'linkshared' used in non-shared library"); + } + } + // Also add all shared libraries from srcs. + for (Artifact library : common.getSharedLibrariesFromSrcs()) { + LibraryToLink symlink = common.getDynamicLibrarySymlink(library, true); + linkingOutputsBuilder.addDynamicLibrary(symlink); + linkingOutputsBuilder.addExecutionDynamicLibrary(symlink); + } + CcLinkingOutputs linkingOutputs = linkingOutputsBuilder.build(); + NestedSet filesToBuild = NestedSetBuilder.create(Order.STABLE_ORDER, executable); + + // Create the stripped binary, but don't add it to filesToBuild; it's only built when requested. + Artifact strippedFile = ruleContext.getImplicitOutputArtifact( + CppRuleClasses.CC_BINARY_STRIPPED); + createStripAction(ruleContext, cppConfiguration, executable, strippedFile); + + DwoArtifactsCollector dwoArtifacts = + collectTransitiveDwoArtifacts(ruleContext, common, cppConfiguration, ccCompilationOutputs); + Artifact dwpFile = + ruleContext.getImplicitOutputArtifact(CppRuleClasses.CC_BINARY_DEBUG_PACKAGE); + createDebugPackagerActions(ruleContext, cppConfiguration, dwpFile, dwoArtifacts); + + // The debug package should include the dwp file only if it was explicitly requested. + Artifact explicitDwpFile = dwpFile; + if (!cppConfiguration.useFission()) { + explicitDwpFile = null; + } + + // TODO(bazel-team): Do we need to put original shared libraries (along with + // mangled symlinks) into the RunfilesSupport object? It does not seem + // logical since all symlinked libraries will be linked anyway and would + // not require manual loading but if we do, then we would need to collect + // their names and use a different constructor below. + Runfiles runfiles = collectRunfiles(ruleContext, common, linkingOutputs, + cppCompilationContext, linkStaticness, filesToBuild, fakeLinkerInputs, fake); + RunfilesSupport runfilesSupport = RunfilesSupport.withExecutable( + ruleContext, runfiles, executable, ruleContext.getConfiguration().buildRunfiles()); + + TransitiveLipoInfoProvider transitiveLipoInfo; + if (cppConfiguration.isLipoContextCollector()) { + transitiveLipoInfo = common.collectTransitiveLipoLabels(ccCompilationOutputs); + } else { + transitiveLipoInfo = TransitiveLipoInfoProvider.EMPTY; + } + + RuleConfiguredTargetBuilder ruleBuilder = new RuleConfiguredTargetBuilder(ruleContext); + common.addTransitiveInfoProviders( + ruleBuilder, filesToBuild, ccCompilationOutputs, cppCompilationContext, linkingOutputs, + dwoArtifacts, transitiveLipoInfo); + + Map scannableMap = new LinkedHashMap<>(); + if (cppConfiguration.isLipoContextCollector()) { + for (IncludeScannable scannable : transitiveLipoInfo.getTransitiveIncludeScannables()) { + // These should all be CppCompileActions, which should have only one source file. + // This is also checked when they are put into the nested set. + Artifact source = + Iterables.getOnlyElement(scannable.getIncludeScannerSources()); + scannableMap.put(source, scannable); + } + } + + return ruleBuilder + .add(RunfilesProvider.class, RunfilesProvider.simple(runfiles)) + .add( + CppDebugPackageProvider.class, + new CppDebugPackageProvider(strippedFile, executable, explicitDwpFile)) + .setRunfilesSupport(runfilesSupport, executable) + .setBaselineCoverageArtifacts(createBaselineCoverageArtifacts( + ruleContext, common, ccCompilationOutputs, fake)) + .addProvider(LipoContextProvider.class, new LipoContextProvider( + cppCompilationContext, ImmutableMap.copyOf(scannableMap))) + .addProvider(CppLinkAction.Context.class, linkContext) + .build(); + } + + /** + * Creates an action to strip an executable. + */ + private static void createStripAction(RuleContext context, + CppConfiguration cppConfiguration, Artifact input, Artifact output) { + context.registerAction(new SpawnAction.Builder() + .addInput(input) + .addTransitiveInputs(CppHelper.getToolchain(context).getStrip()) + .addOutput(output) + .useDefaultShellEnvironment() + .setExecutable(cppConfiguration.getStripExecutable()) + .addArguments("-S", "-p", "-o", output.getExecPathString()) + .addArguments("-R", ".gnu.switches.text.quote_paths") + .addArguments("-R", ".gnu.switches.text.bracket_paths") + .addArguments("-R", ".gnu.switches.text.system_paths") + .addArguments("-R", ".gnu.switches.text.cpp_defines") + .addArguments("-R", ".gnu.switches.text.cpp_includes") + .addArguments("-R", ".gnu.switches.text.cl_args") + .addArguments("-R", ".gnu.switches.text.lipo_info") + .addArguments("-R", ".gnu.switches.text.annotation") + .addArguments(cppConfiguration.getStripOpts()) + .addArgument(input.getExecPathString()) + .setProgressMessage("Stripping " + output.prettyPrint() + " for " + context.getLabel()) + .setMnemonic("CcStrip") + .build(context)); + } + + /** + * Given 'temps', traverse this target and its dependencies and collect up all + * the object files, libraries, linker options, linkstamps attributes and linker scripts. + */ + private static CppLinkAction.Builder determineLinkerArguments(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs, + ImmutableSet compilationPrerequisites, + boolean fake, PathFragment executableName) { + CppLinkAction.Builder builder = new CppLinkAction.Builder(context, executableName) + .setCrosstoolInputs(CppHelper.getToolchain(context).getLink()) + .addNonLibraryInputs(compilationPrerequisites); + + // Determine the object files to link in. + boolean usePic = CppHelper.usePic(context, !isLinkShared(context)) && !fake; + Iterable compiledObjectFiles = compilationOutputs.getObjectFiles(usePic); + + if (fake) { + builder.addFakeNonLibraryInputs(compiledObjectFiles); + } else { + builder.addNonLibraryInputs(compiledObjectFiles); + } + + builder.addNonLibraryInputs(common.getObjectFilesFromSrcs(usePic)); + builder.addNonLibraryInputs(common.getLinkerScripts()); + + // Determine the libraries to link in. + // First libraries from srcs. Shared library artifacts here are substituted with mangled symlink + // artifacts generated by getDynamicLibraryLink(). This is done to minimize number of -rpath + // entries during linking process. + for (Artifact library : common.getLibrariesFromSrcs()) { + if (SHARED_LIBRARY_FILETYPES.matches(library.getFilename())) { + builder.addLibrary(common.getDynamicLibrarySymlink(library, true)); + } else { + builder.addLibrary(LinkerInputs.opaqueLibraryToLink(library)); + } + } + + // Then libraries from the closure of deps. + List linkopts = new ArrayList<>(); + Map> linkstamps = new LinkedHashMap<>(); + + NestedSet librariesInDepsClosure = + findLibrariesToLinkInDepsClosure(context, common, cppConfiguration, linkopts, linkstamps); + builder.addLinkopts(linkopts); + builder.addLinkstamps(linkstamps); + + builder.addLibraries(librariesInDepsClosure); + return builder; + } + + /** + * Explore the transitive closure of our deps to collect linking information. + */ + private static NestedSet findLibrariesToLinkInDepsClosure( + RuleContext context, + CcCommon common, + CppConfiguration cppConfiguration, + List linkopts, + Map> linkstamps) { + // This is true for both FULLY STATIC and MOSTLY STATIC linking. + boolean linkingStatically = + getLinkStaticness(context, common, cppConfiguration) != LinkStaticness.DYNAMIC; + + CcLinkParams linkParams = collectCcLinkParams( + context, common, linkingStatically, isLinkShared(context)); + linkopts.addAll(linkParams.flattenedLinkopts()); + linkstamps.putAll(CppHelper.resolveLinkstamps(context, linkParams)); + return linkParams.getLibraries(); + } + + /** + * Gets the linkopts to use for this binary. These options are NOT used when + * linking other binaries that depend on this binary. + * + * @return a new List instance that contains the linkopts for this binary + * target. + */ + private static ImmutableList getBinaryLinkopts(RuleContext context, + CcCommon common) { + List linkopts = new ArrayList<>(); + if (isLinkShared(context)) { + linkopts.add("-shared"); + } + linkopts.addAll(common.getLinkopts()); + return ImmutableList.copyOf(linkopts); + } + + private static boolean linkstaticAttribute(RuleContext context) { + return context.attributes().get("linkstatic", Type.BOOLEAN); + } + + /** + * Returns "true" if the {@code linkshared} attribute exists and is set. + */ + private static final boolean isLinkShared(RuleContext context) { + return context.getRule().getRuleClassObject().hasAttr("linkshared", Type.BOOLEAN) + && context.attributes().get("linkshared", Type.BOOLEAN); + } + + private static final boolean dashStaticInLinkopts(CcCommon common, + CppConfiguration cppConfiguration) { + return common.getLinkopts().contains("-static") + || cppConfiguration.getLinkOptions().contains("-static"); + } + + private static final LinkStaticness getLinkStaticness(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration) { + if (cppConfiguration.getDynamicMode() == DynamicMode.FULLY) { + return LinkStaticness.DYNAMIC; + } else if (dashStaticInLinkopts(common, cppConfiguration)) { + return LinkStaticness.FULLY_STATIC; + } else if (cppConfiguration.getDynamicMode() == DynamicMode.OFF + || linkstaticAttribute(context)) { + return LinkStaticness.MOSTLY_STATIC; + } else { + return LinkStaticness.DYNAMIC; + } + } + + /** + * Collects .dwo artifacts either transitively or directly, depending on the link type. + * + *

For a cc_binary, we only include the .dwo files corresponding to the .o files that are + * passed into the link. For static linking, this includes all transitive dependencies. But + * for dynamic linking, dependencies are separately linked into their own shared libraries, + * so we don't need them here. + */ + private static DwoArtifactsCollector collectTransitiveDwoArtifacts(RuleContext context, + CcCommon common, CppConfiguration cppConfiguration, CcCompilationOutputs compilationOutputs) { + if (getLinkStaticness(context, common, cppConfiguration) == LinkStaticness.DYNAMIC) { + return DwoArtifactsCollector.directCollector(compilationOutputs); + } else { + return CcCommon.collectTransitiveDwoArtifacts(context, compilationOutputs); + } + } + + @VisibleForTesting + public static Iterable getDwpInputs( + RuleContext context, NestedSet picDwoArtifacts, NestedSet dwoArtifacts) { + return CppHelper.usePic(context, !isLinkShared(context)) ? picDwoArtifacts : dwoArtifacts; + } + + /** + * Creates the actions needed to generate this target's "debug info package" + * (i.e. its .dwp file). + */ + private static void createDebugPackagerActions(RuleContext context, + CppConfiguration cppConfiguration, Artifact dwpOutput, + DwoArtifactsCollector dwoArtifactsCollector) { + Iterable allInputs = getDwpInputs(context, + dwoArtifactsCollector.getPicDwoArtifacts(), + dwoArtifactsCollector.getDwoArtifacts()); + + // No inputs? Just generate a trivially empty .dwp. + // + // Note this condition automatically triggers for any build where fission is disabled. + // Because rules referencing .dwp targets may be invoked with or without fission, we need + // to support .dwp generation even when fission is disabled. Since no actual functionality + // is expected then, an empty file is appropriate. + if (Iterables.isEmpty(allInputs)) { + context.registerAction( + new FileWriteAction(context.getActionOwner(), dwpOutput, "", false)); + return; + } + + // Get the tool inputs necessary to run the dwp command. + NestedSet dwpTools = CppHelper.getToolchain(context).getDwp(); + Preconditions.checkState(!dwpTools.isEmpty()); + + // We apply a hierarchical action structure to limit the maximum number of inputs to any + // single action. + // + // While the dwp tools consumes .dwo files, it can also consume intermediate .dwp files, + // allowing us to split a large input set into smaller batches of arbitrary size and order. + // Aside from the parallelism performance benefits this offers, this also reduces input + // size requirements: if a.dwo, b.dwo, c.dwo, and e.dwo are each 1 KB files, we can apply + // two intermediate actions DWP(a.dwo, b.dwo) --> i1.dwp and DWP(c.dwo, e.dwo) --> i2.dwp. + // When we then apply the final action DWP(i1.dwp, i2.dwp) --> finalOutput.dwp, the inputs + // to this action will usually total far less than 4 KB. + // + // This list tracks every action we'll need to generate the output .dwp with batching. + List packagers = new ArrayList<>(); + + // Step 1: generate our batches. We currently break into arbitrary batches of fixed maximum + // input counts, but we can always apply more intelligent heuristics if the need arises. + SpawnAction.Builder currentPackager = newDwpAction(cppConfiguration, dwpTools); + int inputsForCurrentPackager = 0; + + for (Artifact dwoInput : allInputs) { + if (inputsForCurrentPackager == MAX_INPUTS_PER_DWP_ACTION) { + packagers.add(currentPackager); + currentPackager = newDwpAction(cppConfiguration, dwpTools); + inputsForCurrentPackager = 0; + } + currentPackager.addInputArgument(dwoInput); + inputsForCurrentPackager++; + } + packagers.add(currentPackager); + + // Step 2: given the batches, create the actions. + if (packagers.size() == 1) { + // If we only have one batch, make a single "original inputs --> final output" action. + context.registerAction(Iterables.getOnlyElement(packagers) + .addArgument("-o") + .addOutputArgument(dwpOutput) + .setMnemonic("CcGenerateDwp") + .build(context)); + } else { + // If we have multiple batches, make them all intermediate actions, then pipe their outputs + // into an additional action that outputs the final artifact. + // + // Note this only creates a hierarchy one level deep (i.e. we don't check if the number of + // intermediate outputs exceeds the maximum batch size). This is okay for current needs, + // which shouldn't stress those limits. + List intermediateOutputs = new ArrayList<>(); + + int count = 1; + for (SpawnAction.Builder packager : packagers) { + Artifact intermediateOutput = + getIntermediateDwpFile(context.getAnalysisEnvironment(), dwpOutput, count++); + context.registerAction(packager + .addArgument("-o") + .addOutputArgument(intermediateOutput) + .setMnemonic("CcGenerateIntermediateDwp") + .build(context)); + intermediateOutputs.add(intermediateOutput); + } + + // Now create the final action. + context.registerAction(newDwpAction(cppConfiguration, dwpTools) + .addInputArguments(intermediateOutputs) + .addArgument("-o") + .addOutputArgument(dwpOutput) + .setMnemonic("CcGenerateDwp") + .build(context)); + } + } + + /** + * Returns a new SpawnAction builder for generating dwp files, pre-initialized with + * standard settings. + */ + private static SpawnAction.Builder newDwpAction(CppConfiguration cppConfiguration, + NestedSet dwpTools) { + return new SpawnAction.Builder() + .addTransitiveInputs(dwpTools) + .setExecutable(cppConfiguration.getDwpExecutable()) + .useParameterFile(ParameterFile.ParameterFileType.UNQUOTED); + } + + /** + * Creates an intermediate dwp file keyed off the name and path of the final output. + */ + private static Artifact getIntermediateDwpFile(AnalysisEnvironment env, Artifact dwpOutput, + int orderNumber) { + PathFragment outputPath = dwpOutput.getRootRelativePath(); + PathFragment intermediatePath = + FileSystemUtils.appendWithoutExtension(outputPath, "-" + orderNumber); + return env.getDerivedArtifact( + outputPath.getParentDirectory().getRelative( + INTERMEDIATE_DWP_DIR + "/" + intermediatePath.getPathString()), + dwpOutput.getRoot()); + } + + /** + * Collect link parameters from the transitive closure. + */ + private static CcLinkParams collectCcLinkParams(RuleContext context, CcCommon common, + boolean linkingStatically, boolean linkShared) { + CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared); + + if (isLinkShared(context)) { + // CcLinkingOutputs is empty because this target is not configured yet + builder.addCcLibrary(context, common, false, CcLinkingOutputs.EMPTY); + } else { + builder.addTransitiveTargets( + context.getPrerequisites("deps", Mode.TARGET), + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + builder.addTransitiveTarget(CppHelper.mallocForTarget(context)); + builder.addLinkOpts(getBinaryLinkopts(context, common)); + } + return builder.build(); + } + + private static ImmutableList createBaselineCoverageArtifacts( + RuleContext context, CcCommon common, CcCompilationOutputs compilationOutputs, + boolean fake) { + if (!TargetUtils.isTestRule(context.getRule()) && !fake) { + Iterable objectFiles = compilationOutputs.getObjectFiles( + CppHelper.usePic(context, !isLinkShared(context))); + return BaselineCoverageAction.getBaselineCoverageArtifacts(context, + common.getInstrumentedFilesProvider(objectFiles).getInstrumentedFiles()); + } else { + return ImmutableList.of(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java new file mode 100644 index 0000000000..3f5ff76a0d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java @@ -0,0 +1,678 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationPrerequisitesProvider; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.FilesToCompileProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TempsProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.DynamicMode; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesCollector.LocalMetadataCollector; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProviderImpl; +import com.google.devtools.build.lib.shell.ShellUtils; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Common parts of the implementation of cc rules. + */ +public final class CcCommon { + + private static final String NO_COPTS_ATTRIBUTE = "nocopts"; + + private static final FileTypeSet SOURCE_TYPES = FileTypeSet.of( + CppFileTypes.CPP_SOURCE, + CppFileTypes.CPP_HEADER, + CppFileTypes.C_SOURCE, + CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR); + + /** + * Collects all metadata files generated by C++ compilation actions that output the .o files + * on the input. + */ + private static final LocalMetadataCollector CC_METADATA_COLLECTOR = + new LocalMetadataCollector() { + @Override + public void collectMetadataArtifacts(Iterable objectFiles, + AnalysisEnvironment analysisEnvironment, NestedSetBuilder metadataFilesBuilder) { + for (Artifact artifact : objectFiles) { + Action action = analysisEnvironment.getLocalGeneratingAction(artifact); + if (action instanceof CppCompileAction) { + addOutputs(metadataFilesBuilder, action, CppFileTypes.COVERAGE_NOTES); + } + } + } + }; + + /** C++ configuration */ + private final CppConfiguration cppConfiguration; + + /** The Artifacts from srcs. */ + private final ImmutableList sources; + + private final ImmutableList> cAndCppSources; + + /** Expanded and tokenized copts attribute. Set by initCopts(). */ + private final ImmutableList copts; + + /** + * The expanded linkopts for this rule. + */ + private final ImmutableList linkopts; + + private final RuleContext ruleContext; + + public CcCommon(RuleContext ruleContext) { + this.ruleContext = ruleContext; + this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + this.sources = hasAttribute("srcs", Type.LABEL_LIST) + ? ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list() + : ImmutableList.of(); + + this.cAndCppSources = collectCAndCppSources(); + copts = initCopts(); + linkopts = initLinkopts(); + } + + ImmutableList getTemps(CcCompilationOutputs compilationOutputs) { + return cppConfiguration.isLipoContextCollector() + ? ImmutableList.of() + : compilationOutputs.getTemps(); + } + + /** + * Returns our own linkopts from the rule attribute. This determines linker + * options to use when building this target and anything that depends on it. + */ + public ImmutableList getLinkopts() { + return linkopts; + } + + public ImmutableList getCopts() { + return copts; + } + + private boolean hasAttribute(String name, Type type) { + return ruleContext.getRule().getRuleClassObject().hasAttr(name, type); + } + + private static NestedSet collectExecutionDynamicLibraryArtifacts( + RuleContext ruleContext, + List executionDynamicLibraries) { + Iterable artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries); + if (!Iterables.isEmpty(artifacts)) { + return NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts); + } + + Iterable deps = ruleContext + .getPrerequisites("deps", Mode.TARGET, CcExecutionDynamicLibrariesProvider.class); + + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (CcExecutionDynamicLibrariesProvider dep : deps) { + builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts()); + } + return builder.build(); + } + + /** + * Collects all .dwo artifacts in this target's transitive closure. + */ + public static DwoArtifactsCollector collectTransitiveDwoArtifacts( + RuleContext ruleContext, + CcCompilationOutputs compilationOutputs) { + ImmutableList.Builder deps = + ImmutableList.builder(); + + deps.addAll(ruleContext.getPrerequisites("deps", Mode.TARGET)); + + if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) { + deps.add(CppHelper.mallocForTarget(ruleContext)); + } + if (ruleContext.getRule().getRuleClassObject().hasAttr("implementation", Type.LABEL_LIST)) { + deps.addAll(ruleContext.getPrerequisites("implementation", Mode.TARGET)); + } + + return compilationOutputs == null // Possible in LIPO collection mode (see initializationHook). + ? DwoArtifactsCollector.emptyCollector() + : DwoArtifactsCollector.transitiveCollector(compilationOutputs, deps.build()); + } + + public TransitiveLipoInfoProvider collectTransitiveLipoLabels(CcCompilationOutputs outputs) { + if (cppConfiguration.getFdoSupport().getFdoRoot() == null + || !cppConfiguration.isLipoContextCollector()) { + return TransitiveLipoInfoProvider.EMPTY; + } + + NestedSetBuilder scannableBuilder = NestedSetBuilder.stableOrder(); + CppHelper.addTransitiveLipoInfoForCommonAttributes(ruleContext, outputs, scannableBuilder); + if (hasAttribute("implementation", Type.LABEL_LIST)) { + for (TransitiveLipoInfoProvider impl : AnalysisUtils.getProviders( + ruleContext.getPrerequisites("implementation", Mode.TARGET), + TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(impl.getTransitiveIncludeScannables()); + } + } + + return new TransitiveLipoInfoProvider(scannableBuilder.build()); + } + + private NestedSet collectTransitiveCcNativeLibraries( + RuleContext ruleContext, + List dynamicLibraries) { + NestedSetBuilder builder = NestedSetBuilder.linkOrder(); + builder.addAll(dynamicLibraries); + for (CcNativeLibraryProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, CcNativeLibraryProvider.class)) { + builder.addTransitive(dep.getTransitiveCcNativeLibraries()); + } + return builder.build(); + } + + /** + * Returns a list of ({@link Artifact}, {@link Label}) pairs. Each pair represents an input + * source file and the label of the rule that generates it (or the label of the source file + * itself if it is an input file) + */ + ImmutableList> getCAndCppSources() { + return cAndCppSources; + } + + private boolean shouldProcessHeaders() { + boolean crosstoolSupportsHeaderParsing = + CppHelper.getToolchain(ruleContext).supportsHeaderParsing(); + return crosstoolSupportsHeaderParsing && ( + ruleContext.getFeatures().contains(CppRuleClasses.PREPROCESS_HEADERS) + || ruleContext.getFeatures().contains(CppRuleClasses.PARSE_HEADERS)); + } + + private ImmutableList> collectCAndCppSources() { + Map map = Maps.newLinkedHashMap(); + if (!hasAttribute("srcs", Type.LABEL_LIST)) { + return ImmutableList.>of(); + } + Iterable providers = + ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class); + // TODO(bazel-team): Move header processing logic down in the stack (to CcLibraryHelper or + // such). + boolean processHeaders = shouldProcessHeaders(); + if (processHeaders && hasAttribute("hdrs", Type.LABEL_LIST)) { + providers = Iterables.concat(providers, + ruleContext.getPrerequisites("hdrs", Mode.TARGET, FileProvider.class)); + } + for (FileProvider provider : providers) { + for (Artifact artifact : FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)) { + boolean isHeader = CppFileTypes.CPP_HEADER.matches(artifact.getExecPath()); + if ((isHeader && !processHeaders) + || CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(artifact.getExecPath())) { + continue; + } + Label oldLabel = map.put(artifact, provider.getLabel()); + // TODO(bazel-team): We currently do not warn for duplicate headers with + // different labels, as that would require cleaning up the code base + // without significant benefit; we should eventually make this + // consistent one way or the other. + if (!isHeader && oldLabel != null && !oldLabel.equals(provider.getLabel())) { + ruleContext.attributeError("srcs", String.format( + "Artifact '%s' is duplicated (through '%s' and '%s')", + artifact.getExecPathString(), oldLabel, provider.getLabel())); + } + } + } + + ImmutableList.Builder> result = ImmutableList.builder(); + for (Map.Entry entry : map.entrySet()) { + result.add(Pair.of(entry.getKey(), entry.getValue())); + } + + return result.build(); + } + + Iterable getLibrariesFromSrcs() { + return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + } + + Iterable getSharedLibrariesFromSrcs() { + return getSharedLibrariesFrom(sources); + } + + static Iterable getSharedLibrariesFrom(Iterable collection) { + return FileType.filter(collection, CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + } + + Iterable getStaticLibrariesFromSrcs() { + return FileType.filter(sources, CppFileTypes.ARCHIVE, CppFileTypes.ALWAYS_LINK_LIBRARY); + } + + Iterable getPicStaticLibrariesFromSrcs() { + return LinkerInputs.opaqueLibrariesToLink( + FileType.filter(sources, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY)); + } + + Iterable getObjectFilesFromSrcs(final boolean usePic) { + if (usePic) { + return Iterables.filter(sources, new Predicate() { + @Override + public boolean apply(Artifact artifact) { + String filename = artifact.getExecPathString(); + + // For compatibility with existing BUILD files, any ".o" files listed + // in srcs are assumed to be position-independent code, or + // at least suitable for inclusion in shared libraries, unless they + // end with ".nopic.o". (The ".nopic.o" extension is an undocumented + // feature to give users at least some control over this.) Note that + // some target platforms do not require shared library code to be PIC. + return CppFileTypes.PIC_OBJECT_FILE.matches(filename) || + (CppFileTypes.OBJECT_FILE.matches(filename) && !filename.endsWith(".nopic.o")); + } + }); + } else { + return FileType.filter(sources, CppFileTypes.OBJECT_FILE); + } + } + + /** + * Returns the files from headers and does some sanity checks. Note that this method reports + * warnings to the {@link RuleContext} as a side effect, and so should only be called once for any + * given rule. + */ + public static List getHeaders(RuleContext ruleContext) { + List hdrs = new ArrayList<>(); + for (TransitiveInfoCollection target : + ruleContext.getPrerequisitesIf("hdrs", Mode.TARGET, FileProvider.class)) { + FileProvider provider = target.getProvider(FileProvider.class); + for (Artifact artifact : provider.getFilesToBuild()) { + if (!CppRuleClasses.DISALLOWED_HDRS_FILES.matches(artifact.getFilename())) { + hdrs.add(artifact); + } else { + ruleContext.attributeWarning("hdrs", "file '" + artifact.getFilename() + + "' from target '" + target.getLabel() + "' is not allowed in hdrs"); + } + } + } + return hdrs; + } + + /** + * Uses {@link #getHeaders(RuleContext)} to get the {@code hdrs} on this target. This method will + * return an empty list if there is no {@code hdrs} attribute on this rule type. + */ + List getHeaders() { + if (!hasAttribute("hdrs", Type.LABEL_LIST)) { + return ImmutableList.of(); + } + return getHeaders(ruleContext); + } + + HeadersCheckingMode determineHeadersCheckingMode() { + HeadersCheckingMode headersCheckingMode = cppConfiguration.getHeadersCheckingMode(); + + // Package default overrides command line option. + if (ruleContext.getRule().getPackage().isDefaultHdrsCheckSet()) { + String value = + ruleContext.getRule().getPackage().getDefaultHdrsCheck().toUpperCase(Locale.ENGLISH); + headersCheckingMode = HeadersCheckingMode.valueOf(value); + } + + // 'hdrs_check' attribute overrides package default. + if (hasAttribute("hdrs_check", Type.STRING) + && ruleContext.getRule().isAttributeValueExplicitlySpecified("hdrs_check")) { + try { + String value = ruleContext.attributes().get("hdrs_check", Type.STRING) + .toUpperCase(Locale.ENGLISH); + headersCheckingMode = HeadersCheckingMode.valueOf(value); + } catch (IllegalArgumentException e) { + ruleContext.attributeError("hdrs_check", "must be one of: 'loose', 'warn' or 'strict'"); + } + } + + return headersCheckingMode; + } + + /** + * Expand and tokenize the copts and nocopts attributes. + */ + private ImmutableList initCopts() { + if (!hasAttribute("copts", Type.STRING_LIST)) { + return ImmutableList.of(); + } + // TODO(bazel-team): getAttributeCopts should not tokenize the strings. + // Make a warning for now. + List tokens = new ArrayList<>(); + for (String str : ruleContext.attributes().get("copts", Type.STRING_LIST)) { + tokens.clear(); + try { + ShellUtils.tokenize(tokens, str); + if (tokens.size() > 1) { + ruleContext.attributeWarning("copts", + "each item in the list should contain only one option"); + } + } catch (ShellUtils.TokenizationException e) { + // ignore, the error is reported in the getAttributeCopts call + } + } + + Pattern nocopts = getNoCopts(ruleContext); + if (nocopts != null && nocopts.matcher("-Wno-future-warnings").matches()) { + ruleContext.attributeWarning("nocopts", + "Regular expression '" + nocopts.pattern() + "' is too general; for example, it matches " + + "'-Wno-future-warnings'. Thus it might *re-enable* compiler warnings we wish to " + + "disable globally. To disable all compiler warnings, add '-w' to copts instead"); + } + + return ImmutableList.builder() + .addAll(getPackageCopts(ruleContext)) + .addAll(CppHelper.getAttributeCopts(ruleContext, "copts")) + .build(); + } + + private static ImmutableList getPackageCopts(RuleContext ruleContext) { + List unexpanded = ruleContext.getRule().getPackage().getDefaultCopts(); + return ImmutableList.copyOf(CppHelper.expandMakeVariables(ruleContext, "copts", unexpanded)); + } + + Pattern getNoCopts() { + return getNoCopts(ruleContext); + } + + /** + * Returns nocopts pattern built from the make variable expanded nocopts + * attribute. + */ + private static Pattern getNoCopts(RuleContext ruleContext) { + Pattern nocopts = null; + if (ruleContext.getRule().isAttrDefined(NO_COPTS_ATTRIBUTE, Type.STRING)) { + String nocoptsAttr = ruleContext.expandMakeVariables(NO_COPTS_ATTRIBUTE, + ruleContext.attributes().get(NO_COPTS_ATTRIBUTE, Type.STRING)); + try { + nocopts = Pattern.compile(nocoptsAttr); + } catch (PatternSyntaxException e) { + ruleContext.attributeError(NO_COPTS_ATTRIBUTE, + "invalid regular expression '" + nocoptsAttr + "': " + e.getMessage()); + } + } + return nocopts; + } + + // TODO(bazel-team): calculating nocopts every time is not very efficient, + // fix this after the rule migration. The problem is that in some cases we call this after + // the RCT is created (so RuleContext is not accessible), in some cases during the creation. + // It would probably make more sense to use TransitiveInfoProviders. + /** + * Returns true if the rule context has a nocopts regex that matches the given value, false + * otherwise. + */ + static boolean noCoptsMatches(String option, RuleContext ruleContext) { + Pattern nocopts = getNoCopts(ruleContext); + return nocopts == null ? false : nocopts.matcher(option).matches(); + } + + private static final String DEFINES_ATTRIBUTE = "defines"; + + /** + * Returns a list of define tokens from "defines" attribute. + * + *

We tokenize the "defines" attribute, to ensure that the handling of + * quotes and backslash escapes is consistent Bazel's treatment of the "copts" attribute. + * + *

But we require that the "defines" attribute consists of a single token. + */ + public List getDefines() { + List defines = new ArrayList<>(); + for (String define : + ruleContext.attributes().get(DEFINES_ATTRIBUTE, Type.STRING_LIST)) { + List tokens = new ArrayList<>(); + try { + ShellUtils.tokenize(tokens, ruleContext.expandMakeVariables(DEFINES_ATTRIBUTE, define)); + if (tokens.size() == 1) { + defines.add(tokens.get(0)); + } else if (tokens.isEmpty()) { + ruleContext.attributeError(DEFINES_ATTRIBUTE, "empty definition not allowed"); + } else { + ruleContext.attributeError(DEFINES_ATTRIBUTE, + "definition contains too many tokens (found " + tokens.size() + + ", expecting exactly one)"); + } + } catch (ShellUtils.TokenizationException e) { + ruleContext.attributeError(DEFINES_ATTRIBUTE, e.getMessage()); + } + } + return defines; + } + + /** + * Collects our own linkopts from the rule attribute. This determines linker + * options to use when building this library and anything that depends on it. + */ + private final ImmutableList initLinkopts() { + if (!hasAttribute("linkopts", Type.STRING_LIST)) { + return ImmutableList.of(); + } + List ourLinkopts = ruleContext.attributes().get("linkopts", Type.STRING_LIST); + List result = new ArrayList<>(); + if (ourLinkopts != null) { + boolean allowDashStatic = !cppConfiguration.forceIgnoreDashStatic() + && (cppConfiguration.getDynamicMode() != DynamicMode.FULLY); + for (String linkopt : ourLinkopts) { + if (linkopt.equals("-static") && !allowDashStatic) { + continue; + } + CppHelper.expandAttribute(ruleContext, result, "linkopts", linkopt, true); + } + } + return ImmutableList.copyOf(result); + } + + /** + * Determines a list of loose include directories that are only allowed to be referenced when + * headers checking is {@link HeadersCheckingMode#LOOSE} or {@link HeadersCheckingMode#WARN}. + */ + List getLooseIncludeDirs() { + List result = new ArrayList<>(); + // The package directory of the rule contributes includes. Note that this also covers all + // non-subpackage sub-directories. + PathFragment rulePackage = ruleContext.getLabel().getPackageFragment(); + result.add(rulePackage); + + // Gather up all the dirs from the rule's srcs as well as any of the srcs outputs. + if (hasAttribute("srcs", Type.LABEL_LIST)) { + for (FileProvider src : + ruleContext.getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + PathFragment packageDir = src.getLabel().getPackageFragment(); + for (Artifact a : src.getFilesToBuild()) { + result.add(packageDir); + // Attempt to gather subdirectories that might contain include files. + result.add(a.getRootRelativePath().getParentDirectory()); + } + } + } + + // Add in any 'includes' attribute values as relative path fragments + if (ruleContext.getRule().isAttributeValueExplicitlySpecified("includes")) { + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + // For now, anything with an 'includes' needs a blanket declaration + result.add(packageFragment.getRelative("**")); + } + return result; + } + + List getSystemIncludeDirs() { + // Add in any 'includes' attribute values as relative path fragments + if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes") + || !cppConfiguration.useIsystemForIncludes()) { + return ImmutableList.of(); + } + return getIncludeDirsFromIncludesAttribute(); + } + + List getIncludeDirs() { + if (!ruleContext.getRule().isAttributeValueExplicitlySpecified("includes") + || cppConfiguration.useIsystemForIncludes()) { + return ImmutableList.of(); + } + return getIncludeDirsFromIncludesAttribute(); + } + + private List getIncludeDirsFromIncludesAttribute() { + List result = new ArrayList<>(); + PathFragment packageFragment = ruleContext.getLabel().getPackageFragment(); + for (String includesAttr : ruleContext.attributes().get("includes", Type.STRING_LIST)) { + includesAttr = ruleContext.expandMakeVariables("includes", includesAttr); + if (includesAttr.startsWith("/")) { + ruleContext.attributeWarning("includes", + "ignoring invalid absolute path '" + includesAttr + "'"); + continue; + } + PathFragment includesPath = packageFragment.getRelative(includesAttr).normalize(); + if (!includesPath.isNormalized()) { + ruleContext.attributeError("includes", + "Path references a path above the execution root."); + } + result.add(includesPath); + result.add(ruleContext.getConfiguration().getGenfilesFragment().getRelative(includesPath)); + } + return result; + } + + /** + * Collects compilation prerequisite artifacts. + */ + static CompilationPrerequisitesProvider collectCompilationPrerequisites( + RuleContext ruleContext, CppCompilationContext context) { + // TODO(bazel-team): Use context.getCompilationPrerequisites() instead. + NestedSetBuilder prerequisites = NestedSetBuilder.stableOrder(); + if (ruleContext.getRule().getRuleClassObject().hasAttr("srcs", Type.LABEL_LIST)) { + for (FileProvider provider : ruleContext + .getPrerequisites("srcs", Mode.TARGET, FileProvider.class)) { + prerequisites.addAll(FileType.filter(provider.getFilesToBuild(), SOURCE_TYPES)); + } + } + prerequisites.addTransitive(context.getDeclaredIncludeSrcs()); + return new CompilationPrerequisitesProvider(prerequisites.build()); + } + + /** + * Replaces shared library artifact with mangled symlink and creates related + * symlink action. For artifacts that should retain filename (e.g. libraries + * with SONAME tag), link is created to the parent directory instead. + * + * This action is performed to minimize number of -rpath entries used during + * linking process (by essentially "collecting" as many shared libraries as + * possible in the single directory), since we will be paying quadratic price + * for each additional entry on the -rpath. + * + * @param library Shared library artifact that needs to be mangled + * @param preserveName true if filename should be preserved, false - mangled. + * @return mangled symlink artifact. + */ + public LibraryToLink getDynamicLibrarySymlink(Artifact library, boolean preserveName) { + return SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, library, preserveName, true, ruleContext.getConfiguration()); + } + + /** + * Returns any linker scripts found in the dependencies of the rule. + */ + Iterable getLinkerScripts() { + return FileType.filter(ruleContext.getPrerequisiteArtifacts("deps", Mode.TARGET).list(), + CppFileTypes.LINKER_SCRIPT); + } + + ImmutableList getFilesToCompile(CcCompilationOutputs compilationOutputs) { + return cppConfiguration.isLipoContextCollector() + ? ImmutableList.of() + : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false)); + } + + InstrumentedFilesProvider getInstrumentedFilesProvider(Iterable files) { + return cppConfiguration.isLipoContextCollector() + ? InstrumentedFilesProviderImpl.EMPTY + : new InstrumentedFilesProviderImpl(new InstrumentedFilesCollector( + ruleContext, CppRuleClasses.INSTRUMENTATION_SPEC, CC_METADATA_COLLECTOR, files)); + } + + public static FeatureConfiguration configureFeatures(RuleContext ruleContext) { + CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext); + Set requestedFeatures = ImmutableSet.of(CppRuleClasses.MODULE_MAP_HOME_CWD); + return toolchain.getFeatures().getFeatureConfiguration(requestedFeatures); + } + + public void addTransitiveInfoProviders(RuleConfiguredTargetBuilder builder, + NestedSet filesToBuild, + CcCompilationOutputs ccCompilationOutputs, + CppCompilationContext cppCompilationContext, + CcLinkingOutputs linkingOutputs, + DwoArtifactsCollector dwoArtifacts, + TransitiveLipoInfoProvider transitiveLipoInfo) { + List instrumentedObjectFiles = new ArrayList<>(); + instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(false)); + instrumentedObjectFiles.addAll(ccCompilationOutputs.getObjectFiles(true)); + builder + .setFilesToBuild(filesToBuild) + .add(CppCompilationContext.class, cppCompilationContext) + .add(TransitiveLipoInfoProvider.class, transitiveLipoInfo) + .add(CcExecutionDynamicLibrariesProvider.class, + new CcExecutionDynamicLibrariesProvider(collectExecutionDynamicLibraryArtifacts( + ruleContext, linkingOutputs.getExecutionDynamicLibraries()))) + .add(CcNativeLibraryProvider.class, new CcNativeLibraryProvider( + collectTransitiveCcNativeLibraries(ruleContext, linkingOutputs.getDynamicLibraries()))) + .add(InstrumentedFilesProvider.class, getInstrumentedFilesProvider( + instrumentedObjectFiles)) + .add(FilesToCompileProvider.class, new FilesToCompileProvider( + getFilesToCompile(ccCompilationOutputs))) + .add(CompilationPrerequisitesProvider.class, + collectCompilationPrerequisites(ruleContext, cppCompilationContext)) + .add(TempsProvider.class, new TempsProvider(getTemps(ccCompilationOutputs))) + .add(CppDebugFileProvider.class, new CppDebugFileProvider( + dwoArtifacts.getDwoArtifacts(), + dwoArtifacts.getPicDwoArtifacts())); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java new file mode 100644 index 0000000000..b9fa4e8f49 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java @@ -0,0 +1,207 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A structured representation of the compilation outputs of a C++ rule. + */ +public class CcCompilationOutputs { + /** + * All .o files built by the target. + */ + private final ImmutableList objectFiles; + + /** + * All .pic.o files built by the target. + */ + private final ImmutableList picObjectFiles; + + /** + * All .dwo files built by the target, corresponding to .o outputs. + */ + private final ImmutableList dwoFiles; + + /** + * All .pic.dwo files built by the target, corresponding to .pic.o outputs. + */ + private final ImmutableList picDwoFiles; + + /** + * All artifacts that are created if "--save_temps" is true. + */ + private final ImmutableList temps; + + /** + * All token .h.processed files created when preprocessing or parsing headers. + */ + private final ImmutableList headerTokenFiles; + + private final List lipoScannables; + + private CcCompilationOutputs(ImmutableList objectFiles, + ImmutableList picObjectFiles, ImmutableList dwoFiles, + ImmutableList picDwoFiles, ImmutableList temps, + ImmutableList headerTokenFiles, + ImmutableList lipoScannables) { + this.objectFiles = objectFiles; + this.picObjectFiles = picObjectFiles; + this.dwoFiles = dwoFiles; + this.picDwoFiles = picDwoFiles; + this.temps = temps; + this.headerTokenFiles = headerTokenFiles; + this.lipoScannables = lipoScannables; + } + + /** + * Returns an unmodifiable view of the .o or .pic.o files set. + * + * @param usePic whether to return .pic.o files + */ + public ImmutableList getObjectFiles(boolean usePic) { + return usePic ? picObjectFiles : objectFiles; + } + + /** + * Returns an unmodifiable view of the .dwo files set. + */ + public ImmutableList getDwoFiles() { + return dwoFiles; + } + + /** + * Returns an unmodifiable view of the .pic.dwo files set. + */ + public ImmutableList getPicDwoFiles() { + return picDwoFiles; + } + + /** + * Returns an unmodifiable view of the temp files set. + */ + public ImmutableList getTemps() { + return temps; + } + + /** + * Returns an unmodifiable view of the .h.processed files. + */ + public Iterable getHeaderTokenFiles() { + return headerTokenFiles; + } + + /** + * Returns the {@link IncludeScannable} objects this C++ compile action contributes to a + * LIPO context collector. + */ + public List getLipoScannables() { + return lipoScannables; + } + + public static final class Builder { + private final Set objectFiles = new LinkedHashSet<>(); + private final Set picObjectFiles = new LinkedHashSet<>(); + private final Set dwoFiles = new LinkedHashSet<>(); + private final Set picDwoFiles = new LinkedHashSet<>(); + private final Set temps = new LinkedHashSet<>(); + private final Set headerTokenFiles = new LinkedHashSet<>(); + private final List lipoScannables = new ArrayList<>(); + + public CcCompilationOutputs build() { + return new CcCompilationOutputs(ImmutableList.copyOf(objectFiles), + ImmutableList.copyOf(picObjectFiles), ImmutableList.copyOf(dwoFiles), + ImmutableList.copyOf(picDwoFiles), ImmutableList.copyOf(temps), + ImmutableList.copyOf(headerTokenFiles), + ImmutableList.copyOf(lipoScannables)); + } + + public Builder merge(CcCompilationOutputs outputs) { + this.objectFiles.addAll(outputs.objectFiles); + this.picObjectFiles.addAll(outputs.picObjectFiles); + this.dwoFiles.addAll(outputs.dwoFiles); + this.picDwoFiles.addAll(outputs.picDwoFiles); + this.temps.addAll(outputs.temps); + this.headerTokenFiles.addAll(outputs.headerTokenFiles); + this.lipoScannables.addAll(outputs.lipoScannables); + return this; + } + + /** + * Adds an .o file. + */ + public Builder addObjectFile(Artifact artifact) { + objectFiles.add(artifact); + return this; + } + + public Builder addObjectFiles(Iterable artifacts) { + Iterables.addAll(objectFiles, artifacts); + return this; + } + + /** + * Adds a .pic.o file. + */ + public Builder addPicObjectFile(Artifact artifact) { + picObjectFiles.add(artifact); + return this; + } + + public Builder addPicObjectFiles(Iterable artifacts) { + Iterables.addAll(picObjectFiles, artifacts); + return this; + } + + public Builder addDwoFile(Artifact artifact) { + dwoFiles.add(artifact); + return this; + } + + public Builder addPicDwoFile(Artifact artifact) { + picDwoFiles.add(artifact); + return this; + } + + /** + * Adds temp files. + */ + public Builder addTemps(Iterable artifacts) { + Iterables.addAll(temps, artifacts); + return this; + } + + public Builder addHeaderTokenFile(Artifact artifact) { + headerTokenFiles.add(artifact); + return this; + } + + /** + * Adds an {@link IncludeScannable} that this compilation output object contributes to a + * LIPO context collector. + */ + public Builder addLipoScannable(IncludeScannable scannable) { + lipoScannables.add(scannable); + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java new file mode 100644 index 0000000000..39ce942f60 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcExecutionDynamicLibrariesProvider.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides the execution-time dynamic libraries of a C++ rule. + */ +@Immutable +public final class CcExecutionDynamicLibrariesProvider implements TransitiveInfoProvider { + public static final CcExecutionDynamicLibrariesProvider EMPTY = + new CcExecutionDynamicLibrariesProvider( + NestedSetBuilder.emptySet(Order.STABLE_ORDER)); + + private final NestedSet ccExecutionDynamicLibraries; + + public CcExecutionDynamicLibrariesProvider(NestedSet ccExecutionDynamicLibraries) { + this.ccExecutionDynamicLibraries = ccExecutionDynamicLibraries; + } + + /** + * Returns the execution-time dynamic libraries. + * + *

This normally returns the dynamic library created by the rule itself. However, if the rule + * does not create any dynamic libraries, then it returns the combined results of calling + * getExecutionDynamicLibraryArtifacts on all the rule's deps. This behaviour is so that this + * method is useful for a cc_library with deps but no srcs. + */ + public NestedSet getExecutionDynamicLibraryArtifacts() { + return ccExecutionDynamicLibraries; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java new file mode 100644 index 0000000000..428eedb6e2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLibrary.java @@ -0,0 +1,395 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AlwaysBuiltArtifactsProvider; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.AttributeMap; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; +import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.FileTypeSet; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.List; + +/** + * A ConfiguredTarget for cc_library rules. + */ +public abstract class CcLibrary implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcLibrary(CppSemantics semantics) { + this.semantics = semantics; + } + + // These file extensions don't generate object files. + private static final FileTypeSet NO_OBJECT_GENERATING_FILETYPES = FileTypeSet.of( + CppFileTypes.CPP_HEADER, CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.SHARED_LIBRARY); + + private static final Predicate PIC_STATIC_FILTER = new Predicate() { + @Override + public boolean apply(LibraryToLink input) { + String name = input.getArtifact().getExecPath().getBaseName(); + return !name.endsWith(".nopic.a") && !name.endsWith(".nopic.lo"); + } + }; + + private static Runfiles collectRunfiles(RuleContext context, + CcLinkingOutputs ccLinkingOutputs, + boolean neverLink, boolean addDynamicRuntimeInputArtifactsToRunfiles, + boolean linkingStatically) { + Runfiles.Builder builder = new Runfiles.Builder(); + + // neverlink= true creates a library that will never be linked into any binary that depends on + // it, but instead be loaded as an extension. So we need the dynamic library for this in the + // runfiles. + builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically && !neverLink)); + builder.add(context, CppRunfilesProvider.runfilesFunction(linkingStatically)); + if (context.getRule().isAttrDefined("implements", Type.LABEL_LIST)) { + builder.addTargets(context.getPrerequisites("implements", Mode.TARGET), + RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(context.getPrerequisites("implements", Mode.TARGET), + CppRunfilesProvider.runfilesFunction(linkingStatically)); + } + if (context.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) { + builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET), + RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(context.getPrerequisites("implementation", Mode.TARGET), + CppRunfilesProvider.runfilesFunction(linkingStatically)); + } + + builder.addDataDeps(context); + + if (addDynamicRuntimeInputArtifactsToRunfiles) { + builder.addTransitiveArtifacts(CppHelper.getToolchain(context).getDynamicRuntimeLinkInputs()); + } + return builder.build(); + } + + @Override + public ConfiguredTarget create(RuleContext context) { + RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(context); + LinkTargetType linkType = getStaticLinkType(context); + boolean linkStatic = context.attributes().get("linkstatic", Type.BOOLEAN); + init(semantics, context, builder, linkType, + /*neverLink =*/ false, + linkStatic, + /*collectLinkstamp =*/ true, + /*addDynamicRuntimeInputArtifactsToRunfiles =*/ false); + return builder.build(); + } + + public static void init(CppSemantics semantics, RuleContext ruleContext, + RuleConfiguredTargetBuilder targetBuilder, LinkTargetType linkType, + boolean neverLink, + boolean linkStatic, + boolean collectLinkstamp, + boolean addDynamicRuntimeInputArtifactsToRunfiles) { + final CcCommon common = new CcCommon(ruleContext); + + CcLibraryHelper helper = new CcLibraryHelper(ruleContext, semantics) + .setLinkType(linkType) + .enableCcNativeLibrariesProvider() + .enableInterfaceSharedObjects() + .enableCompileProviders() + .setNeverLink(neverLink) + .setHeadersCheckingMode(common.determineHeadersCheckingMode()) + .addCopts(common.getCopts()) + .setNoCopts(common.getNoCopts()) + .addLinkopts(common.getLinkopts()) + .addDefines(common.getDefines()) + .addCompilationPrerequisites(common.getSharedLibrariesFromSrcs()) + .addCompilationPrerequisites(common.getStaticLibrariesFromSrcs()) + .addSources(common.getCAndCppSources()) + .addPublicHeaders(common.getHeaders()) + .addObjectFiles(common.getObjectFilesFromSrcs(false)) + .addPicObjectFiles(common.getObjectFilesFromSrcs(true)) + .addPicIndependentObjectFiles(common.getLinkerScripts()) + .addDeps(ruleContext.getPrerequisites("deps", Mode.TARGET)) + .setEnableLayeringCheck(ruleContext.getFeatures().contains(CppRuleClasses.LAYERING_CHECK)) + .setCompileHeaderModules(ruleContext.getFeatures().contains(CppRuleClasses.HEADER_MODULES)) + .addSystemIncludeDirs(common.getSystemIncludeDirs()) + .addIncludeDirs(common.getIncludeDirs()) + .addLooseIncludeDirs(common.getLooseIncludeDirs()) + .setEmitHeaderTargetModuleMaps( + ruleContext.getRule().getRuleClass().equals("cc_public_library")); + + if (collectLinkstamp) { + helper.addLinkstamps(ruleContext.getPrerequisites("linkstamp", Mode.TARGET)); + } + + if (ruleContext.getRule().isAttrDefined("implements", Type.LABEL_LIST)) { + helper.addDeps(ruleContext.getPrerequisites("implements", Mode.TARGET)); + } + + if (ruleContext.getRule().isAttrDefined("implementation", Type.LABEL_LIST)) { + helper.addDeps(ruleContext.getPrerequisites("implementation", Mode.TARGET)); + } + + PathFragment soImplFilename = null; + if (ruleContext.getRule().isAttrDefined("outs", Type.STRING_LIST)) { + List outs = ruleContext.attributes().get("outs", Type.STRING_LIST); + if (outs.size() > 1) { + ruleContext.attributeError("outs", "must be a singleton list"); + } else if (outs.size() == 1) { + soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY); + soImplFilename = soImplFilename.replaceName(outs.get(0)); + if (!soImplFilename.getPathString().endsWith(".so")) { // Sanity check. + ruleContext.attributeError("outs", "file name must end in '.so'"); + } + } + } + + if (ruleContext.getRule().isAttrDefined("srcs", Type.LABEL_LIST)) { + helper.addPrivateHeaders(FileType.filter( + ruleContext.getPrerequisiteArtifacts("srcs", Mode.TARGET).list(), + CppFileTypes.CPP_HEADER)); + ruleContext.checkSrcsSamePackage(true); + } + + if (common.getLinkopts().contains("-static")) { + ruleContext.attributeWarning("linkopts", "Using '-static' here won't work. " + + "Did you mean to use 'linkstatic=1' instead?"); + } + + boolean createDynamicLibrary = + !linkStatic && !appearsToHaveNoObjectFiles(ruleContext.attributes()); + helper.setCreateDynamicLibrary(createDynamicLibrary); + helper.setDynamicLibraryPath(soImplFilename); + + /* + * Add the libraries from srcs, if any. For static/mostly static + * linking we setup the dynamic libraries if there are no static libraries + * to choose from. Path to the libraries will be mangled to avoid using + * absolute path names on the -rpath, but library filenames will be + * preserved (since some libraries might have SONAME tag) - symlink will + * be created to the parent directory instead. + * + * For compatibility with existing BUILD files, any ".a" or ".lo" files listed in + * srcs are assumed to be position-independent code, or at least suitable for + * inclusion in shared libraries, unless they end with ".nopic.a" or ".nopic.lo". + * + * Note that some target platforms do not require shared library code to be PIC. + */ + Iterable staticLibrariesFromSrcs = + LinkerInputs.opaqueLibrariesToLink(common.getStaticLibrariesFromSrcs()); + helper.addStaticLibraries(staticLibrariesFromSrcs); + helper.addPicStaticLibraries(Iterables.filter(staticLibrariesFromSrcs, PIC_STATIC_FILTER)); + helper.addPicStaticLibraries(common.getPicStaticLibrariesFromSrcs()); + helper.addDynamicLibraries(Iterables.transform(common.getSharedLibrariesFromSrcs(), + new Function() { + @Override + public LibraryToLink apply(Artifact library) { + return common.getDynamicLibrarySymlink(library, true); + } + })); + CcLibraryHelper.Info info = helper.build(); + + /* + * We always generate a static library, even if there aren't any source files. + * This keeps things simpler by avoiding special cases when making use of the library. + * For example, this is needed to ensure that building a library with "bazel build" + * will also build all of the library's "deps". + * However, we only generate a dynamic library if there are source files. + */ + // For now, we don't add the precompiled libraries to the files to build. + CcLinkingOutputs linkedLibraries = info.getCcLinkingOutputsExcludingPrecompiledLibraries(); + + NestedSet artifactsToForce = + collectArtifactsToForce(ruleContext, common, info.getCcCompilationOutputs()); + + NestedSetBuilder filesBuilder = NestedSetBuilder.stableOrder(); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkedLibraries.getPicStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkedLibraries.getDynamicLibraries())); + filesBuilder.addAll( + LinkerInputs.toNonSolibArtifacts(linkedLibraries.getExecutionDynamicLibraries())); + + CcLinkingOutputs linkingOutputs = info.getCcLinkingOutputs(); + warnAboutEmptyLibraries( + ruleContext, info.getCcCompilationOutputs(), linkType, linkStatic); + NestedSet filesToBuild = filesBuilder.build(); + + Runfiles staticRunfiles = collectRunfiles(ruleContext, + linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, true); + Runfiles sharedRunfiles = collectRunfiles(ruleContext, + linkingOutputs, neverLink, addDynamicRuntimeInputArtifactsToRunfiles, false); + + List instrumentedObjectFiles = new ArrayList<>(); + instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(false)); + instrumentedObjectFiles.addAll(info.getCcCompilationOutputs().getObjectFiles(true)); + InstrumentedFilesProvider instrumentedFilesProvider = + common.getInstrumentedFilesProvider(instrumentedObjectFiles); + targetBuilder + .setFilesToBuild(filesToBuild) + .addProviders(info.getProviders()) + .add(InstrumentedFilesProvider.class, instrumentedFilesProvider) + .add(RunfilesProvider.class, RunfilesProvider.withData(staticRunfiles, sharedRunfiles)) + // Remove this? + .add(CppRunfilesProvider.class, new CppRunfilesProvider(staticRunfiles, sharedRunfiles)) + .setBaselineCoverageArtifacts(BaselineCoverageAction.getBaselineCoverageArtifacts( + ruleContext, instrumentedFilesProvider.getInstrumentedFiles())) + .add(ImplementedCcPublicLibrariesProvider.class, + new ImplementedCcPublicLibrariesProvider(getImplementedCcPublicLibraries(ruleContext))) + .add(AlwaysBuiltArtifactsProvider.class, + new AlwaysBuiltArtifactsProvider(artifactsToForce)); + } + + private static NestedSet collectArtifactsToForce(RuleContext ruleContext, + CcCommon common, CcCompilationOutputs ccCompilationOutputs) { + // Ensure that we build all the dependencies, otherwise users may get confused. + NestedSetBuilder artifactsToForceBuilder = NestedSetBuilder.stableOrder(); + artifactsToForceBuilder.addTransitive( + NestedSetBuilder.wrap(Order.STABLE_ORDER, common.getFilesToCompile(ccCompilationOutputs))); + for (AlwaysBuiltArtifactsProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, AlwaysBuiltArtifactsProvider.class)) { + artifactsToForceBuilder.addTransitive(dep.getArtifactsToAlwaysBuild()); + } + return artifactsToForceBuilder.build(); + } + + /** + * Returns the type of the generated static library. + */ + private static LinkTargetType getStaticLinkType(RuleContext context) { + return context.attributes().get("alwayslink", Type.BOOLEAN) + ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + : LinkTargetType.STATIC_LIBRARY; + } + + private static void warnAboutEmptyLibraries(RuleContext ruleContext, + CcCompilationOutputs ccCompilationOutputs, LinkTargetType linkType, + boolean linkstaticAttribute) { + if (ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) { + // Do not signal warnings in the lipo context collector configuration. These will be duly + // signaled in the target configuration, and there can be spurious warnings since targets in + // the LIPO context collector configuration do not compile anything. + return; + } + if (ccCompilationOutputs.getObjectFiles(false).isEmpty() + && ccCompilationOutputs.getObjectFiles(true).isEmpty()) { + if (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + || linkType == LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY) { + ruleContext.attributeWarning("alwayslink", + "'alwayslink' has no effect if there are no 'srcs'"); + } + if (!linkstaticAttribute && !appearsToHaveNoObjectFiles(ruleContext.attributes())) { + ruleContext.attributeWarning("linkstatic", + "setting 'linkstatic=1' is recommended if there are no object files"); + } + } else { + if (!linkstaticAttribute && appearsToHaveNoObjectFiles(ruleContext.attributes())) { + Artifact element = ccCompilationOutputs.getObjectFiles(false).isEmpty() + ? ccCompilationOutputs.getObjectFiles(true).get(0) + : ccCompilationOutputs.getObjectFiles(false).get(0); + ruleContext.attributeWarning("srcs", + "this library appears at first glance to have no object files, " + + "but on closer inspection it does have something to link, e.g. " + + element.prettyPrint() + ". " + + "(You may have used some very confusing rule names in srcs? " + + "Or the library consists entirely of a linker script?) " + + "Bazel assumed linkstatic=1, but this may be inappropriate. " + + "You may need to add an explicit '.cc' file to 'srcs'. " + + "Alternatively, add 'linkstatic=1' to suppress this warning"); + } + } + } + + private static ImmutableList

+ * In some cases, this may return "false" even + * though the rule actually has no object files. + * For example, it will return false for a rule such as + * cc_library(name = 'foo', srcs = [':bar']) + * because we can't tell what ':bar' is; it might + * be a genrule that generates a source file, or it might + * be a genrule that generates a header file. + * + *

+ * In other cases, this may return "true" even + * though the rule actually does have object files. + * For example, it will return true for a rule such as + * cc_library(name = 'foo', srcs = ['bar.h']) + * but as in the other example above, we can't tell whether + * 'bar.h' is a file name or a rule name, and 'bar.h' could + * in fact be the name of a genrule that generates a source file. + */ + public static boolean appearsToHaveNoObjectFiles(AttributeMap rule) { + // Temporary hack while configurable attributes is under development. This has no effect + // for any rule that doesn't use configurable attributes. + // TODO(bazel-team): remove this hack for a more principled solution. + try { + rule.get("srcs", Type.LABEL_LIST); + } catch (ClassCastException e) { + // "srcs" is actually a configurable selector. Assume object files are possible somewhere. + return false; + } + + List

Rules that want to use this class are required to have implicit dependencies on the + * toolchain, the STL, the lipo context, and so on. Optionally, they can also have copts, + * and malloc attributes, but note that these require explicit calls to the corresponding setter + * methods. + */ +public final class CcLibraryHelper { + /** Function for extracting module maps from CppCompilationDependencies. */ + public static final Function CPP_DEPS_TO_MODULES = + new Function() { + @Override + @Nullable + public CppModuleMap apply(TransitiveInfoCollection dep) { + CppCompilationContext context = dep.getProvider(CppCompilationContext.class); + return context == null ? null : context.getCppModuleMap(); + } + }; + + /** + * Contains the providers as well as the compilation and linking outputs, and the compilation + * context. + */ + public static final class Info { + private final Map, TransitiveInfoProvider> providers; + private final CcCompilationOutputs compilationOutputs; + private final CcLinkingOutputs linkingOutputs; + private final CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries; + private final CppCompilationContext context; + + private Info(Map, TransitiveInfoProvider> providers, + CcCompilationOutputs compilationOutputs, CcLinkingOutputs linkingOutputs, + CcLinkingOutputs linkingOutputsExcludingPrecompiledLibraries, + CppCompilationContext context) { + this.providers = Collections.unmodifiableMap(providers); + this.compilationOutputs = compilationOutputs; + this.linkingOutputs = linkingOutputs; + this.linkingOutputsExcludingPrecompiledLibraries = + linkingOutputsExcludingPrecompiledLibraries; + this.context = context; + } + + public Map, TransitiveInfoProvider> getProviders() { + return providers; + } + + public CcCompilationOutputs getCcCompilationOutputs() { + return compilationOutputs; + } + + public CcLinkingOutputs getCcLinkingOutputs() { + return linkingOutputs; + } + + /** + * Returns the linking outputs before adding the pre-compiled libraries. Avoid using this - + * pre-compiled and locally compiled libraries should be treated identically. This method only + * exists for backwards compatibility. + */ + public CcLinkingOutputs getCcLinkingOutputsExcludingPrecompiledLibraries() { + return linkingOutputsExcludingPrecompiledLibraries; + } + + public CppCompilationContext getCppCompilationContext() { + return context; + } + + /** + * Adds the static, pic-static, and dynamic (both compile-time and execution-time) libraries to + * the given builder. + */ + public void addLinkingOutputsTo(NestedSetBuilder filesBuilder) { + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toLibraryArtifacts(linkingOutputs.getPicStaticLibraries())); + filesBuilder.addAll(LinkerInputs.toNonSolibArtifacts(linkingOutputs.getDynamicLibraries())); + filesBuilder.addAll( + LinkerInputs.toNonSolibArtifacts(linkingOutputs.getExecutionDynamicLibraries())); + } + } + + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + private final CppSemantics semantics; + + private final List publicHeaders = new ArrayList<>(); + private final List privateHeaders = new ArrayList<>(); + private final List additionalExportedHeaders = new ArrayList<>(); + private final List> sources = new ArrayList<>(); + private final List objectFiles = new ArrayList<>(); + private final List picObjectFiles = new ArrayList<>(); + private final List copts = new ArrayList<>(); + @Nullable private Pattern nocopts; + private final List linkopts = new ArrayList<>(); + private final Set defines = new LinkedHashSet<>(); + private final List deps = new ArrayList<>(); + private final List linkstamps = new ArrayList<>(); + private final List prerequisites = new ArrayList<>(); + private final List looseIncludeDirs = new ArrayList<>(); + private final List systemIncludeDirs = new ArrayList<>(); + private final List includeDirs = new ArrayList<>(); + @Nullable private PathFragment dynamicLibraryPath; + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private HeadersCheckingMode headersCheckingMode = HeadersCheckingMode.LOOSE; + private boolean neverlink; + private boolean fake; + + private final List staticLibraries = new ArrayList<>(); + private final List picStaticLibraries = new ArrayList<>(); + private final List dynamicLibraries = new ArrayList<>(); + + private boolean emitCppModuleMaps = true; + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + private boolean emitCompileActionsIfEmpty = true; + private boolean emitCcNativeLibrariesProvider; + private boolean emitCcSpecificLinkParamsProvider; + private boolean emitInterfaceSharedObjects; + private boolean emitDynamicLibrary = true; + private boolean checkDepsGenerateCpp = true; + private boolean emitCompileProviders; + private boolean emitHeaderTargetModuleMaps = false; + + public CcLibraryHelper(RuleContext ruleContext, CppSemantics semantics) { + this.ruleContext = Preconditions.checkNotNull(ruleContext); + this.configuration = ruleContext.getConfiguration(); + this.semantics = Preconditions.checkNotNull(semantics); + } + + /** + * Add the corresponding files as header files, i.e., these files will not be compiled, but are + * made visible as includes to dependent rules. + */ + public CcLibraryHelper addPublicHeaders(Collection headers) { + this.publicHeaders.addAll(headers); + return this; + } + + /** + * Add the corresponding files as public header files, i.e., these files will not be compiled, but + * are made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addPublicHeaders(Artifact... headers) { + return addPublicHeaders(Arrays.asList(headers)); + } + + /** + * Add the corresponding files as private header files, i.e., these files will not be compiled, + * but are not made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addPrivateHeaders(Iterable privateHeaders) { + Iterables.addAll(this.privateHeaders, privateHeaders); + return this; + } + + /** + * Add the corresponding files as public header files, i.e., these files will not be compiled, but + * are made visible as includes to dependent rules in module maps. + */ + public CcLibraryHelper addAdditionalExportedHeaders( + Iterable additionalExportedHeaders) { + Iterables.addAll(this.additionalExportedHeaders, additionalExportedHeaders); + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + // TODO(bazel-team): This is inconsistent with the documentation on CppModel. + public CcLibraryHelper addSources(Collection sources) { + for (Artifact source : sources) { + this.sources.add(Pair.of(source, ruleContext.getLabel())); + } + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + // TODO(bazel-team): This is inconsistent with the documentation on CppModel. + public CcLibraryHelper addSources(Iterable> sources) { + Iterables.addAll(this.sources, sources); + return this; + } + + /** + * Add the corresponding files as source files. These may also be header files, in which case + * they will not be compiled, but also not made visible as includes to dependent rules. + */ + public CcLibraryHelper addSources(Artifact... sources) { + return addSources(Arrays.asList(sources)); + } + + /** + * Add the corresponding files as linker inputs for non-PIC links. If the corresponding files are + * compiled with PIC, the final link may or may not fail. Note that the final link may not happen + * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively + * depends on the current rule. + */ + public CcLibraryHelper addObjectFiles(Iterable objectFiles) { + Iterables.addAll(this.objectFiles, objectFiles); + return this; + } + + /** + * Add the corresponding files as linker inputs for PIC links. If the corresponding files are not + * compiled with PIC, the final link may or may not fail. Note that the final link may not happen + * here, if {@code --start_end_lib} is enabled, but instead at any binary that transitively + * depends on the current rule. + */ + public CcLibraryHelper addPicObjectFiles(Iterable picObjectFiles) { + Iterables.addAll(this.picObjectFiles, picObjectFiles); + return this; + } + + /** + * Add the corresponding files as linker inputs for both PIC and non-PIC links. + */ + public CcLibraryHelper addPicIndependentObjectFiles(Iterable objectFiles) { + addPicObjectFiles(objectFiles); + return addObjectFiles(objectFiles); + } + + /** + * Add the corresponding files as linker inputs for both PIC and non-PIC links. + */ + public CcLibraryHelper addPicIndependentObjectFiles(Artifact... objectFiles) { + return addPicIndependentObjectFiles(Arrays.asList(objectFiles)); + } + + /** + * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker + * action) - this makes them available for linking to binary rules that depend on this rule. + */ + public CcLibraryHelper addStaticLibraries(Iterable libraries) { + Iterables.addAll(staticLibraries, libraries); + return this; + } + + /** + * Add the corresponding files as static libraries into the linker outputs (i.e., after the linker + * action) - this makes them available for linking to binary rules that depend on this rule. + */ + public CcLibraryHelper addPicStaticLibraries(Iterable libraries) { + Iterables.addAll(picStaticLibraries, libraries); + return this; + } + + /** + * Add the corresponding files as dynamic libraries into the linker outputs (i.e., after the + * linker action) - this makes them available for linking to binary rules that depend on this + * rule. + */ + public CcLibraryHelper addDynamicLibraries(Iterable libraries) { + Iterables.addAll(dynamicLibraries, libraries); + return this; + } + + /** + * Adds the copts to the compile command line. + */ + public CcLibraryHelper addCopts(Iterable copts) { + Iterables.addAll(this.copts, copts); + return this; + } + + /** + * Sets a pattern that is used to filter copts; set to {@code null} for no filtering. + */ + public CcLibraryHelper setNoCopts(@Nullable Pattern nocopts) { + this.nocopts = nocopts; + return this; + } + + /** + * Adds the given options as linker options to the link command. + */ + public CcLibraryHelper addLinkopts(Iterable linkopts) { + Iterables.addAll(this.linkopts, linkopts); + return this; + } + + /** + * Adds the given defines to the compiler command line. + */ + public CcLibraryHelper addDefines(Iterable defines) { + Iterables.addAll(this.defines, defines); + return this; + } + + /** + * Adds the given targets as dependencies - this can include explicit dependencies on other + * rules (like from a "deps" attribute) and also implicit dependencies on runtime libraries. + */ + public CcLibraryHelper addDeps(Iterable deps) { + for (TransitiveInfoCollection dep : deps) { + Preconditions.checkArgument(dep.getConfiguration() == null + || dep.getConfiguration().equals(configuration)); + this.deps.add(dep); + } + return this; + } + + /** + * Adds the given linkstamps. Note that linkstamps are usually not compiled at the library level, + * but only in the dependent binary rules. + */ + public CcLibraryHelper addLinkstamps(Iterable linkstamps) { + for (TransitiveInfoCollection linkstamp : linkstamps) { + Iterables.addAll(this.linkstamps, + linkstamp.getProvider(FileProvider.class).getFilesToBuild()); + } + return this; + } + + /** + * Adds the given prerequisites as prerequisites for the generated compile actions. This ensures + * that the corresponding files exist - otherwise the action fails. Note that these dependencies + * add edges to the action graph, and can therefore increase the length of the critical path, + * i.e., make the build slower. + */ + public CcLibraryHelper addCompilationPrerequisites(Iterable prerequisites) { + Iterables.addAll(this.prerequisites, prerequisites); + return this; + } + + /** + * Adds the given directories to the loose include directories that are only allowed to be + * referenced when headers checking is {@link HeadersCheckingMode#LOOSE} or {@link + * HeadersCheckingMode#WARN}. + */ + public CcLibraryHelper addLooseIncludeDirs(Iterable looseIncludeDirs) { + Iterables.addAll(this.looseIncludeDirs, looseIncludeDirs); + return this; + } + + /** + * Adds the given directories to the system include directories (they are passed with {@code + * "-isystem"} to the compiler); these are also passed to dependent rules. + */ + public CcLibraryHelper addSystemIncludeDirs(Iterable systemIncludeDirs) { + Iterables.addAll(this.systemIncludeDirs, systemIncludeDirs); + return this; + } + + /** + * Adds the given directories to the quote include directories (they are passed with {@code + * "-iquote"} to the compiler); these are also passed to dependent rules. + */ + public CcLibraryHelper addIncludeDirs(Iterable includeDirs) { + Iterables.addAll(this.includeDirs, includeDirs); + return this; + } + + /** + * Overrides the path for the generated dynamic library - this should only be called if the + * dynamic library is an implicit or explicit output of the rule, i.e., if it is accessible by + * name from other rules in the same package. Set to {@code null} to use the default computation. + */ + public CcLibraryHelper setDynamicLibraryPath(@Nullable PathFragment dynamicLibraryPath) { + this.dynamicLibraryPath = dynamicLibraryPath; + return this; + } + + /** + * Marks the output of this rule as alwayslink, i.e., the corresponding symbols will be retained + * by the linker even if they are not otherwise used. This is useful for libraries that register + * themselves somewhere during initialization. + * + *

This only sets the link type (see {@link #setLinkType}), either to a static library or to + * an alwayslink static library (blaze uses a different file extension to signal alwayslink to + * downstream code). + */ + public CcLibraryHelper setAlwayslink(boolean alwayslink) { + linkType = alwayslink + ? LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY + : LinkTargetType.STATIC_LIBRARY; + return this; + } + + /** + * Directly set the link type. This can be used instead of {@link #setAlwayslink}. Setting + * anything other than a static link causes this class to skip the link action creation. + */ + public CcLibraryHelper setLinkType(LinkTargetType linkType) { + this.linkType = Preconditions.checkNotNull(linkType); + return this; + } + + /** + * Marks the resulting code as neverlink, i.e., the code will not be linked into dependent + * libraries or binaries - the header files are still available. + */ + public CcLibraryHelper setNeverLink(boolean neverlink) { + this.neverlink = neverlink; + return this; + } + + /** + * Sets the given headers checking mode. The default is {@link HeadersCheckingMode#LOOSE}. + */ + public CcLibraryHelper setHeadersCheckingMode(HeadersCheckingMode headersCheckingMode) { + this.headersCheckingMode = Preconditions.checkNotNull(headersCheckingMode); + return this; + } + + /** + * Marks the resulting code as fake, i.e., the code will not actually be compiled or linked, but + * instead, the compile command is written to a file and added to the runfiles. This is currently + * used for non-compilation tests. Unfortunately, the design is problematic, so please don't add + * any further uses. + */ + public CcLibraryHelper setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * This adds the {@link CcNativeLibraryProvider} to the providers created by this class. + */ + public CcLibraryHelper enableCcNativeLibrariesProvider() { + this.emitCcNativeLibrariesProvider = true; + return this; + } + + /** + * This adds the {@link CcSpecificLinkParamsProvider} to the providers created by this class. + * Otherwise the result will contain an instance of {@link CcLinkParamsProvider}. + */ + public CcLibraryHelper enableCcSpecificLinkParamsProvider() { + this.emitCcSpecificLinkParamsProvider = true; + return this; + } + + /** + * This disables C++ module map generation for the current rule. Don't call this unless you know + * what you are doing. + */ + public CcLibraryHelper disableCppModuleMapGeneration() { + this.emitCppModuleMaps = false; + return this; + } + + /** + * This enables or disables use of module maps during compilation, i.e., layering checks. + */ + public CcLibraryHelper setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * This enabled or disables compilation of C++ header modules. + * TODO(bazel-team): Add a cc_toolchain flag that allows fully disabling this feature and document + * this feature. + * See http://clang.llvm.org/docs/Modules.html. + */ + public CcLibraryHelper setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + /** + * Enables or disables generation of compile actions if there are no sources. Some rules declare a + * .a or .so implicit output, which requires that these files are created even if there are no + * source files, so be careful when calling this. + */ + public CcLibraryHelper setGenerateCompileActionsIfEmpty(boolean emitCompileActionsIfEmpty) { + this.emitCompileActionsIfEmpty = emitCompileActionsIfEmpty; + return this; + } + + /** + * Enables the optional generation of interface dynamic libraries - this is only used when the + * linker generates a dynamic library, and only if the crosstool supports it. The default is not + * to generate interface dynamic libraries. + */ + public CcLibraryHelper enableInterfaceSharedObjects() { + this.emitInterfaceSharedObjects = true; + return this; + } + + /** + * This enables or disables the generation of a dynamic library link action. The default is to + * generate a dynamic library. Note that the selection between dynamic or static linking is + * performed at the binary rule level. + */ + public CcLibraryHelper setCreateDynamicLibrary(boolean emitDynamicLibrary) { + this.emitDynamicLibrary = emitDynamicLibrary; + return this; + } + + /** + * Disables checking that the deps actually are C++ rules. By default, the {@link #build} method + * uses {@link LanguageDependentFragment.Checker#depSupportsLanguage} to check that all deps + * provide C++ providers. + */ + public CcLibraryHelper setCheckDepsGenerateCpp(boolean checkDepsGenerateCpp) { + this.checkDepsGenerateCpp = checkDepsGenerateCpp; + return this; + } + + /** + * Enables the output of {@link FilesToCompileProvider} and {@link + * CompilationPrerequisitesProvider}. + */ + // TODO(bazel-team): We probably need to adjust this for the multi-language rules. + public CcLibraryHelper enableCompileProviders() { + this.emitCompileProviders = true; + return this; + } + + /** + * Sets whether to emit the transitive module map references of a public library headers target. + */ + public CcLibraryHelper setEmitHeaderTargetModuleMaps(boolean emitHeaderTargetModuleMaps) { + this.emitHeaderTargetModuleMaps = emitHeaderTargetModuleMaps; + return this; + } + + /** + * Create the C++ compile and link actions, and the corresponding C++-related providers. + */ + public Info build() { + // Fail early if there is no lipo context collector on the rule - otherwise we end up failing + // in lipo optimization. + Preconditions.checkState( + // 'cc_inc_library' rules do not compile, and thus are not affected by LIPO. + ruleContext.getRule().getRuleClass().equals("cc_inc_library") + || ruleContext.getRule().isAttrDefined(":lipo_context_collector", Type.LABEL)); + + if (checkDepsGenerateCpp) { + for (LanguageDependentFragment dep : + AnalysisUtils.getProviders(deps, LanguageDependentFragment.class)) { + LanguageDependentFragment.Checker.depSupportsLanguage( + ruleContext, dep, CppRuleClasses.LANGUAGE); + } + } + + CcLinkingOutputs ccLinkingOutputs = CcLinkingOutputs.EMPTY; + CcCompilationOutputs ccOutputs = new CcCompilationOutputs.Builder().build(); + FeatureConfiguration featureConfiguration = CcCommon.configureFeatures(ruleContext); + + CppModel model = new CppModel(ruleContext, semantics) + .addSources(sources) + .addCopts(copts) + .setLinkTargetType(linkType) + .setNeverLink(neverlink) + .setFake(fake) + .setAllowInterfaceSharedObjects(emitInterfaceSharedObjects) + .setCreateDynamicLibrary(emitDynamicLibrary) + // Note: this doesn't actually save the temps, it just makes the CppModel use the + // configurations --save_temps setting to decide whether to actually save the temps. + .setSaveTemps(true) + .setEnableLayeringCheck(enableLayeringCheck) + .setCompileHeaderModules(compileHeaderModules) + .setNoCopts(nocopts) + .setDynamicLibraryPath(dynamicLibraryPath) + .addLinkopts(linkopts) + .setFeatureConfiguration(featureConfiguration); + CppCompilationContext cppCompilationContext = + initializeCppCompilationContext(model, featureConfiguration); + model.setContext(cppCompilationContext); + if (emitCompileActionsIfEmpty || !sources.isEmpty() || compileHeaderModules) { + Preconditions.checkState( + !compileHeaderModules || cppCompilationContext.getCppModuleMap() != null, + "All cc rules must support module maps."); + ccOutputs = model.createCcCompileActions(); + if (!objectFiles.isEmpty() || !picObjectFiles.isEmpty()) { + // Merge the pre-compiled object files into the compiler outputs. + ccOutputs = new CcCompilationOutputs.Builder() + .merge(ccOutputs) + .addObjectFiles(objectFiles) + .addPicObjectFiles(picObjectFiles) + .build(); + } + if (linkType.isStaticLibraryLink()) { + // TODO(bazel-team): This can't create the link action for a cc_binary yet. + ccLinkingOutputs = model.createCcLinkActions(ccOutputs); + } + } + CcLinkingOutputs originalLinkingOutputs = ccLinkingOutputs; + if (!( + staticLibraries.isEmpty() && picStaticLibraries.isEmpty() && dynamicLibraries.isEmpty())) { + // Merge the pre-compiled libraries (static & dynamic) into the linker outputs. + ccLinkingOutputs = new CcLinkingOutputs.Builder() + .merge(ccLinkingOutputs) + .addStaticLibraries(staticLibraries) + .addPicStaticLibraries(picStaticLibraries) + .addDynamicLibraries(dynamicLibraries) + .addExecutionDynamicLibraries(dynamicLibraries) + .build(); + } + + DwoArtifactsCollector dwoArtifacts = DwoArtifactsCollector.transitiveCollector(ccOutputs, deps); + Runfiles cppStaticRunfiles = collectCppRunfiles(ccLinkingOutputs, true); + Runfiles cppSharedRunfiles = collectCppRunfiles(ccLinkingOutputs, false); + + // By very careful when adding new providers here - it can potentially affect a lot of rules. + // We should consider merging most of these providers into a single provider. + Map, TransitiveInfoProvider> providers = + new LinkedHashMap<>(); + providers.put(CppRunfilesProvider.class, + new CppRunfilesProvider(cppStaticRunfiles, cppSharedRunfiles)); + providers.put(CppCompilationContext.class, cppCompilationContext); + providers.put(CppDebugFileProvider.class, new CppDebugFileProvider( + dwoArtifacts.getDwoArtifacts(), dwoArtifacts.getPicDwoArtifacts())); + providers.put(TransitiveLipoInfoProvider.class, collectTransitiveLipoInfo(ccOutputs)); + providers.put(TempsProvider.class, getTemps(ccOutputs)); + if (emitCompileProviders) { + providers.put(FilesToCompileProvider.class, new FilesToCompileProvider( + getFilesToCompile(ccOutputs))); + providers.put(CompilationPrerequisitesProvider.class, + CcCommon.collectCompilationPrerequisites(ruleContext, cppCompilationContext)); + } + + // TODO(bazel-team): Maybe we can infer these from other data at the places where they are + // used. + if (emitCcNativeLibrariesProvider) { + providers.put(CcNativeLibraryProvider.class, + new CcNativeLibraryProvider(collectNativeCcLibraries(ccLinkingOutputs))); + } + providers.put(CcExecutionDynamicLibrariesProvider.class, + collectExecutionDynamicLibraryArtifacts(ccLinkingOutputs.getExecutionDynamicLibraries())); + + boolean forcePic = ruleContext.getFragment(CppConfiguration.class).forcePic(); + if (emitCcSpecificLinkParamsProvider) { + providers.put(CcSpecificLinkParamsProvider.class, new CcSpecificLinkParamsProvider( + createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic))); + } else { + providers.put(CcLinkParamsProvider.class, new CcLinkParamsProvider( + createCcLinkParamsStore(ccLinkingOutputs, cppCompilationContext, forcePic))); + } + return new Info(providers, ccOutputs, ccLinkingOutputs, originalLinkingOutputs, + cppCompilationContext); + } + + /** + * Create context for cc compile action from generated inputs. + */ + private CppCompilationContext initializeCppCompilationContext(CppModel model, + FeatureConfiguration featureConfiguration) { + CppCompilationContext.Builder contextBuilder = + new CppCompilationContext.Builder(ruleContext); + + // Setup the include path; local include directories come before those inherited from deps or + // from the toolchain; in case of aliasing (same include file found on different entries), + // prefer the local include rather than the inherited one. + + // Add in the roots for well-formed include names for source files and + // generated files. It is important that the execRoot (EMPTY_FRAGMENT) comes + // before the genfilesFragment to preferably pick up source files. Otherwise + // we might pick up stale generated files. + contextBuilder.addQuoteIncludeDir(PathFragment.EMPTY_FRAGMENT); + contextBuilder.addQuoteIncludeDir(ruleContext.getConfiguration().getGenfilesFragment()); + + for (PathFragment systemIncludeDir : systemIncludeDirs) { + contextBuilder.addSystemIncludeDir(systemIncludeDir); + } + for (PathFragment includeDir : includeDirs) { + contextBuilder.addIncludeDir(includeDir); + } + + contextBuilder.mergeDependentContexts( + AnalysisUtils.getProviders(deps, CppCompilationContext.class)); + CppHelper.mergeToolchainDependentContext(ruleContext, contextBuilder); + + // But defines come after those inherited from deps. + contextBuilder.addDefines(defines); + + // There are no ordering constraints for declared include dirs/srcs, or the pregrepped headers. + contextBuilder.addDeclaredIncludeSrcs(publicHeaders); + contextBuilder.addDeclaredIncludeSrcs(privateHeaders); + contextBuilder.addPregreppedHeaderMap( + CppHelper.createExtractInclusions(ruleContext, publicHeaders)); + contextBuilder.addPregreppedHeaderMap( + CppHelper.createExtractInclusions(ruleContext, privateHeaders)); + contextBuilder.addCompilationPrerequisites(prerequisites); + + // Add this package's dir to declaredIncludeDirs, & this rule's headers to declaredIncludeSrcs + // Note: no include dir for STRICT mode. + if (headersCheckingMode == HeadersCheckingMode.WARN) { + contextBuilder.addDeclaredIncludeWarnDir(ruleContext.getLabel().getPackageFragment()); + for (PathFragment looseIncludeDir : looseIncludeDirs) { + contextBuilder.addDeclaredIncludeWarnDir(looseIncludeDir); + } + } else if (headersCheckingMode == HeadersCheckingMode.LOOSE) { + contextBuilder.addDeclaredIncludeDir(ruleContext.getLabel().getPackageFragment()); + for (PathFragment looseIncludeDir : looseIncludeDirs) { + contextBuilder.addDeclaredIncludeDir(looseIncludeDir); + } + } + + if (emitCppModuleMaps) { + CppModuleMap cppModuleMap = CppHelper.addCppModuleMapToContext(ruleContext, contextBuilder); + // TODO(bazel-team): addCppModuleMapToContext second-guesses whether module maps should + // actually be enabled, so we need to double-check here. Who would write code like this? + if (cppModuleMap != null) { + CppModuleMapAction action = new CppModuleMapAction(ruleContext.getActionOwner(), + cppModuleMap, + privateHeaders, + publicHeaders, + collectModuleMaps(), + additionalExportedHeaders, + compileHeaderModules, + featureConfiguration.isEnabled(CppRuleClasses.MODULE_MAP_HOME_CWD)); + ruleContext.registerAction(action); + } + if (model.getGeneratesPicHeaderModule()) { + contextBuilder.setPicHeaderModule(model.getPicHeaderModule(cppModuleMap.getArtifact())); + } + if (model.getGeratesNoPicHeaderModule()) { + contextBuilder.setHeaderModule(model.getHeaderModule(cppModuleMap.getArtifact())); + } + } + + semantics.setupCompilationContext(ruleContext, contextBuilder); + return contextBuilder.build(); + } + + private Iterable collectModuleMaps() { + // Cpp module maps may be null for some rules. We filter the nulls out at the end. + List result = new ArrayList<>(); + Iterables.addAll(result, Iterables.transform(deps, CPP_DEPS_TO_MODULES)); + CppCompilationContext stl = + ruleContext.getPrerequisite(":stl", Mode.TARGET, CppCompilationContext.class); + if (stl != null) { + result.add(stl.getCppModuleMap()); + } + + CcToolchainProvider toolchain = CppHelper.getToolchain(ruleContext); + if (toolchain != null) { + result.add(toolchain.getCppCompilationContext().getCppModuleMap()); + } + + if (emitHeaderTargetModuleMaps) { + for (HeaderTargetModuleMapProvider provider : AnalysisUtils.getProviders( + deps, HeaderTargetModuleMapProvider.class)) { + result.addAll(provider.getCppModuleMaps()); + } + } + + return Iterables.filter(result, Predicates.notNull()); + } + + private TransitiveLipoInfoProvider collectTransitiveLipoInfo(CcCompilationOutputs outputs) { + if (ruleContext.getFragment(CppConfiguration.class).getFdoSupport().getFdoRoot() == null) { + return TransitiveLipoInfoProvider.EMPTY; + } + NestedSetBuilder scannableBuilder = NestedSetBuilder.stableOrder(); + // TODO(bazel-team): Only fetch the STL prerequisite in one place. + TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET); + if (stl != null) { + TransitiveLipoInfoProvider provider = stl.getProvider(TransitiveLipoInfoProvider.class); + if (provider != null) { + scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables()); + } + } + + for (TransitiveLipoInfoProvider dep : + AnalysisUtils.getProviders(deps, TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables()); + } + + for (IncludeScannable scannable : outputs.getLipoScannables()) { + Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1); + scannableBuilder.add(scannable); + } + return new TransitiveLipoInfoProvider(scannableBuilder.build()); + } + + private Runfiles collectCppRunfiles( + CcLinkingOutputs ccLinkingOutputs, boolean linkingStatically) { + Runfiles.Builder builder = new Runfiles.Builder(); + builder.addTargets(deps, RunfilesProvider.DEFAULT_RUNFILES); + builder.addTargets(deps, CppRunfilesProvider.runfilesFunction(linkingStatically)); + // Add the shared libraries to the runfiles. + builder.addArtifacts(ccLinkingOutputs.getLibrariesForRunfiles(linkingStatically)); + return builder.build(); + } + + private CcLinkParamsStore createCcLinkParamsStore( + final CcLinkingOutputs ccLinkingOutputs, final CppCompilationContext cppCompilationContext, + final boolean forcePic) { + return new CcLinkParamsStore() { + @Override + protected void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared) { + builder.addLinkstamps(linkstamps, cppCompilationContext); + builder.addTransitiveTargets(deps, + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + if (!neverlink) { + builder.addLibraries(ccLinkingOutputs.getPreferredLibraries(linkingStatically, + /*preferPic=*/linkShared || forcePic)); + builder.addLinkOpts(linkopts); + } + } + }; + } + + private NestedSet collectNativeCcLibraries(CcLinkingOutputs ccLinkingOutputs) { + NestedSetBuilder result = NestedSetBuilder.linkOrder(); + result.addAll(ccLinkingOutputs.getDynamicLibraries()); + for (CcNativeLibraryProvider dep : AnalysisUtils.getProviders( + deps, CcNativeLibraryProvider.class)) { + result.addTransitive(dep.getTransitiveCcNativeLibraries()); + } + + return result.build(); + } + + private CcExecutionDynamicLibrariesProvider collectExecutionDynamicLibraryArtifacts( + List executionDynamicLibraries) { + Iterable artifacts = LinkerInputs.toLibraryArtifacts(executionDynamicLibraries); + if (!Iterables.isEmpty(artifacts)) { + return new CcExecutionDynamicLibrariesProvider( + NestedSetBuilder.wrap(Order.STABLE_ORDER, artifacts)); + } + + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (CcExecutionDynamicLibrariesProvider dep : + AnalysisUtils.getProviders(deps, CcExecutionDynamicLibrariesProvider.class)) { + builder.addTransitive(dep.getExecutionDynamicLibraryArtifacts()); + } + return builder.isEmpty() + ? CcExecutionDynamicLibrariesProvider.EMPTY + : new CcExecutionDynamicLibrariesProvider(builder.build()); + } + + private TempsProvider getTemps(CcCompilationOutputs compilationOutputs) { + return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector() + ? new TempsProvider(ImmutableList.of()) + : new TempsProvider(compilationOutputs.getTemps()); + } + + private ImmutableList getFilesToCompile(CcCompilationOutputs compilationOutputs) { + return ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector() + ? ImmutableList.of() + : compilationOutputs.getObjectFiles(CppHelper.usePic(ruleContext, false)); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java new file mode 100644 index 0000000000..4e4804b177 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParams.java @@ -0,0 +1,357 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; + +import java.util.Collection; +import java.util.Objects; + +/** + * Parameters to be passed to the linker. + * + *

The parameters concerned are the link options (strings) passed to the linker, linkstamps and a + * list of libraries to be linked in. + * + *

Items in the collections are stored in nested sets. Link options and libraries are stored in + * link order (preorder) and linkstamps are sorted. + */ +public final class CcLinkParams { + private final NestedSet> linkOpts; + private final NestedSet linkstamps; + private final NestedSet libraries; + + private CcLinkParams(NestedSet> linkOpts, + NestedSet linkstamps, + NestedSet libraries) { + this.linkOpts = linkOpts; + this.linkstamps = linkstamps; + this.libraries = libraries; + } + + /** + * @return the linkopts + */ + public NestedSet> getLinkopts() { + return linkOpts; + } + + public ImmutableList flattenedLinkopts() { + return ImmutableList.copyOf(Iterables.concat(linkOpts)); + } + + /** + * @return the linkstamps + */ + public NestedSet getLinkstamps() { + return linkstamps; + } + + /** + * @return the libraries + */ + public NestedSet getLibraries() { + return libraries; + } + + public static final Builder builder(boolean linkingStatically, boolean linkShared) { + return new Builder(linkingStatically, linkShared); + } + + /** + * Builder for {@link CcLinkParams}. + * + * + */ + public static final class Builder { + + /** + * linkingStatically is true when we're linking this target in either FULLY STATIC mode + * (linkopts=["-static"]) or MOSTLY STATIC mode (linkstatic=1). When this is true, we want to + * use static versions of any libraries that this target depends on (except possibly system + * libraries, which are not handled by CcLinkParams). When this is false, we want to use dynamic + * versions of any libraries that this target depends on. + */ + private final boolean linkingStatically; + + /** + * linkShared is true when we're linking with "-shared" (linkshared=1). + */ + private final boolean linkShared; + + private ImmutableList.Builder localLinkoptsBuilder = ImmutableList.builder(); + + private final NestedSetBuilder> linkOptsBuilder = + NestedSetBuilder.linkOrder(); + private final NestedSetBuilder linkstampsBuilder = + NestedSetBuilder.compileOrder(); + private final NestedSetBuilder librariesBuilder = + NestedSetBuilder.linkOrder(); + + private boolean built = false; + + private Builder(boolean linkingStatically, boolean linkShared) { + this.linkingStatically = linkingStatically; + this.linkShared = linkShared; + } + + /** + * Build a {@link CcLinkParams} object. + */ + public CcLinkParams build() { + Preconditions.checkState(!built); + // Not thread-safe, but builders should not be shared across threads. + built = true; + ImmutableList localLinkopts = localLinkoptsBuilder.build(); + if (!localLinkopts.isEmpty()) { + linkOptsBuilder.add(localLinkopts); + } + return new CcLinkParams(linkOptsBuilder.build(), linkstampsBuilder.build(), + librariesBuilder.build()); + } + + private boolean add(CcLinkParamsStore store) { + if (store != null) { + CcLinkParams args = store.get(linkingStatically, linkShared); + addTransitiveArgs(args); + } + return store != null; + } + + /** + * Includes link parameters from a collection of dependency targets. + */ + public Builder addTransitiveTargets(Iterable targets) { + for (TransitiveInfoCollection target : targets) { + addTransitiveTarget(target); + } + return this; + } + + /** + * Includes link parameters from a dependency target. + * + *

The target should implement {@link CcLinkParamsProvider}. If it does not, + * the method does not do anything. + */ + public Builder addTransitiveTarget(TransitiveInfoCollection target) { + return addTransitiveProvider(target.getProvider(CcLinkParamsProvider.class)); + } + + /** + * Includes link parameters from a dependency target. The target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + */ + @SafeVarargs + public final Builder addTransitiveTarget(TransitiveInfoCollection target, + Function firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function... remainingMappings) { + if (add(firstMapping.apply(target))) { + return this; + } + for (Function mapping : remainingMappings) { + if (add(mapping.apply(target))) { + return this; + } + } + return this; + } + + /** + * Includes link parameters from a CcLinkParamsProvider provider. + */ + public Builder addTransitiveProvider(CcLinkParamsProvider provider) { + if (provider == null) { + return this; + } + + CcLinkParams args = provider.getCcLinkParams(linkingStatically, linkShared); + addTransitiveArgs(args); + return this; + } + + /** + * Includes link parameters from the given targets. Each target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + */ + @SafeVarargs + public final Builder addTransitiveTargets( + Iterable targets, + Function firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function... remainingMappings) { + for (TransitiveInfoCollection target : targets) { + addTransitiveTarget(target, firstMapping, remainingMappings); + } + return this; + } + + /** + * Includes link parameters from the given targets. Each target is checked for the given + * mappings in the order specified, and the first mapping that returns a non-null result is + * added. + * + * @deprecated don't add any new uses; all existing uses need to be audited and possibly merged + * into a single call - some of them may introduce semantic changes which need to be + * carefully vetted + */ + @Deprecated + @SafeVarargs + public final Builder addTransitiveLangTargets( + Iterable targets, + Function firstMapping, + @SuppressWarnings("unchecked") // Java arrays don't preserve generic arguments. + Function... remainingMappings) { + return addTransitiveTargets(targets, firstMapping, remainingMappings); + } + + /** + * Merges the other {@link CcLinkParams} object into this one. + */ + public Builder addTransitiveArgs(CcLinkParams args) { + linkOptsBuilder.addTransitive(args.getLinkopts()); + linkstampsBuilder.addTransitive(args.getLinkstamps()); + librariesBuilder.addTransitive(args.getLibraries()); + return this; + } + + /** + * Adds a collection of link options. + */ + public Builder addLinkOpts(Collection linkOpts) { + localLinkoptsBuilder.addAll(linkOpts); + return this; + } + + /** + * Adds a collection of linkstamps. + */ + public Builder addLinkstamps(Iterable linkstamps, CppCompilationContext context) { + ImmutableList declaredIncludeSrcs = + ImmutableList.copyOf(context.getDeclaredIncludeSrcs()); + for (Artifact linkstamp : linkstamps) { + linkstampsBuilder.add(new Linkstamp(linkstamp, declaredIncludeSrcs)); + } + return this; + } + + /** + * Adds a library artifact. + */ + public Builder addLibrary(LibraryToLink library) { + librariesBuilder.add(library); + return this; + } + + /** + * Adds a collection of library artifacts. + */ + public Builder addLibraries(Iterable libraries) { + librariesBuilder.addAll(libraries); + return this; + } + + /** + * Processes typical dependencies a C/C++ library. + * + *

A helper method that processes getValues() and merges contents of + * getPreferredLibraries() and getLinkOpts() into the current link params + * object. + */ + public Builder addCcLibrary(RuleContext context, CcCommon common, boolean neverlink, + CcLinkingOutputs linkingOutputs) { + addTransitiveTargets( + context.getPrerequisites("deps", Mode.TARGET), + CcLinkParamsProvider.TO_LINK_PARAMS, CcSpecificLinkParamsProvider.TO_LINK_PARAMS); + + if (!neverlink) { + addLibraries(linkingOutputs.getPreferredLibraries(linkingStatically, + linkShared || context.getFragment(CppConfiguration.class).forcePic())); + addLinkOpts(common.getLinkopts()); + } + return this; + } + } + + /** + * A linkstamp that also knows about its declared includes. + * + *

This object is required because linkstamp files may include other headers which + * will have to be provided during compilation. + */ + public static final class Linkstamp { + private final Artifact artifact; + private final ImmutableList declaredIncludeSrcs; + + private Linkstamp(Artifact artifact, ImmutableList declaredIncludeSrcs) { + this.artifact = Preconditions.checkNotNull(artifact); + this.declaredIncludeSrcs = Preconditions.checkNotNull(declaredIncludeSrcs); + } + + /** + * Returns the linkstamp artifact. + */ + public Artifact getArtifact() { + return artifact; + } + + /** + * Returns the declared includes. + */ + public ImmutableList getDeclaredIncludeSrcs() { + return declaredIncludeSrcs; + } + + @Override + public int hashCode() { + return Objects.hash(artifact, declaredIncludeSrcs); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Linkstamp)) { + return false; + } + Linkstamp other = (Linkstamp) obj; + return artifact.equals(other.artifact) + && declaredIncludeSrcs.equals(other.declaredIncludeSrcs); + } + } + + /** + * Empty CcLinkParams. + */ + public static final CcLinkParams EMPTY = new CcLinkParams( + NestedSetBuilder.>emptySet(Order.LINK_ORDER), + NestedSetBuilder.emptySet(Order.COMPILE_ORDER), + NestedSetBuilder.emptySet(Order.LINK_ORDER)); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java new file mode 100644 index 0000000000..11f6011f50 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsProvider.java @@ -0,0 +1,50 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides C linker parameters. + */ +@Immutable +public final class CcLinkParamsProvider implements TransitiveInfoProvider { + public static final Function TO_LINK_PARAMS = + new Function() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + CcLinkParamsProvider provider = input.getProvider( + CcLinkParamsProvider.class); + return provider == null ? null : provider.store; + } + }; + + private final CcLinkParamsStoreImpl store; + + public CcLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + /** + * Returns link parameters given static / shared linking settings. + */ + public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) { + return store.get(linkingStatically, linkShared); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java new file mode 100644 index 0000000000..a150488d84 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkParamsStore.java @@ -0,0 +1,136 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.rules.cpp.CcLinkParams.Builder; + +/** + * A cache of C link parameters. + * + *

The cache holds instances of {@link CcLinkParams} for combinations of + * linkingStatically and linkShared. If a requested value is not available in + * the cache, it is computed and then stored. + * + *

Typically this class is used on targets that may be linked in as C + * libraries as in the following example: + * + * 

+ * class SomeTarget implements CcLinkParamsProvider {
+ *   private final CcLinkParamsStore ccLinkParamsStore = new CcLinkParamsStore() {
+ *     @Override
+ *     protected void collect(CcLinkParams.Builder builder, boolean linkingStatically,
+ *                            boolean linkShared) {
+ *       builder.add[...]
+ *     }
+ *   };
+ *
+ *   @Override
+ *   public CcLinkParams getCcLinkParams(boolean linkingStatically, boolean linkShared) {
+ *     return ccLinkParamsStore.get(linkingStatically, linkShared);
+ *   }
+ * }
+ *
+ */ +public abstract class CcLinkParamsStore { + + private CcLinkParams staticSharedParams; + private CcLinkParams staticNoSharedParams; + private CcLinkParams noStaticSharedParams; + private CcLinkParams noStaticNoSharedParams; + + private CcLinkParams compute(boolean linkingStatically, boolean linkShared) { + CcLinkParams.Builder builder = CcLinkParams.builder(linkingStatically, linkShared); + collect(builder, linkingStatically, linkShared); + return builder.build(); + } + + /** + * Returns {@link CcLinkParams} for a combination of parameters. + * + *

The {@link CcLinkParams} instance is computed lazily and cached. + */ + public synchronized CcLinkParams get(boolean linkingStatically, boolean linkShared) { + CcLinkParams result = lookup(linkingStatically, linkShared); + if (result == null) { + result = compute(linkingStatically, linkShared); + put(linkingStatically, linkShared, result); + } + return result; + } + + private CcLinkParams lookup(boolean linkingStatically, boolean linkShared) { + if (linkingStatically) { + return linkShared ? staticSharedParams : staticNoSharedParams; + } else { + return linkShared ? noStaticSharedParams : noStaticNoSharedParams; + } + } + + private void put(boolean linkingStatically, boolean linkShared, CcLinkParams params) { + Preconditions.checkNotNull(params); + if (linkingStatically) { + if (linkShared) { + staticSharedParams = params; + } else { + staticNoSharedParams = params; + } + } else { + if (linkShared) { + noStaticSharedParams = params; + } else { + noStaticNoSharedParams = params; + } + } + } + + /** + * Hook for building the actual link params. + * + *

Users should override this method and call methods of the builder to + * set up the actual CcLinkParams objects. + * + *

Implementations of this method must not fail or try to report errors on the + * configured target. + */ + protected abstract void collect(CcLinkParams.Builder builder, boolean linkingStatically, + boolean linkShared); + + /** + * An empty CcLinkParamStore. + */ + public static final CcLinkParamsStore EMPTY = new CcLinkParamsStore() { + + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {} + }; + + /** + * An implementation class for the CcLinkParamsStore. + */ + public static final class CcLinkParamsStoreImpl extends CcLinkParamsStore { + + public CcLinkParamsStoreImpl(CcLinkParamsStore store) { + super.staticSharedParams = store.get(true, true); + super.staticNoSharedParams = store.get(true, false); + super.noStaticSharedParams = store.get(false, true); + super.noStaticNoSharedParams = store.get(false, false); + } + + @Override + protected void collect(Builder builder, boolean linkingStatically, boolean linkShared) {} + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java new file mode 100644 index 0000000000..6b45c79645 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcLinkingOutputs.java @@ -0,0 +1,243 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * A structured representation of the link outputs of a C++ rule. + */ +public class CcLinkingOutputs { + + public static final CcLinkingOutputs EMPTY = new Builder().build(); + + private final ImmutableList staticLibraries; + + private final ImmutableList picStaticLibraries; + + private final ImmutableList dynamicLibraries; + + private final ImmutableList executionDynamicLibraries; + + private CcLinkingOutputs(ImmutableList staticLibraries, + ImmutableList picStaticLibraries, + ImmutableList dynamicLibraries, + ImmutableList executionDynamicLibraries) { + this.staticLibraries = staticLibraries; + this.picStaticLibraries = picStaticLibraries; + this.dynamicLibraries = dynamicLibraries; + this.executionDynamicLibraries = executionDynamicLibraries; + } + + public ImmutableList getStaticLibraries() { + return staticLibraries; + } + + public ImmutableList getPicStaticLibraries() { + return picStaticLibraries; + } + + public ImmutableList getDynamicLibraries() { + return dynamicLibraries; + } + + public ImmutableList getExecutionDynamicLibraries() { + return executionDynamicLibraries; + } + + /** + * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of preference depending on the + * link preferences. + * + *

This method tries to simulate a search path for adding static and dynamic libraries, + * allowing either to be preferred over the other depending on the link {@link LinkStaticness}. + * + * TODO(bazel-team): (2009) we should preserve the relative ordering of first and second + * choice libraries. E.g. if srcs=['foo.a','bar.so','baz.a'] then we should link them in the + * same order. Currently we link entries from the first choice list before those from the + * second choice list, i.e. in the order {@code ['bar.so', 'foo.a', 'baz.a']}. + * + * @param linkingStatically whether to prefer static over dynamic libraries. Should be + * true for binaries that are linked in fully static or mostly static mode. + * @param preferPic whether to prefer pic over non pic libraries (usually used when linking + * shared) + */ + public List getPreferredLibraries( + boolean linkingStatically, boolean preferPic) { + return getPreferredLibraries(linkingStatically, preferPic, false); + } + + /** + * Returns the shared libraries that are linked against and therefore also need to be in the + * runfiles. + */ + public Iterable getLibrariesForRunfiles(boolean linkingStatically) { + List libraries = + getPreferredLibraries(linkingStatically, /*preferPic*/false, true); + return CcCommon.getSharedLibrariesFrom(LinkerInputs.toLibraryArtifacts(libraries)); + } + + /** + * Add the ".a", ".pic.a" and/or ".so" files in appropriate order of + * preference depending on the link preferences. + */ + private List getPreferredLibraries(boolean linkingStatically, boolean preferPic, + boolean forRunfiles) { + List candidates = new ArrayList<>(); + // It's important that this code keeps the invariant that preferPic has no effect on the output + // of .so libraries. That is, the resulting list should contain the same .so files in the same + // order. + if (linkingStatically) { // Prefer the static libraries. + if (preferPic) { + // First choice is the PIC static libraries. + // Second choice is the other static libraries (may cause link error if they're not PIC, + // but I think this is preferable to linking dynamically when you asked for statically). + candidates.addAll(picStaticLibraries); + candidates.addAll(staticLibraries); + } else { + // First choice is the non-pic static libraries (best performance); + // second choice is the staticPicLibraries (at least they're static; + // we can live with the extra overhead of PIC). + candidates.addAll(staticLibraries); + candidates.addAll(picStaticLibraries); + } + candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries); + } else { + // First choice is the dynamicLibraries. + candidates.addAll(forRunfiles ? executionDynamicLibraries : dynamicLibraries); + if (preferPic) { + // Second choice is the staticPicLibraries (at least they're PIC, so we won't get a + // link error). + candidates.addAll(picStaticLibraries); + candidates.addAll(staticLibraries); + } else { + candidates.addAll(staticLibraries); + candidates.addAll(picStaticLibraries); + } + } + return filterCandidates(candidates); + } + + /** + * Helper method to filter the candidates by removing equivalent library + * entries from the list of candidates. + * + * @param candidates the library candidates to filter + * @return the list of libraries with equivalent duplicate libraries removed. + */ + private List filterCandidates(List candidates) { + List libraries = new ArrayList<>(); + Set identifiers = new HashSet<>(); + for (LibraryToLink library : candidates) { + if (identifiers.add(libraryIdentifierOf(library.getOriginalLibraryArtifact()))) { + libraries.add(library); + } + } + return libraries; + } + + /** + * Returns the library identifier of an artifact: a string that is different for different + * libraries, but is the same for the shared, static and pic versions of the same library. + */ + private static String libraryIdentifierOf(Artifact libraryArtifact) { + String name = libraryArtifact.getRootRelativePath().getPathString(); + String basename = FileSystemUtils.removeExtension(name); + // Need to special-case file types with double extension. + return name.endsWith(".pic.a") + ? FileSystemUtils.removeExtension(basename) + : name.endsWith(".nopic.a") + ? FileSystemUtils.removeExtension(basename) + : name.endsWith(".pic.lo") + ? FileSystemUtils.removeExtension(basename) + : basename; + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private final Set staticLibraries = new LinkedHashSet<>(); + private final Set picStaticLibraries = new LinkedHashSet<>(); + private final Set dynamicLibraries = new LinkedHashSet<>(); + private final Set executionDynamicLibraries = new LinkedHashSet<>(); + + public CcLinkingOutputs build() { + return new CcLinkingOutputs(ImmutableList.copyOf(staticLibraries), + ImmutableList.copyOf(picStaticLibraries), ImmutableList.copyOf(dynamicLibraries), + ImmutableList.copyOf(executionDynamicLibraries)); + } + + public Builder merge(CcLinkingOutputs outputs) { + staticLibraries.addAll(outputs.getStaticLibraries()); + picStaticLibraries.addAll(outputs.getPicStaticLibraries()); + dynamicLibraries.addAll(outputs.getDynamicLibraries()); + executionDynamicLibraries.addAll(outputs.getExecutionDynamicLibraries()); + return this; + } + + public Builder addStaticLibrary(LibraryToLink library) { + staticLibraries.add(library); + return this; + } + + public Builder addStaticLibraries(Iterable libraries) { + Iterables.addAll(staticLibraries, libraries); + return this; + } + + public Builder addPicStaticLibrary(LibraryToLink library) { + picStaticLibraries.add(library); + return this; + } + + public Builder addPicStaticLibraries(Iterable libraries) { + Iterables.addAll(picStaticLibraries, libraries); + return this; + } + + public Builder addDynamicLibrary(LibraryToLink library) { + dynamicLibraries.add(library); + return this; + } + + public Builder addDynamicLibraries(Iterable libraries) { + Iterables.addAll(dynamicLibraries, libraries); + return this; + } + + public Builder addExecutionDynamicLibrary(LibraryToLink library) { + executionDynamicLibraries.add(library); + return this; + } + + public Builder addExecutionDynamicLibraries(Iterable libraries) { + Iterables.addAll(executionDynamicLibraries, libraries); + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java new file mode 100644 index 0000000000..5e96291520 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcNativeLibraryProvider.java @@ -0,0 +1,43 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that provides native libraries in the transitive closure of its deps that are needed for + * executing C++ code. + */ +@Immutable +public final class CcNativeLibraryProvider implements TransitiveInfoProvider { + + private final NestedSet transitiveCcNativeLibraries; + + public CcNativeLibraryProvider(NestedSet transitiveCcNativeLibraries) { + this.transitiveCcNativeLibraries = transitiveCcNativeLibraries; + } + + /** + * Collects native libraries in the transitive closure of its deps that are needed for executing + * C/C++ code. + * + *

In effect, returns all dynamic library (.so) artifacts provided by the transitive closure. + */ + public NestedSet getTransitiveCcNativeLibraries() { + return transitiveCcNativeLibraries; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java new file mode 100644 index 0000000000..dfcecc276c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcSpecificLinkParamsProvider.java @@ -0,0 +1,48 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.CcLinkParamsStore.CcLinkParamsStoreImpl; + +/** + * A target that provides libraries to be only linked into other C++ targets (and not targets + * for other languages) + */ +@Immutable +public final class CcSpecificLinkParamsProvider implements TransitiveInfoProvider { + private final CcLinkParamsStoreImpl store; + + public CcSpecificLinkParamsProvider(CcLinkParamsStore store) { + this.store = new CcLinkParamsStoreImpl(store); + } + + public CcLinkParamsStore getLinkParams() { + return store; + } + + public static final Function TO_LINK_PARAMS = + new Function() { + @Override + public CcLinkParamsStore apply(TransitiveInfoCollection input) { + CcSpecificLinkParamsProvider provider = input.getProvider( + CcSpecificLinkParamsProvider.class); + return provider == null ? null : provider.getLinkParams(); + } + }; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java new file mode 100644 index 0000000000..78271836b0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcTest.java @@ -0,0 +1,36 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; + +/** + * A configured target class for cc_test rules. + */ +public abstract class CcTest implements RuleConfiguredTargetFactory { + + private final CppSemantics semantics; + + protected CcTest(CppSemantics semantics) { + this.semantics = semantics; + } + + @Override + public ConfiguredTarget create(RuleContext context) throws InterruptedException { + return CcBinary.init(semantics, context, /*fake =*/ false, /*useTestOnlyFlags =*/ true); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java new file mode 100644 index 0000000000..bd39d0f745 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java @@ -0,0 +1,249 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisUtils; +import com.google.devtools.build.lib.analysis.CompilationHelper; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.FileProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider; +import com.google.devtools.build.lib.analysis.LicensesProvider.TargetLicense; +import com.google.devtools.build.lib.analysis.MiddlemanProvider; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.Runfiles; +import com.google.devtools.build.lib.analysis.RunfilesProvider; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.packages.License; +import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.List; + +/** + * Implementation for the cc_toolchain rule. + */ +public class CcToolchain implements RuleConfiguredTargetFactory { + + @Override + public ConfiguredTarget create(RuleContext ruleContext) { + final Label label = ruleContext.getLabel(); + final NestedSet crosstool = ruleContext.getPrerequisite("all_files", Mode.HOST) + .getProvider(FileProvider.class).getFilesToBuild(); + final NestedSet crosstoolMiddleman = getFiles(ruleContext, "all_files"); + final NestedSet compile = getFiles(ruleContext, "compiler_files"); + final NestedSet strip = getFiles(ruleContext, "strip_files"); + final NestedSet objcopy = getFiles(ruleContext, "objcopy_files"); + final NestedSet link = getFiles(ruleContext, "linker_files"); + final NestedSet dwp = getFiles(ruleContext, "dwp_files"); + final NestedSet libcLink = inputsForLibcLink(ruleContext); + String purposePrefix = Actions.escapeLabel(label) + "_"; + String runtimeSolibDirBase = "_solib_" + "_" + Actions.escapeLabel(label); + final PathFragment runtimeSolibDir = ruleContext.getConfiguration() + .getBinFragment().getRelative(runtimeSolibDirBase); + + CppConfiguration cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + // Static runtime inputs. + TransitiveInfoCollection staticRuntimeLibDep = selectDep(ruleContext, "static_runtime_libs", + cppConfiguration.getStaticRuntimeLibsLabel()); + final NestedSet staticRuntimeLinkInputs; + final Artifact staticRuntimeLinkMiddleman; + if (cppConfiguration.supportsEmbeddedRuntimes()) { + staticRuntimeLinkInputs = staticRuntimeLibDep + .getProvider(FileProvider.class) + .getFilesToBuild(); + } else { + staticRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (!staticRuntimeLinkInputs.isEmpty()) { + NestedSet staticRuntimeLinkMiddlemanSet = CompilationHelper.getAggregatingMiddleman( + ruleContext, + purposePrefix + "static_runtime_link", + staticRuntimeLibDep); + staticRuntimeLinkMiddleman = staticRuntimeLinkMiddlemanSet.isEmpty() + ? null : Iterables.getOnlyElement(staticRuntimeLinkMiddlemanSet); + } else { + staticRuntimeLinkMiddleman = null; + } + + Preconditions.checkState( + (staticRuntimeLinkMiddleman == null) == staticRuntimeLinkInputs.isEmpty()); + + // Dynamic runtime inputs. + TransitiveInfoCollection dynamicRuntimeLibDep = selectDep(ruleContext, "dynamic_runtime_libs", + cppConfiguration.getDynamicRuntimeLibsLabel()); + final NestedSet dynamicRuntimeLinkInputs; + final Artifact dynamicRuntimeLinkMiddleman; + if (cppConfiguration.supportsEmbeddedRuntimes()) { + NestedSetBuilder dynamicRuntimeLinkInputsBuilder = NestedSetBuilder.stableOrder(); + for (Artifact artifact : dynamicRuntimeLibDep + .getProvider(FileProvider.class).getFilesToBuild()) { + if (CppHelper.SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) { + dynamicRuntimeLinkInputsBuilder.add(SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext, artifact, runtimeSolibDirBase, + ruleContext.getConfiguration()).getArtifact()); + } else { + dynamicRuntimeLinkInputsBuilder.add(artifact); + } + } + dynamicRuntimeLinkInputs = dynamicRuntimeLinkInputsBuilder.build(); + } else { + dynamicRuntimeLinkInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (!dynamicRuntimeLinkInputs.isEmpty()) { + List dynamicRuntimeLinkMiddlemanSet = + CppHelper.getAggregatingMiddlemanForCppRuntimes( + ruleContext, + purposePrefix + "dynamic_runtime_link", + dynamicRuntimeLibDep, + runtimeSolibDirBase, + ruleContext.getConfiguration()); + dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddlemanSet.isEmpty() + ? null : Iterables.getOnlyElement(dynamicRuntimeLinkMiddlemanSet); + } else { + dynamicRuntimeLinkMiddleman = null; + } + + Preconditions.checkState( + (dynamicRuntimeLinkMiddleman == null) == dynamicRuntimeLinkInputs.isEmpty()); + + CppCompilationContext.Builder contextBuilder = + new CppCompilationContext.Builder(ruleContext); + CppModuleMap moduleMap = createCrosstoolModuleMap(ruleContext); + if (moduleMap != null) { + contextBuilder.setCppModuleMap(moduleMap); + } + final CppCompilationContext context = contextBuilder.build(); + boolean supportsParamFiles = ruleContext.attributes().get("supports_param_files", BOOLEAN); + boolean supportsHeaderParsing = + ruleContext.attributes().get("supports_header_parsing", BOOLEAN); + + CcToolchainProvider provider = new CcToolchainProvider( + Preconditions.checkNotNull(ruleContext.getFragment(CppConfiguration.class)), + crosstool, + fullInputsForCrosstool(ruleContext, crosstoolMiddleman), + compile, + strip, + objcopy, + fullInputsForLink(ruleContext, link), + dwp, + libcLink, + staticRuntimeLinkInputs, + staticRuntimeLinkMiddleman, + dynamicRuntimeLinkInputs, + dynamicRuntimeLinkMiddleman, + runtimeSolibDir, + context, + supportsParamFiles, + supportsHeaderParsing); + RuleConfiguredTargetBuilder builder = + new RuleConfiguredTargetBuilder(ruleContext) + .add(CcToolchainProvider.class, provider) + .setFilesToBuild(new NestedSetBuilder(Order.STABLE_ORDER).build()) + .add(RunfilesProvider.class, RunfilesProvider.simple(Runfiles.EMPTY)); + + // If output_license is specified on the cc_toolchain rule, override the transitive licenses + // with that one. This is necessary because cc_toolchain is used in the target configuration, + // but it is sort-of-kind-of a tool, but various parts of it are linked into the output... + // ...so we trust the judgment of the author of the cc_toolchain rule to figure out what + // licenses should be propagated to C++ targets. + License outputLicense = ruleContext.getRule().getToolOutputLicense(ruleContext.attributes()); + if (outputLicense != null && outputLicense != License.NO_LICENSE) { + final NestedSet license = NestedSetBuilder.create(Order.STABLE_ORDER, + new TargetLicense(ruleContext.getLabel(), outputLicense)); + LicensesProvider licensesProvider = new LicensesProvider() { + @Override + public NestedSet getTransitiveLicenses() { + return license; + } + }; + + builder.add(LicensesProvider.class, licensesProvider); + } + + return builder.build(); + } + + private NestedSet inputsForLibcLink(RuleContext ruleContext) { + TransitiveInfoCollection libcLink = ruleContext.getPrerequisite(":libc_link", Mode.HOST); + return libcLink != null + ? libcLink.getProvider(FileProvider.class).getFilesToBuild() + : NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + private NestedSet fullInputsForCrosstool(RuleContext ruleContext, + NestedSet crosstoolMiddleman) { + return NestedSetBuilder.stableOrder() + .addTransitive(crosstoolMiddleman) + // Use "libc_link" here, because it is functionally identical to the case + // below. If we introduce separate filegroups for compiling and linking, we + // need to fix that here. + .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link")) + .build(); + } + + private NestedSet fullInputsForLink(RuleContext ruleContext, NestedSet link) { + return NestedSetBuilder.stableOrder() + .addTransitive(link) + .addTransitive(AnalysisUtils.getMiddlemanFor(ruleContext, ":libc_link")) + .add(ruleContext.getAnalysisEnvironment().getEmbeddedToolArtifact( + CppRuleClasses.BUILD_INTERFACE_SO)) + .build(); + } + + private CppModuleMap createCrosstoolModuleMap(RuleContext ruleContext) { + if (ruleContext.getPrerequisite("module_map", Mode.HOST) == null) { + return null; + } + Artifact moduleMapArtifact = ruleContext.getPrerequisiteArtifact("module_map", Mode.HOST); + if (moduleMapArtifact == null) { + return null; + } + return new CppModuleMap(moduleMapArtifact, "crosstool"); + } + + private TransitiveInfoCollection selectDep( + RuleContext ruleContext, String attribute, Label label) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites(attribute, Mode.TARGET)) { + if (dep.getLabel().equals(label)) { + return dep; + } + } + + return ruleContext.getPrerequisites(attribute, Mode.TARGET).get(0); + } + + private NestedSet getFiles(RuleContext context, String attribute) { + TransitiveInfoCollection dep = context.getPrerequisite(attribute, Mode.HOST); + MiddlemanProvider middlemanProvider = dep.getProvider(MiddlemanProvider.class); + // We use the middleman if we can (if the dep is a filegroup), otherwise, just the regular + // filesToBuild (e.g. if it is a simple input file) + return middlemanProvider != null + ? middlemanProvider.getMiddlemanArtifact() + : dep.getProvider(FileProvider.class).getFilesToBuild(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java new file mode 100644 index 0000000000..29ab45cf4a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainFeatures.java @@ -0,0 +1,802 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; + +/** + * Provides access to features supported by a specific toolchain. + * + *

This class can be generated from the CToolchain protocol buffer. + * + *

TODO(bazel-team): Implement support for specifying the toolchain configuration directly from + * the BUILD file. + * + *

TODO(bazel-team): Find a place to put the public-facing documentation and link to it from + * here. + * + *

TODO(bazel-team): Split out Feature as CcToolchainFeature, which will modularize the + * crosstool configuration into one part that is about handling a set of features (including feature + * selection) and one part that is about how to apply a single feature (parsing flags and expanding + * them from build variables). + */ +@Immutable +public class CcToolchainFeatures implements Serializable { + + /** + * Thrown when a flag value cannot be expanded under a set of build variables. + * + *

This happens for example when a flag references a variable that is not provided by the + * action, or when a flag group references multiple variables of sequence type. + */ + public static class ExpansionException extends RuntimeException { + ExpansionException(String message) { + super(message); + } + } + + /** + * A piece of a single flag. + * + *

A single flag can contain a combination of text and variables (for example + * "-f %{var1}/%{var2}"). We split the flag into chunks, where each chunk represents either a + * text snippet, or a variable that is to be replaced. + */ + interface FlagChunk { + + /** + * Expands this chunk. + * + * @param variables variable names mapped to their values for a single flag expansion. + * @param flag the flag content to append to. + */ + void expand(Map variables, StringBuilder flag); + } + + /** + * A plain text chunk of a flag. + */ + @Immutable + private static class StringChunk implements FlagChunk, Serializable { + private final String text; + + private StringChunk(String text) { + this.text = text; + } + + @Override + public void expand(Map variables, StringBuilder flag) { + flag.append(text); + } + } + + /** + * A chunk of a flag into which a variable should be expanded. + */ + @Immutable + private static class VariableChunk implements FlagChunk, Serializable { + private final String variableName; + + private VariableChunk(String variableName) { + this.variableName = variableName; + } + + @Override + public void expand(Map variables, StringBuilder flag) { + String value = variables.get(variableName); + if (value == null) { + // We check all variables in FlagGroup.expandCommandLine, so if we arrive here with a + // null value, the variable map originally handed to the feature selection must have + // contained an explicit null value. + throw new ExpansionException("Internal blaze error: build variable was set to 'null'."); + } + flag.append(variables.get(variableName)); + } + } + + /** + * Parser for toolchain flags. + * + *

A flag contains a snippet of text supporting variable expansion. For example, a flag value + * "-f %{var1}/%{var2}" will expand the values of the variables "var1" and "var2" in the + * corresponding places in the string. + * + *

The {@code FlagParser} takes a flag string and parses it into a list of {@code FlagChunk} + * objects, where each chunk represents either a snippet of text or a variable to be expanded. In + * the above example, the resulting chunks would be ["-f ", var1, "/", var2]. + * + *

In addition to the list of chunks, the {@code FlagParser} also provides the set of variables + * necessary for the expansion of this flag via {@code getUsedVariables}. + * + *

To get a literal percent character, "%%" can be used in the flag text. + */ + private static class FlagParser { + + /** + * The given flag value. + */ + private final String value; + + /** + * The current position in {@value} during parsing. + */ + private int current = 0; + + private final ImmutableList.Builder chunks = ImmutableList.builder(); + private final ImmutableSet.Builder usedVariables = ImmutableSet.builder(); + + private FlagParser(String value) throws InvalidConfigurationException { + this.value = value; + parse(); + } + + /** + * @return the parsed chunks for this flag. + */ + private ImmutableList getChunks() { + return chunks.build(); + } + + /** + * @return all variable names needed to expand this flag. + */ + private ImmutableSet getUsedVariables() { + return usedVariables.build(); + } + + /** + * Parses the flag. + * + * @throws InvalidConfigurationException if there is a parsing error. + */ + private void parse() throws InvalidConfigurationException { + while (current < value.length()) { + if (atVariableStart()) { + parseVariableChunk(); + } else { + parseStringChunk(); + } + } + } + + /** + * @return whether the current position is the start of a variable. + */ + private boolean atVariableStart() { + // We parse a variable when value starts with '%', but not '%%'. + return value.charAt(current) == '%' + && (current + 1 >= value.length() || value.charAt(current + 1) != '%'); + } + + /** + * Parses a chunk of text until the next '%', which indicates either an escaped literal '%' + * or a variable. + */ + private void parseStringChunk() { + int start = current; + // We only parse string chunks starting with '%' if they also start with '%%'. + // In that case, we want to have a single '%' in the string, so we start at the second + // character. + // Note that for flags like "abc%%def" this will lead to two string chunks, the first + // referencing the subtring "abc", and a second referencing the substring "%def". + if (value.charAt(current) == '%') { + current = current + 1; + start = current; + } + current = value.indexOf('%', current + 1); + if (current == -1) { + current = value.length(); + } + final String text = value.substring(start, current); + chunks.add(new StringChunk(text)); + } + + /** + * Parses a variable to be expanded. + * + * @throws InvalidConfigurationException if there is a parsing error. + */ + private void parseVariableChunk() throws InvalidConfigurationException { + current = current + 1; + if (current >= value.length() || value.charAt(current) != '{') { + abort("expected '{'"); + } + current = current + 1; + if (current >= value.length() || value.charAt(current) == '}') { + abort("expected variable name"); + } + int end = value.indexOf('}', current); + final String name = value.substring(current, end); + usedVariables.add(name); + chunks.add(new VariableChunk(name)); + current = end + 1; + } + + /** + * @throws InvalidConfigurationException with the given error text, adding information about + * the current position in the flag. + */ + private void abort(String error) throws InvalidConfigurationException { + throw new InvalidConfigurationException("Invalid toolchain configuration: " + error + + " at position " + current + " while parsing a flag containing '" + value + "'"); + } + } + + /** + * A single flag to be expanded under a set of variables. + * + *

TODO(bazel-team): Consider specializing Flag for the simple case that a flag is just a bit + * of text. + */ + @Immutable + private static class Flag implements Serializable { + private final ImmutableList chunks; + + private Flag(ImmutableList chunks) { + this.chunks = chunks; + } + + /** + * Expand this flag into a single new entry in {@code commandLine}. + */ + private void expandCommandLine(Map variables, List commandLine) { + StringBuilder flag = new StringBuilder(); + for (FlagChunk chunk : chunks) { + chunk.expand(variables, flag); + } + commandLine.add(flag.toString()); + } + } + + /** + * A group of flags. + */ + @Immutable + private static class FlagGroup implements Serializable { + private final ImmutableList flags; + private final ImmutableSet usedVariables; + + private FlagGroup(CToolchain.FlagGroup flagGroup) throws InvalidConfigurationException { + ImmutableList.Builder flags = ImmutableList.builder(); + ImmutableSet.Builder usedVariables = ImmutableSet.builder(); + for (String flag : flagGroup.getFlagList()) { + FlagParser parser = new FlagParser(flag); + flags.add(new Flag(parser.getChunks())); + usedVariables.addAll(parser.getUsedVariables()); + } + this.flags = flags.build(); + this.usedVariables = usedVariables.build(); + } + + /** + * Expands all flags in this group and adds them to {@code commandLine}. + * + *

The flags of the group will be expanded either: + *

    + *
  • once, if there is no variable of sequence type in any of the group's flags, or
    • + *
  • for each element in the sequence, if there is one variable of sequence type within + * the flags.
    • + *
+ * + *

Having more than a single variable of sequence type in a single flag group is not + * supported. + */ + private void expandCommandLine(Multimap variables, List commandLine) { + Map variableView = new HashMap<>(); + String sequenceName = null; + for (String name : usedVariables) { + Collection value = variables.get(name); + if (value.isEmpty()) { + throw new ExpansionException("Invalid toolchain configuration: unknown variable '" + name + + "' can not be expanded."); + } else if (value.size() > 1) { + if (sequenceName != null) { + throw new ExpansionException( + "Invalid toolchain configuration: trying to expand two variable list in one " + + "flag group: '" + sequenceName + "' and '" + name + "'"); + } + sequenceName = name; + } else { + variableView.put(name, value.iterator().next()); + } + } + if (sequenceName != null) { + for (String value : variables.get(sequenceName)) { + variableView.put(sequenceName, value); + expandOnce(variableView, commandLine); + } + } else { + expandOnce(variableView, commandLine); + } + } + + /** + * Expanding all flags of this group into {@code commandLine}. + */ + private void expandOnce(Map variables, List commandLine) { + for (Flag flag : flags) { + flag.expandCommandLine(variables, commandLine); + } + } + } + + /** + * Groups a set of flags to apply for certain actions. + */ + @Immutable + private static class FlagSet implements Serializable { + private final ImmutableSet actions; + private final ImmutableList flagGroups; + + private FlagSet(CToolchain.FlagSet flagSet) throws InvalidConfigurationException { + this.actions = ImmutableSet.copyOf(flagSet.getActionList()); + ImmutableList.Builder builder = ImmutableList.builder(); + for (CToolchain.FlagGroup flagGroup : flagSet.getFlagGroupList()) { + builder.add(new FlagGroup(flagGroup)); + } + this.flagGroups = builder.build(); + } + + /** + * Adds the flags that apply to the given {@code action} to {@code commandLine}. + */ + private void expandCommandLine(String action, Multimap variables, + List commandLine) { + if (!actions.contains(action)) { + return; + } + for (FlagGroup flagGroup : flagGroups) { + flagGroup.expandCommandLine(variables, commandLine); + } + } + } + + /** + * Contains flags for a specific feature. + */ + @Immutable + private static class Feature implements Serializable { + private final String name; + private final ImmutableList flagSets; + + private Feature(CToolchain.Feature feature) throws InvalidConfigurationException { + this.name = feature.getName(); + ImmutableList.Builder builder = ImmutableList.builder(); + for (CToolchain.FlagSet flagSet : feature.getFlagSetList()) { + builder.add(new FlagSet(flagSet)); + } + this.flagSets = builder.build(); + } + + /** + * @return the features's name. + */ + private String getName() { + return name; + } + + /** + * Adds the flags that apply to the given {@code action} to {@code commandLine}. + */ + private void expandCommandLine(String action, Multimap variables, + List commandLine) { + for (FlagSet flagSet : flagSets) { + flagSet.expandCommandLine(action, variables, commandLine); + } + } + } + + /** + * Captures the set of enabled features for a rule. + */ + @Immutable + public static class FeatureConfiguration { + private final ImmutableSet enabledFeatureNames; + private final ImmutableList enabledFeatures; + + public FeatureConfiguration() { + enabledFeatureNames = ImmutableSet.of(); + enabledFeatures = ImmutableList.of(); + } + + private FeatureConfiguration(ImmutableList enabledFeatures) { + this.enabledFeatures = enabledFeatures; + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (Feature feature : enabledFeatures) { + builder.add(feature.getName()); + } + this.enabledFeatureNames = builder.build(); + } + + /** + * @return whether the given {@code feature} is enabled. + */ + boolean isEnabled(String feature) { + return enabledFeatureNames.contains(feature); + } + + /** + * @return the command line for the given {@code action}. + */ + List getCommandLine(String action, Multimap variables) { + List commandLine = new ArrayList<>(); + for (Feature feature : enabledFeatures) { + feature.expandCommandLine(action, variables, commandLine); + } + return commandLine; + } + } + + /** + * All features in the order in which they were specified in the configuration. + * + *

We guarantee the command line to be in the order in which the flags were specified in the + * configuration. + */ + private final ImmutableList features; + + /** + * Maps from the feature's name to the feature. + */ + private final ImmutableMap featuresByName; + + /** + * Maps from a feature to a set of all the features it has a direct 'implies' edge to. + */ + private final ImmutableMultimap implies; + + /** + * Maps from a feature to all features that have an direct 'implies' edge to this feature. + */ + private final ImmutableMultimap impliedBy; + + /** + * Maps from a feature to a set of feature sets, where: + *

    + *
  • a feature set satisfies the 'requires' condition, if all features in the feature set are + * enabled
    • + *
  • the 'requires' condition is satisfied, if at least one of the feature sets satisfies the + * 'requires' condition.
    • + *
+ */ + private final ImmutableMultimap> requires; + + /** + * Maps from a feature to all features that have a requirement referencing it. + * + *

This will be used to determine which features need to be re-checked after a feature was + * disabled. + */ + private final ImmutableMultimap requiredBy; + + /** + * A cache of feature selection results, so we do not recalculate the feature selection for + * all actions. + */ + private transient LoadingCache, FeatureConfiguration> + configurationCache = buildConfigurationCache(); + + /** + * Constructs the feature configuration from a {@code CToolchain} protocol buffer. + * + * @param toolchain the toolchain configuration as specified by the user. + * @throws InvalidConfigurationException if the configuration has logical errors. + */ + CcToolchainFeatures(CToolchain toolchain) throws InvalidConfigurationException { + // Build up the feature graph. + // First, we build up the map of name -> features in one pass, so that earlier features can + // reference later features in their configuration. + ImmutableList.Builder features = ImmutableList.builder(); + HashMap featuresByName = new HashMap<>(); + for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) { + Feature feature = new Feature(toolchainFeature); + features.add(feature); + if (featuresByName.put(feature.getName(), feature) != null) { + throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + + feature.getName() + "' was specified multiple times."); + } + } + this.features = features.build(); + this.featuresByName = ImmutableMap.copyOf(featuresByName); + + // Next, we build up all forward references for 'implies' and 'requires' edges. + ImmutableMultimap.Builder implies = ImmutableMultimap.builder(); + ImmutableMultimap.Builder> requires = + ImmutableMultimap.builder(); + // We also store the reverse 'implied by' and 'required by' edges during this pass. + ImmutableMultimap.Builder impliedBy = ImmutableMultimap.builder(); + ImmutableMultimap.Builder requiredBy = ImmutableMultimap.builder(); + for (CToolchain.Feature toolchainFeature : toolchain.getFeatureList()) { + String name = toolchainFeature.getName(); + Feature feature = featuresByName.get(name); + for (CToolchain.FeatureSet requiredFeatures : toolchainFeature.getRequiresList()) { + ImmutableSet.Builder allOf = ImmutableSet.builder(); + for (String requiredName : requiredFeatures.getFeatureList()) { + Feature required = getFeatureOrFail(requiredName, name); + allOf.add(required); + requiredBy.put(required, feature); + } + requires.put(feature, allOf.build()); + } + for (String impliedName : toolchainFeature.getImpliesList()) { + Feature implied = getFeatureOrFail(impliedName, name); + impliedBy.put(implied, feature); + implies.put(feature, implied); + } + } + this.implies = implies.build(); + this.requires = requires.build(); + this.impliedBy = impliedBy.build(); + this.requiredBy = requiredBy.build(); + } + + /** + * Assign an empty cache after default-deserializing all non-transient members. + */ + private void readObject(ObjectInputStream in) throws ClassNotFoundException, IOException { + in.defaultReadObject(); + this.configurationCache = buildConfigurationCache(); + } + + /** + * @return an empty {@code FeatureConfiguration} cache. + */ + private LoadingCache, FeatureConfiguration> buildConfigurationCache() { + return CacheBuilder.newBuilder() + // TODO(klimek): Benchmark and tweak once we support a larger configuration. + .maximumSize(10000) + .build(new CacheLoader, FeatureConfiguration>() { + @Override + public FeatureConfiguration load(Collection requestedFeatures) { + return computeFeatureConfiguration(requestedFeatures); + } + }); + } + + /** + * Given a list of {@code requestedFeatures}, returns all features that are enabled by the + * toolchain configuration. + * + *

A requested feature will not be enabled if the toolchain does not support it (which may + * depend on other requested features). + * + *

Additional features will be enabled if the toolchain supports them and they are implied by + * requested features. + */ + FeatureConfiguration getFeatureConfiguration(Collection requestedFeatures) { + return configurationCache.getUnchecked(requestedFeatures); + } + + private FeatureConfiguration computeFeatureConfiguration(Collection requestedFeatures) { + // Command line flags will be output in the order in which they are specified in the toolchain + // configuration. + return new FeatureSelection(requestedFeatures).run(); + } + + /** + * Convenience method taking a variadic string argument list for testing. + */ + FeatureConfiguration getFeatureConfiguration(String... requestedFeatures) { + return getFeatureConfiguration(Arrays.asList(requestedFeatures)); + } + + /** + * @return the feature with the given {@code name}. + * + * @throws InvalidConfigurationException if no feature with the given name was configured. + */ + private Feature getFeatureOrFail(String name, String reference) + throws InvalidConfigurationException { + if (!featuresByName.containsKey(name)) { + throw new InvalidConfigurationException("Invalid toolchain configuration: feature '" + name + + "', which is referenced from feature '" + reference + "', is not defined."); + } + return featuresByName.get(name); + } + + @VisibleForTesting + Collection getFeatureNames() { + Collection featureNames = new HashSet<>(); + for (Feature feature : features) { + featureNames.add(feature.getName()); + } + return featureNames; + } + + /** + * Implements the feature selection algorithm. + * + *

Feature selection is done by first enabling all features reachable by an 'implies' edge, + * and then iteratively pruning features that have unmet requirements. + */ + private class FeatureSelection { + + /** + * The features Bazel would like to enable; either because they are supported and generally + * useful, or because the user required them (for example through the command line). + */ + private final ImmutableSet requestedFeatures; + + /** + * The currently enabled feature; during feature selection, we first put all features reachable + * via an 'implies' edge into the enabled feature set, and than prune that set from features + * that have unmet requirements. + */ + private Set enabled = new HashSet<>(); + + private FeatureSelection(Collection requestedFeatures) { + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (String name : requestedFeatures) { + if (featuresByName.containsKey(name)) { + builder.add(featuresByName.get(name)); + } + } + this.requestedFeatures = builder.build(); + } + + /** + * @return all enabled features in the order in which they were specified in the configuration. + */ + private FeatureConfiguration run() { + for (Feature feature : requestedFeatures) { + enableAllImpliedBy(feature); + } + disableUnsupportedFeatures(); + ImmutableList.Builder enabledFeaturesInOrder = ImmutableList.builder(); + for (Feature feature : features) { + if (enabled.contains(feature)) { + enabledFeaturesInOrder.add(feature); + } + } + return new FeatureConfiguration(enabledFeaturesInOrder.build()); + } + + /** + * Transitively and unconditionally enable all features implied by the given feature and the + * feature itself to the enabled feature set. + */ + private void enableAllImpliedBy(Feature feature) { + if (enabled.contains(feature)) { + return; + } + enabled.add(feature); + for (Feature implied : implies.get(feature)) { + enableAllImpliedBy(implied); + } + } + + /** + * Remove all unsupported features from the enabled feature set. + */ + private void disableUnsupportedFeatures() { + Queue check = new ArrayDeque<>(enabled); + while (!check.isEmpty()) { + checkFeature(check.poll()); + } + } + + /** + * Check if the given feature is still satisfied within the set of currently enabled features. + * + *

If it is not, remove the feature from the set of enabled features, and re-check all + * features that may now also become disabled. + */ + private void checkFeature(Feature feature) { + if (!enabled.contains(feature) || isSatisfied(feature)) { + return; + } + enabled.remove(feature); + + // Once we disable a feature, we have to re-check all features that can be affected by + // that removal. + // 1. A feature that implied the current feature is now going to be disabled. + for (Feature impliesCurrent : impliedBy.get(feature)) { + checkFeature(impliesCurrent); + } + // 2. A feature that required the current feature may now be disabled, depending on whether + // the requirement was optional. + for (Feature requiresCurrent : requiredBy.get(feature)) { + checkFeature(requiresCurrent); + } + // 3. A feature that this feature implied may now be disabled if no other feature also implies + // it. + for (Feature implied : implies.get(feature)) { + checkFeature(implied); + } + } + + /** + * @return whether all requirements of the feature are met in the set of currently enabled + * features. + */ + private boolean isSatisfied(Feature feature) { + return (requestedFeatures.contains(feature) || isImpliedByEnabledFeature(feature)) + && allImplicationsEnabled(feature) && allRequirementsMet(feature); + } + + /** + * @return whether a currently enabled feature implies the given feature. + */ + private boolean isImpliedByEnabledFeature(Feature feature) { + for (Feature implies : impliedBy.get(feature)) { + if (enabled.contains(implies)) { + return true; + } + } + return false; + } + + /** + * @return whether all implications of the given feature are enabled. + */ + private boolean allImplicationsEnabled(Feature feature) { + for (Feature implied : implies.get(feature)) { + if (!enabled.contains(implied)) { + return false; + } + } + return true; + } + + /** + * @return whether all requirements are enabled. + * + *

This implies that for any of the feature sets all of the specified features are enabled. + */ + private boolean allRequirementsMet(Feature feature) { + if (!requires.containsKey(feature)) { + return true; + } + for (ImmutableSet requiresAllOf : requires.get(feature)) { + boolean requirementMet = true; + for (Feature required : requiresAllOf) { + if (!enabled.contains(required)) { + requirementMet = false; + break; + } + } + if (requirementMet) { + return true; + } + } + return false; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java new file mode 100644 index 0000000000..e1940a5d1f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProvider.java @@ -0,0 +1,226 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.vfs.PathFragment; + +import javax.annotation.Nullable; + +/** + * Information about a C++ compiler used by the cc_* rules. + */ +@Immutable +public final class CcToolchainProvider implements TransitiveInfoProvider { + /** + * An empty toolchain to be returned in the error case (instead of null). + */ + public static final CcToolchainProvider EMPTY_TOOLCHAIN_IS_ERROR = new CcToolchainProvider( + null, + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + null, + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + null, + PathFragment.EMPTY_FRAGMENT, + CppCompilationContext.EMPTY, + false, + false); + + @Nullable private final CppConfiguration cppConfiguration; + private final NestedSet crosstool; + private final NestedSet crosstoolMiddleman; + private final NestedSet compile; + private final NestedSet strip; + private final NestedSet objCopy; + private final NestedSet link; + private final NestedSet dwp; + private final NestedSet libcLink; + private final NestedSet staticRuntimeLinkInputs; + @Nullable private final Artifact staticRuntimeLinkMiddleman; + private final NestedSet dynamicRuntimeLinkInputs; + @Nullable private final Artifact dynamicRuntimeLinkMiddleman; + private final PathFragment dynamicRuntimeSolibDir; + private final CppCompilationContext cppCompilationContext; + private final boolean supportsParamFiles; + private final boolean supportsHeaderParsing; + + public CcToolchainProvider( + @Nullable CppConfiguration cppConfiguration, + NestedSet crosstool, + NestedSet crosstoolMiddleman, + NestedSet compile, + NestedSet strip, + NestedSet objCopy, + NestedSet link, + NestedSet dwp, + NestedSet libcLink, + NestedSet staticRuntimeLinkInputs, + @Nullable Artifact staticRuntimeLinkMiddleman, + NestedSet dynamicRuntimeLinkInputs, + @Nullable Artifact dynamicRuntimeLinkMiddleman, + PathFragment dynamicRuntimeSolibDir, + CppCompilationContext cppCompilationContext, + boolean supportsParamFiles, + boolean supportsHeaderParsing) { + this.cppConfiguration = cppConfiguration; + this.crosstool = Preconditions.checkNotNull(crosstool); + this.crosstoolMiddleman = Preconditions.checkNotNull(crosstoolMiddleman); + this.compile = Preconditions.checkNotNull(compile); + this.strip = Preconditions.checkNotNull(strip); + this.objCopy = Preconditions.checkNotNull(objCopy); + this.link = Preconditions.checkNotNull(link); + this.dwp = Preconditions.checkNotNull(dwp); + this.libcLink = Preconditions.checkNotNull(libcLink); + this.staticRuntimeLinkInputs = Preconditions.checkNotNull(staticRuntimeLinkInputs); + this.staticRuntimeLinkMiddleman = staticRuntimeLinkMiddleman; + this.dynamicRuntimeLinkInputs = Preconditions.checkNotNull(dynamicRuntimeLinkInputs); + this.dynamicRuntimeLinkMiddleman = dynamicRuntimeLinkMiddleman; + this.dynamicRuntimeSolibDir = Preconditions.checkNotNull(dynamicRuntimeSolibDir); + this.cppCompilationContext = Preconditions.checkNotNull(cppCompilationContext); + this.supportsParamFiles = supportsParamFiles; + this.supportsHeaderParsing = supportsHeaderParsing; + } + + /** + * Returns all the files in Crosstool. Is not a middleman. + */ + public NestedSet getCrosstool() { + return crosstool; + } + + /** + * Returns a middleman for all the files in Crosstool. + */ + public NestedSet getCrosstoolMiddleman() { + return crosstoolMiddleman; + } + + /** + * Returns the files necessary for compilation. + */ + public NestedSet getCompile() { + // If include scanning is disabled, we need the entire crosstool filegroup, including header + // files. If it is enabled, we use the filegroup without header files - they are found by + // include scanning. For go, we also don't need the header files. + return cppConfiguration != null && cppConfiguration.shouldScanIncludes() ? compile : crosstool; + } + + /** + * Returns the files necessary for a 'strip' invocation. + */ + public NestedSet getStrip() { + return strip; + } + + /** + * Returns the files necessary for an 'objcopy' invocation. + */ + public NestedSet getObjcopy() { + return objCopy; + } + + /** + * Returns the files necessary for linking, including the files needed for libc. + */ + public NestedSet getLink() { + return link; + } + + public NestedSet getDwp() { + return dwp; + } + + public NestedSet getLibcLink() { + return libcLink; + } + + /** + * Returns the static runtime libraries. + */ + public NestedSet getStaticRuntimeLinkInputs() { + return staticRuntimeLinkInputs; + } + + /** + * Returns an aggregating middleman that represents the static runtime libraries. + */ + @Nullable public Artifact getStaticRuntimeLinkMiddleman() { + return staticRuntimeLinkMiddleman; + } + + /** + * Returns the dynamic runtime libraries. + */ + public NestedSet getDynamicRuntimeLinkInputs() { + return dynamicRuntimeLinkInputs; + } + + /** + * Returns an aggregating middleman that represents the dynamic runtime libraries. + */ + @Nullable public Artifact getDynamicRuntimeLinkMiddleman() { + return dynamicRuntimeLinkMiddleman; + } + + /** + * Returns the name of the directory where the solib symlinks for the dynamic runtime libraries + * live. The directory itself will be under the root of the host configuration in the 'bin' + * directory. + */ + public PathFragment getDynamicRuntimeSolibDir() { + return dynamicRuntimeSolibDir; + } + + /** + * Returns the C++ compilation context for the toolchain. + */ + public CppCompilationContext getCppCompilationContext() { + return cppCompilationContext; + } + + /** + * Whether the toolchains supports parameter files. + */ + public boolean supportsParamFiles() { + return supportsParamFiles; + } + + /** + * Whether the toolchains supports header parsing. + */ + public boolean supportsHeaderParsing() { + return supportsHeaderParsing; + } + + /** + * Returns the configured features of the toolchain. + */ + public CcToolchainFeatures getFeatures() { + return cppConfiguration.getFeatures(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java new file mode 100644 index 0000000000..6c68f00bab --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java @@ -0,0 +1,71 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import static com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition.HOST; +import static com.google.devtools.build.lib.packages.Attribute.attr; +import static com.google.devtools.build.lib.packages.Type.BOOLEAN; +import static com.google.devtools.build.lib.packages.Type.LABEL; +import static com.google.devtools.build.lib.packages.Type.LABEL_LIST; +import static com.google.devtools.build.lib.packages.Type.LICENSE; +import static com.google.devtools.build.lib.packages.Type.STRING; + +import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.BlazeRule; +import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Attribute.LateBoundLabel; +import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.packages.RuleClass; +import com.google.devtools.build.lib.packages.RuleClass.Builder; +import com.google.devtools.build.lib.syntax.Label; + +/** + * Rule definition for compiler definition. + */ +@BlazeRule(name = "cc_toolchain", + ancestors = { BaseRuleClasses.BaseRule.class }, + factoryClass = CcToolchain.class) +public final class CcToolchainRule implements RuleDefinition { + private static final LateBoundLabel LIBC_LINK = + new LateBoundLabel() { + @Override + public Label getDefault(Rule rule, BuildConfiguration configuration) { + return configuration.getFragment(CppConfiguration.class).getLibcLabel(); + } + }; + + @Override + public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + return builder + .setUndocumented() + .add(attr("output_licenses", LICENSE)) + .add(attr("cpu", STRING).mandatory()) + .add(attr("all_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("compiler_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("strip_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("objcopy_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("linker_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("dwp_files", LABEL).legacyAllowAnyFileType().cfg(HOST).mandatory()) + .add(attr("static_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory()) + .add(attr("dynamic_runtime_libs", LABEL_LIST).legacyAllowAnyFileType().mandatory()) + .add(attr("module_map", LABEL).legacyAllowAnyFileType().cfg(HOST)) + .add(attr("supports_param_files", BOOLEAN).value(true)) + .add(attr("supports_header_parsing", BOOLEAN).value(false)) + // TODO(bazel-team): Should be using the TARGET configuration. + .add(attr(":libc_link", LABEL).cfg(HOST).value(LIBC_LINK)) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java new file mode 100644 index 0000000000..78a5f89700 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppBuildInfo.java @@ -0,0 +1,89 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoCollection; +import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * C++ build info creation - generates header files that contain the corresponding build-info data. + */ +public final class CppBuildInfo implements BuildInfoFactory { + public static final BuildInfoKey KEY = new BuildInfoKey("C++"); + + private static final PathFragment BUILD_INFO_NONVOLATILE_HEADER_NAME = + new PathFragment("build-info-nonvolatile.h"); + private static final PathFragment BUILD_INFO_VOLATILE_HEADER_NAME = + new PathFragment("build-info-volatile.h"); + // TODO(bazel-team): (2011) Get rid of the redacted build info. We should try to make + // the linkstamping process handle the case where those values are undefined. + private static final PathFragment BUILD_INFO_REDACTED_HEADER_NAME = + new PathFragment("build-info-redacted.h"); + + @Override + public BuildInfoCollection create(BuildInfoContext buildInfoContext, BuildConfiguration config, + Artifact buildInfo, Artifact buildChangelist) { + List actions = new ArrayList<>(); + WriteBuildInfoHeaderAction redactedInfo = getHeader(buildInfoContext, config, + BUILD_INFO_REDACTED_HEADER_NAME, + Artifact.NO_ARTIFACTS, true, true); + WriteBuildInfoHeaderAction nonvolatileInfo = getHeader(buildInfoContext, config, + BUILD_INFO_NONVOLATILE_HEADER_NAME, + ImmutableList.of(buildInfo), + false, true); + WriteBuildInfoHeaderAction volatileInfo = getHeader(buildInfoContext, config, + BUILD_INFO_VOLATILE_HEADER_NAME, + ImmutableList.of(buildChangelist), + true, false); + actions.add(redactedInfo); + actions.add(nonvolatileInfo); + actions.add(volatileInfo); + return new BuildInfoCollection(actions, + ImmutableList.of(nonvolatileInfo.getPrimaryOutput(), volatileInfo.getPrimaryOutput()), + ImmutableList.of(redactedInfo.getPrimaryOutput())); + } + + private WriteBuildInfoHeaderAction getHeader(BuildInfoContext buildInfoContext, + BuildConfiguration config, PathFragment headerName, + Collection inputs, + boolean writeVolatileInfo, boolean writeNonVolatileInfo) { + Root outputPath = config.getIncludeDirectory(); + final Artifact header = + buildInfoContext.getBuildInfoArtifact(headerName, outputPath, + writeVolatileInfo && !inputs.isEmpty() + ? BuildInfoType.NO_REBUILD : BuildInfoType.FORCE_REBUILD_IF_CHANGED); + return new WriteBuildInfoHeaderAction( + inputs, header, writeVolatileInfo, writeNonVolatileInfo); + } + + @Override + public BuildInfoKey getKey() { + return KEY; + } + + @Override + public boolean isEnabled(BuildConfiguration config) { + return config.hasFragment(CppConfiguration.class); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java new file mode 100644 index 0000000000..cf39ef57c5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompilationContext.java @@ -0,0 +1,918 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.MiddlemanFactory; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable store of information needed for C++ compilation that is aggregated + * across dependencies. + */ +@Immutable +public final class CppCompilationContext implements TransitiveInfoProvider { + /** An empty compilation context. */ + public static final CppCompilationContext EMPTY = new Builder(null).build(); + + private final CommandLineContext commandLineContext; + private final ImmutableList depsContexts; + private final CppModuleMap cppModuleMap; + private final Artifact headerModule; + private final Artifact picHeaderModule; + private final ImmutableSet compilationPrerequisites; + + private CppCompilationContext(CommandLineContext commandLineContext, + List depsContexts, CppModuleMap cppModuleMap, Artifact headerModule, + Artifact picHeaderModule) { + Preconditions.checkNotNull(commandLineContext); + Preconditions.checkArgument(!depsContexts.isEmpty()); + this.commandLineContext = commandLineContext; + this.depsContexts = ImmutableList.copyOf(depsContexts); + this.cppModuleMap = cppModuleMap; + this.headerModule = headerModule; + this.picHeaderModule = picHeaderModule; + + if (depsContexts.size() == 1) { + // Only LIPO targets have more than one DepsContexts. This codepath avoids creating + // an ImmutableSet.Builder for the vast majority of the cases. + compilationPrerequisites = (depsContexts.get(0).compilationPrerequisiteStampFile != null) + ? ImmutableSet.of(depsContexts.get(0).compilationPrerequisiteStampFile) + : ImmutableSet.of(); + } else { + ImmutableSet.Builder prerequisites = ImmutableSet.builder(); + for (DepsContext depsContext : depsContexts) { + if (depsContext.compilationPrerequisiteStampFile != null) { + prerequisites.add(depsContext.compilationPrerequisiteStampFile); + } + } + compilationPrerequisites = prerequisites.build(); + } + } + + /** + * Returns the compilation prerequisites consolidated into middlemen + * prerequisites, or an empty set if there are no prerequisites. + * + *

For correct dependency tracking, and to reduce the overhead to establish + * dependencies on generated headers, we express the dependency on compilation + * prerequisites as a transitive dependency via a middleman. After they have + * been accumulated (using + * {@link Builder#addCompilationPrerequisites(Iterable)}, + * {@link Builder#mergeDependentContext(CppCompilationContext)}, and + * {@link Builder#mergeDependentContexts(Iterable)}, they are consolidated + * into a single middleman Artifact when {@link Builder#build()} is called. + * + *

The returned set can be empty if there are no prerequisites. Usually it + * contains a single middleman, but if LIPO is used there can be two. + */ + public ImmutableSet getCompilationPrerequisites() { + return compilationPrerequisites; + } + + /** + * Returns the immutable list of include directories to be added with "-I" + * (possibly empty but never null). This includes the include dirs from the + * transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList getIncludeDirs() { + return commandLineContext.includeDirs; + } + + /** + * Returns the immutable list of include directories to be added with + * "-iquote" (possibly empty but never null). This includes the include dirs + * from the transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList getQuoteIncludeDirs() { + return commandLineContext.quoteIncludeDirs; + } + + /** + * Returns the immutable list of include directories to be added with + * "-isystem" (possibly empty but never null). This includes the include dirs + * from the transitive deps closure of the target. This list does not contain + * duplicates. All fragments are either absolute or relative to the exec root + * (see {@link BuildConfiguration#getExecRoot}). + */ + public ImmutableList getSystemIncludeDirs() { + return commandLineContext.systemIncludeDirs; + } + + /** + * Returns the immutable set of declared include directories, relative to a + * "-I" or "-iquote" directory" (possibly empty but never null). The returned + * collection may contain duplicate elements. + * + *

Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet getDeclaredIncludeDirs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeDirs; + } + + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeDirs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of include directories, relative to a "-I" or + * "-iquote" directory", from which inclusion will produce a warning (possibly + * empty but never null). The returned collection may contain duplicate + * elements. + * + *

Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet getDeclaredIncludeWarnDirs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeWarnDirs; + } + + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeWarnDirs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of headers that have been declared in the + * {@code src} or {@code headers attribute} (possibly empty but never null). + * The returned collection may contain duplicate elements. + * + *

Note: The iteration order of this list is preserved as ide_build_info + * writes these directories and sources out and the ordering will help when + * used by consumers. + */ + public NestedSet getDeclaredIncludeSrcs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).declaredIncludeSrcs; + } + + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.declaredIncludeSrcs); + } + + return builder.build(); + } + + /** + * Returns the immutable pairs of (header file, pregrepped header file). + */ + public NestedSet> getPregreppedHeaders() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).pregreppedHdrs; + } + + NestedSetBuilder> builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.pregreppedHdrs); + } + + return builder.build(); + } + + /** + * Returns the immutable set of additional transitive inputs needed for + * compilation, like C++ module map artifacts. + */ + public NestedSet getAdditionalInputs() { + if (depsContexts.isEmpty()) { + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); + } + + if (depsContexts.size() == 1) { + return depsContexts.get(0).auxiliaryInputs; + } + + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.auxiliaryInputs); + } + + return builder.build(); + } + + /** + * Returns optional inputs that are needed by any C++ compilations that use header modules. + * + *

For every target that the current target depends on transitively and that is built as header + * module, contains: + *

    + *
  • the pic/non-pic header module (pcm file)
    • + *
  • the transitive list of module maps.
    • + *
+ */ + private NestedSet getTransitiveAuxiliaryInputs() { + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveAuxiliaryInputs); + } + return builder.build(); + } + + /** + * @return all modules maps in the transitive closure. + */ + private NestedSet getTransitiveModuleMaps() { + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveModuleMaps); + } + return builder.build(); + } + + /** + * @return all headers whose transitive closure of includes needs to be + * available when compiling anything in the current target. + */ + protected NestedSet getTransitiveHeaderModuleSrcs() { + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.transitiveHeaderModuleSrcs); + } + return builder.build(); + } + + /** + * @return all declared headers of the current module if the current target + * is compiled as a module. + */ + protected NestedSet getHeaderModuleSrcs() { + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + for (DepsContext depsContext : depsContexts) { + builder.addTransitive(depsContext.headerModuleSrcs); + } + return builder.build(); + } + + /** + * Returns the set of defines needed to compile this target (possibly empty + * but never null). This includes definitions from the transitive deps closure + * for the target. The order of the returned collection is deterministic. + */ + public ImmutableList getDefines() { + return commandLineContext.defines; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CppCompilationContext)) { + return false; + } + CppCompilationContext other = (CppCompilationContext) obj; + return Objects.equals(headerModule, other.headerModule) + && Objects.equals(picHeaderModule, other.picHeaderModule) + && commandLineContext.equals(other.commandLineContext) + && depsContexts.equals(other.depsContexts); + } + + @Override + public int hashCode() { + return Objects.hash(headerModule, picHeaderModule, commandLineContext, depsContexts); + } + + /** + * Returns a context that is based on a given context but returns empty sets + * for {@link #getDeclaredIncludeDirs()} and {@link #getDeclaredIncludeWarnDirs()}. + */ + public static CppCompilationContext disallowUndeclaredHeaders(CppCompilationContext context) { + ImmutableList.Builder builder = ImmutableList.builder(); + for (DepsContext depsContext : context.depsContexts) { + builder.add(new DepsContext( + depsContext.compilationPrerequisiteStampFile, + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + NestedSetBuilder.emptySet(Order.STABLE_ORDER), + depsContext.declaredIncludeSrcs, + depsContext.pregreppedHdrs, + depsContext.auxiliaryInputs, + depsContext.headerModuleSrcs, + depsContext.transitiveAuxiliaryInputs, + depsContext.transitiveHeaderModuleSrcs, + depsContext.transitiveModuleMaps)); + } + return new CppCompilationContext(context.commandLineContext, builder.build(), + context.cppModuleMap, context.headerModule, context.picHeaderModule); + } + + /** + * Returns the context for a LIPO compile action. This uses the include dirs + * and defines of the library, but the declared inclusion dirs/srcs from both + * the library and the owner binary. + + * TODO(bazel-team): this might make every LIPO target have an unnecessary large set of + * inclusion dirs/srcs. The correct behavior would be to merge only the contexts + * of actual referred targets (as listed in .imports file). + * + *

Undeclared inclusion checking ({@link #getDeclaredIncludeDirs()}, + * {@link #getDeclaredIncludeWarnDirs()}, and + * {@link #getDeclaredIncludeSrcs()}) needs to use the union of the contexts + * of the involved source files. + * + *

For include and define command line flags ({@link #getIncludeDirs()} + * {@link #getQuoteIncludeDirs()}, {@link #getSystemIncludeDirs()}, and + * {@link #getDefines()}) LIPO compilations use the same values as non-LIPO + * compilation. + * + *

Include scanning is not handled by this method. See + * {@code IncludeScannable#getAuxiliaryScannables()} instead. + * + * @param ownerContext the compilation context of the owner binary + * @param libContext the compilation context of the library + */ + public static CppCompilationContext mergeForLipo(CppCompilationContext ownerContext, + CppCompilationContext libContext) { + return new CppCompilationContext(libContext.commandLineContext, + ImmutableList.copyOf(Iterables.concat(ownerContext.depsContexts, libContext.depsContexts)), + libContext.cppModuleMap, libContext.headerModule, libContext.picHeaderModule); + } + + /** + * @return the C++ module map of the owner. + */ + public CppModuleMap getCppModuleMap() { + return cppModuleMap; + } + + /** + * @return the non-pic C++ header module of the owner. + */ + private Artifact getHeaderModule() { + return headerModule; + } + + /** + * @return the pic C++ header module of the owner. + */ + private Artifact getPicHeaderModule() { + return picHeaderModule; + } + + /** + * The parts of the compilation context that influence the command line of + * compilation actions. + */ + @Immutable + private static class CommandLineContext { + private final ImmutableList includeDirs; + private final ImmutableList quoteIncludeDirs; + private final ImmutableList systemIncludeDirs; + private final ImmutableList defines; + + CommandLineContext(ImmutableList includeDirs, + ImmutableList quoteIncludeDirs, + ImmutableList systemIncludeDirs, + ImmutableList defines) { + this.includeDirs = includeDirs; + this.quoteIncludeDirs = quoteIncludeDirs; + this.systemIncludeDirs = systemIncludeDirs; + this.defines = defines; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CommandLineContext)) { + return false; + } + CommandLineContext other = (CommandLineContext) obj; + return Objects.equals(includeDirs, other.includeDirs) + && Objects.equals(quoteIncludeDirs, other.quoteIncludeDirs) + && Objects.equals(systemIncludeDirs, other.systemIncludeDirs) + && Objects.equals(defines, other.defines); + } + + @Override + public int hashCode() { + return Objects.hash(includeDirs, quoteIncludeDirs, systemIncludeDirs, defines); + } + } + + /** + * The parts of the compilation context that defined the dependencies of + * actions of scheduling and inclusion validity checking. + */ + @Immutable + private static class DepsContext { + private final Artifact compilationPrerequisiteStampFile; + private final NestedSet declaredIncludeDirs; + private final NestedSet declaredIncludeWarnDirs; + private final NestedSet declaredIncludeSrcs; + private final NestedSet> pregreppedHdrs; + + /** + * Optional inputs that are used by some forms of compilation, containing: + *

    + *
  • module map of the current target
    • + *
  • module maps of all direct dependencies that are not compiled as header modules
    • + *
  • all transitiveAuxiliaryInputs.
    • + *
+ */ + private final NestedSet auxiliaryInputs; + + /** + * All declared headers of the current module, if compiled as a header module. + */ + private final NestedSet headerModuleSrcs; + + private final NestedSet transitiveAuxiliaryInputs; + + /** + * Headers whose transitive closure of includes needs to be available when compiling the current + * target. For every target that the current target depends on transitively and that is built as + * header module, contains all headers that are part of its header module. + */ + private final NestedSet transitiveHeaderModuleSrcs; + + /** + * The module maps from all targets the current target depends on transitively. + */ + private final NestedSet transitiveModuleMaps; + + DepsContext(Artifact compilationPrerequisiteStampFile, + NestedSet declaredIncludeDirs, + NestedSet declaredIncludeWarnDirs, + NestedSet declaredIncludeSrcs, + NestedSet> pregreppedHdrs, + NestedSet auxiliaryInputs, + NestedSet headerModuleSrcs, + NestedSet transitiveAuxiliaryInputs, + NestedSet transitiveHeaderModuleSrcs, + NestedSet transitiveModuleMaps) { + this.compilationPrerequisiteStampFile = compilationPrerequisiteStampFile; + this.declaredIncludeDirs = declaredIncludeDirs; + this.declaredIncludeWarnDirs = declaredIncludeWarnDirs; + this.declaredIncludeSrcs = declaredIncludeSrcs; + this.pregreppedHdrs = pregreppedHdrs; + this.auxiliaryInputs = auxiliaryInputs; + this.headerModuleSrcs = headerModuleSrcs; + this.transitiveAuxiliaryInputs = transitiveAuxiliaryInputs; + this.transitiveHeaderModuleSrcs = transitiveHeaderModuleSrcs; + this.transitiveModuleMaps = transitiveModuleMaps; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof DepsContext)) { + return false; + } + DepsContext other = (DepsContext) obj; + return Objects.equals( + compilationPrerequisiteStampFile, other.compilationPrerequisiteStampFile) + && Objects.equals(declaredIncludeDirs, other.declaredIncludeDirs) + && Objects.equals(declaredIncludeWarnDirs, other.declaredIncludeWarnDirs) + && Objects.equals(declaredIncludeSrcs, other.declaredIncludeSrcs) + && Objects.equals(auxiliaryInputs, other.auxiliaryInputs) + && Objects.equals(headerModuleSrcs, other.headerModuleSrcs) + // Due to the NestedSet equals being ==, and the code flow only setting them if at least + // auxiliaryInputs is set, these checks cannot be executed. We leave them in so the equals + // is still correct if that connection ever changes.R + && Objects.equals(transitiveAuxiliaryInputs, other.transitiveAuxiliaryInputs) + && Objects.equals(transitiveHeaderModuleSrcs, other.transitiveHeaderModuleSrcs) + && Objects.equals(transitiveModuleMaps, other.transitiveModuleMaps) + ; + } + + @Override + public int hashCode() { + return Objects.hash(compilationPrerequisiteStampFile, + declaredIncludeDirs, + declaredIncludeWarnDirs, + declaredIncludeSrcs, + auxiliaryInputs, + headerModuleSrcs, + transitiveAuxiliaryInputs, + transitiveHeaderModuleSrcs, + transitiveModuleMaps); + } + } + + /** + * Builder class for {@link CppCompilationContext}. + */ + public static class Builder { + private String purpose = "cpp_compilation_prerequisites"; + private final Set compilationPrerequisites = new LinkedHashSet<>(); + private final Set includeDirs = new LinkedHashSet<>(); + private final Set quoteIncludeDirs = new LinkedHashSet<>(); + private final Set systemIncludeDirs = new LinkedHashSet<>(); + private final NestedSetBuilder declaredIncludeDirs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder declaredIncludeWarnDirs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder declaredIncludeSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder> pregreppedHdrs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder auxiliaryInputs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder headerModuleSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder transitiveAuxiliaryInputs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder transitiveHeaderModuleSrcs = + NestedSetBuilder.stableOrder(); + private final NestedSetBuilder transitiveModuleMaps = + NestedSetBuilder.stableOrder(); + private final Set defines = new LinkedHashSet<>(); + private CppModuleMap cppModuleMap; + private Artifact headerModule; + private Artifact picHeaderModule; + + /** The rule that owns the context */ + private final RuleContext ruleContext; + + /** + * Creates a new builder for a {@link CppCompilationContext} instance. + */ + public Builder(RuleContext ruleContext) { + this.ruleContext = ruleContext; + } + + /** + * Overrides the purpose of this context. This is useful if a Target + * needs more than one CppCompilationContext. (The purpose is used to + * construct the name of the prerequisites middleman for the context, and + * all artifacts for a given Target must have distinct names.) + * + * @param purpose must be a string which is suitable for use as a filename. + * A single rule may have many middlemen with distinct purposes. + * + * @see MiddlemanFactory#createErrorPropagatingMiddleman + */ + public Builder setPurpose(String purpose) { + this.purpose = purpose; + return this; + } + + public String getPurpose() { + return purpose; + } + + /** + * Merges the context of a dependency into this one by adding the contents + * of all of its attributes. + */ + public Builder mergeDependentContext(CppCompilationContext otherContext) { + Preconditions.checkNotNull(otherContext); + compilationPrerequisites.addAll(otherContext.getCompilationPrerequisites()); + includeDirs.addAll(otherContext.getIncludeDirs()); + quoteIncludeDirs.addAll(otherContext.getQuoteIncludeDirs()); + systemIncludeDirs.addAll(otherContext.getSystemIncludeDirs()); + declaredIncludeDirs.addTransitive(otherContext.getDeclaredIncludeDirs()); + declaredIncludeWarnDirs.addTransitive(otherContext.getDeclaredIncludeWarnDirs()); + declaredIncludeSrcs.addTransitive(otherContext.getDeclaredIncludeSrcs()); + pregreppedHdrs.addTransitive(otherContext.getPregreppedHeaders()); + + // Forward transitive information. + transitiveAuxiliaryInputs.addTransitive(otherContext.getTransitiveAuxiliaryInputs()); + transitiveModuleMaps.addTransitive(otherContext.getTransitiveModuleMaps()); + transitiveHeaderModuleSrcs.addTransitive(otherContext.getTransitiveHeaderModuleSrcs()); + + // All module maps of direct dependencies are inputs to the current compile independently of + // the build type. + if (otherContext.getCppModuleMap() != null) { + auxiliaryInputs.add(otherContext.getCppModuleMap().getArtifact()); + } + if (otherContext.getHeaderModule() != null || otherContext.getPicHeaderModule() != null) { + // If we depend directly on a target that has a compiled header module, all targets + // transitively depending on us will need that header module, and all transitive module + // maps. + if (otherContext.getHeaderModule() != null) { + transitiveAuxiliaryInputs.add(otherContext.getHeaderModule()); + } + if (otherContext.getPicHeaderModule() != null) { + transitiveAuxiliaryInputs.add(otherContext.getPicHeaderModule()); + } + transitiveAuxiliaryInputs.addAll(otherContext.getTransitiveModuleMaps()); + + // All targets transitively depending on us will need to have the full transitive #include + // closure of the headers in that module available. + transitiveHeaderModuleSrcs.addAll(otherContext.getHeaderModuleSrcs()); + } + // All compile actions in the current target will need the transitive inputs. + auxiliaryInputs.addAll(transitiveAuxiliaryInputs.build().toCollection()); + + defines.addAll(otherContext.getDefines()); + return this; + } + + /** + * Merges the context of some targets into this one by adding the contents + * of all of their attributes. Targets that do not implement + * {@link CppCompilationContext} are ignored. + */ + public Builder mergeDependentContexts(Iterable targets) { + for (CppCompilationContext target : targets) { + mergeDependentContext(target); + } + return this; + } + + /** + * Adds multiple compilation prerequisites. + */ + public Builder addCompilationPrerequisites(Iterable prerequisites) { + // LIPO collector must not add compilation prerequisites in order to avoid + // the creation of a middleman action. + Iterables.addAll(compilationPrerequisites, prerequisites); + return this; + } + + /** + * Add a single include directory to be added with "-I". It can be either + * relative to the exec root (see {@link BuildConfiguration#getExecRoot}) or + * absolute. Before it is stored, the include directory is normalized. + */ + public Builder addIncludeDir(PathFragment includeDir) { + includeDirs.add(includeDir.normalize()); + return this; + } + + /** + * Add multiple include directories to be added with "-I". These can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. The entries are normalized + * before they are stored. + */ + public Builder addIncludeDirs(Iterable includeDirs) { + for (PathFragment includeDir : includeDirs) { + addIncludeDir(includeDir); + } + return this; + } + + /** + * Add a single include directory to be added with "-iquote". It can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the + * include directory is normalized. + */ + public Builder addQuoteIncludeDir(PathFragment quoteIncludeDir) { + quoteIncludeDirs.add(quoteIncludeDir.normalize()); + return this; + } + + /** + * Add a single include directory to be added with "-isystem". It can be + * either relative to the exec root (see {@link + * BuildConfiguration#getExecRoot}) or absolute. Before it is stored, the + * include directory is normalized. + */ + public Builder addSystemIncludeDir(PathFragment systemIncludeDir) { + systemIncludeDirs.add(systemIncludeDir.normalize()); + return this; + } + + /** + * Add a single declared include dir, relative to a "-I" or "-iquote" + * directory". + */ + public Builder addDeclaredIncludeDir(PathFragment dir) { + declaredIncludeDirs.add(dir); + return this; + } + + /** + * Add a single declared include directory, relative to a "-I" or "-iquote" + * directory", from which inclusion will produce a warning. + */ + public Builder addDeclaredIncludeWarnDir(PathFragment dir) { + declaredIncludeWarnDirs.add(dir); + return this; + } + + /** + * Adds a header that has been declared in the {@code src} or {@code headers attribute}. The + * header will also be added to the compilation prerequisites. + */ + public Builder addDeclaredIncludeSrc(Artifact header) { + declaredIncludeSrcs.add(header); + compilationPrerequisites.add(header); + headerModuleSrcs.add(header); + return this; + } + + /** + * Adds multiple headers that have been declared in the {@code src} or {@code headers + * attribute}. The headers will also be added to the compilation prerequisites. + */ + public Builder addDeclaredIncludeSrcs(Iterable declaredIncludeSrcs) { + this.declaredIncludeSrcs.addAll(declaredIncludeSrcs); + this.headerModuleSrcs.addAll(declaredIncludeSrcs); + return addCompilationPrerequisites(declaredIncludeSrcs); + } + + /** + * Add a map of generated source or header Artifact to an output Artifact after grepping + * the file for include statements. + */ + public Builder addPregreppedHeaderMap(Map pregrepped) { + addCompilationPrerequisites(pregrepped.values()); + for (Map.Entry entry : pregrepped.entrySet()) { + this.pregreppedHdrs.add(Pair.of(entry.getKey(), entry.getValue())); + } + return this; + } + + /** + * Adds a single define. + */ + public Builder addDefine(String define) { + defines.add(define); + return this; + } + + /** + * Adds multiple defines. + */ + public Builder addDefines(Iterable defines) { + Iterables.addAll(this.defines, defines); + return this; + } + + /** + * Sets the C++ module map. + */ + public Builder setCppModuleMap(CppModuleMap cppModuleMap) { + this.cppModuleMap = cppModuleMap; + return this; + } + + /** + * Sets the C++ header module in non-pic mode. + */ + public Builder setHeaderModule(Artifact headerModule) { + this.headerModule = headerModule; + return this; + } + + /** + * Sets the C++ header module in pic mode. + */ + public Builder setPicHeaderModule(Artifact picHeaderModule) { + this.picHeaderModule = picHeaderModule; + return this; + } + + /** + * Builds the {@link CppCompilationContext}. + */ + public CppCompilationContext build() { + return build( + ruleContext == null ? null : ruleContext.getActionOwner(), + ruleContext == null ? null : ruleContext.getAnalysisEnvironment().getMiddlemanFactory()); + } + + @VisibleForTesting // productionVisibility = Visibility.PRIVATE + public CppCompilationContext build(ActionOwner owner, MiddlemanFactory middlemanFactory) { + if (cppModuleMap != null) { + // .cppmap files should also be mandatory inputs for compile actions + auxiliaryInputs.add(cppModuleMap.getArtifact()); + transitiveModuleMaps.add(cppModuleMap.getArtifact()); + } + + // We don't create middlemen in LIPO collector subtree, because some target CT + // will do that instead. + Artifact prerequisiteStampFile = (ruleContext != null + && ruleContext.getFragment(CppConfiguration.class).isLipoContextCollector()) + ? getMiddlemanArtifact(middlemanFactory) + : createMiddleman(owner, middlemanFactory); + + return new CppCompilationContext( + new CommandLineContext(ImmutableList.copyOf(includeDirs), + ImmutableList.copyOf(quoteIncludeDirs), ImmutableList.copyOf(systemIncludeDirs), + ImmutableList.copyOf(defines)), + ImmutableList.of(new DepsContext(prerequisiteStampFile, + declaredIncludeDirs.build(), + declaredIncludeWarnDirs.build(), + declaredIncludeSrcs.build(), + pregreppedHdrs.build(), + auxiliaryInputs.build(), + headerModuleSrcs.build(), + transitiveAuxiliaryInputs.build(), + transitiveHeaderModuleSrcs.build(), + transitiveModuleMaps.build())), + cppModuleMap, + headerModule, + picHeaderModule); + } + + /** + * Creates a middleman for the compilation prerequisites. + * + * @return the middleman or null if there are no prerequisites + */ + private Artifact createMiddleman(ActionOwner owner, + MiddlemanFactory middlemanFactory) { + if (compilationPrerequisites.isEmpty()) { + return null; + } + + // Compilation prerequisites gathered in the compilationPrerequisites + // must be generated prior to executing C++ compilation step that depends + // on them (since these prerequisites include all potential header files, etc + // that could be referenced during compilation). So there is a definite need + // to ensure scheduling edge dependency. However, those prerequisites should + // have no effect on the decision whether C++ compilation should happen in + // the first place - only CppCompileAction outputs (*.o and *.d files) and + // all files referenced by the *.d file should be used to make that decision. + // If this action was never executed, then *.d file would be missing, forcing + // compilation to occur. If *.d file is present and has not changed then the + // only reason that would force us to re-compile would be change in one of + // the files referenced by the *.d file, since no other files participated + // in the compilation. We also need to propagate errors through this + // dependency link. So we use an error propagating middleman. + // Such middleman will be ignored by the dependency checker yet will still + // represent an edge in the action dependency graph - forcing proper execution + // order and error propagation. + return middlemanFactory.createErrorPropagatingMiddleman( + owner, ruleContext.getLabel().toString(), purpose, + ImmutableList.copyOf(compilationPrerequisites), + ruleContext.getConfiguration().getMiddlemanDirectory()); + } + + /** + * Returns the same set of artifacts as createMiddleman() would, but without + * actually creating middlemen. + */ + private Artifact getMiddlemanArtifact(MiddlemanFactory middlemanFactory) { + if (compilationPrerequisites.isEmpty()) { + return null; + } + + return middlemanFactory.getErrorPropagatingMiddlemanArtifact(ruleContext.getLabel() + .toString(), purpose, ruleContext.getConfiguration().getMiddlemanDirectory()); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java new file mode 100644 index 0000000000..e90f9f74ba --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java @@ -0,0 +1,1356 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.MiddlemanExpander; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.CppCompileInfo; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.EventKind; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.Tool; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.DependencySet; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.annotation.Nullable; + +/** + * Action that represents some kind of C++ compilation step. + */ +@ThreadCompatible +public class CppCompileAction extends AbstractAction implements IncludeScannable { + /** + * Represents logic that determines which artifacts, if any, should be added to the actual inputs + * for each included file (in addition to the included file itself) + */ + public interface IncludeResolver { + /** + * Returns the set of files to be added for an included file (as returned in the .d file) + */ + Iterable getInputsForIncludedFile( + Artifact includedFile, ArtifactResolver artifactResolver); + } + + public static final IncludeResolver VOID_INCLUDE_RESOLVER = new IncludeResolver() { + @Override + public Iterable getInputsForIncludedFile(Artifact includedFile, + ArtifactResolver artifactResolver) { + return ImmutableList.of(); + } + }; + + private static final int VALIDATION_DEBUG = 0; // 0==none, 1==warns/errors, 2==all + private static final boolean VALIDATION_DEBUG_WARN = VALIDATION_DEBUG >= 1; + + /** + * A string constant for the c compilation action. + */ + public static final String C_COMPILE = "c-compile"; + + /** + * A string constant for the c++ compilation action. + */ + public static final String CPP_COMPILE = "c++-compile"; + + /** + * A string constant for the c++ header parsing. + */ + public static final String CPP_HEADER_PARSING = "c++-header-parsing"; + + /** + * A string constant for the c++ header preprocessing. + */ + public static final String CPP_HEADER_PREPROCESSING = "c++-header-preprocessing"; + + /** + * A string constant for the c++ module compilation action. + * Note: currently we don't support C module compilation. + */ + public static final String CPP_MODULE_COMPILE = "c++-module-compile"; + + /** + * A string constant for the preprocessing assembler action. + */ + public static final String PREPROCESS_ASSEMBLE = "preprocess-assemble"; + + + private final BuildConfiguration configuration; + protected final Artifact outputFile; + private final Label sourceLabel; + private final Artifact dwoFile; + private final Artifact optionalSourceFile; + private final NestedSet mandatoryInputs; + private final CppCompilationContext context; + private final Collection extraSystemIncludePrefixes; + private final Iterable lipoScannables; + private final CppCompileCommandLine cppCompileCommandLine; + private final boolean enableLayeringCheck; + private final boolean compileHeaderModules; + + @VisibleForTesting + final CppConfiguration cppConfiguration; + private final Class actionContext; + private final IncludeResolver includeResolver; + + /** + * Identifier for the actual execution time behavior of the action. + * + *

Required because the behavior of this class can be modified by injecting code in the + * constructor or by inheritance, and we want to have different cache keys for those. + */ + private final UUID actionClassId; + + private boolean inputsKnown = false; + + /** + * Set when the action prepares for execution. Used to preserve state between preparation and + * execution. + */ + private Collection additionalInputs = null; + + /** + * Creates a new action to compile C/C++ source files. + * + * @param owner the owner of the action, usually the configured target that + * emitted it + * @param sourceFile the source file that should be compiled. {@code mandatoryInputs} must + * contain this file + * @param sourceLabel the label of the rule the source file is generated by + * @param mandatoryInputs any additional files that need to be present for the + * compilation to succeed, can be empty but not null, for example, extra sources for FDO. + * @param outputFile the object file that is written as result of the + * compilation, or the fake object for {@link FakeCppCompileAction}s + * @param dotdFile the .d file that is generated as a side-effect of + * compilation + * @param gcnoFile the coverage notes that are written in coverage mode, can + * be null + * @param dwoFile the .dwo output file where debug information is stored for Fission + * builds (null if Fission mode is disabled) + * @param optionalSourceFile an additional optional source file (null if unneeded) + * @param configuration the build configurations + * @param context the compilation context + * @param copts options for the compiler + * @param coptsFilter regular expression to remove options from {@code copts} + * @param compileHeaderModules whether to compile C++ header modules + */ + protected CppCompileAction(ActionOwner owner, + // TODO(bazel-team): Eventually we will remove 'features'; all functionality in 'features' + // will be provided by 'featureConfiguration'. + ImmutableList features, + FeatureConfiguration featureConfiguration, + Artifact sourceFile, + Label sourceLabel, + NestedSet mandatoryInputs, + Artifact outputFile, + DotdFile dotdFile, + @Nullable Artifact gcnoFile, + @Nullable Artifact dwoFile, + Artifact optionalSourceFile, + BuildConfiguration configuration, + CppConfiguration cppConfiguration, + CppCompilationContext context, + Class actionContext, + ImmutableList copts, + ImmutableList pluginOpts, + Predicate coptsFilter, + ImmutableList extraSystemIncludePrefixes, + boolean enableLayeringCheck, + @Nullable String fdoBuildStamp, + IncludeResolver includeResolver, + Iterable lipoScannables, + UUID actionClassId, + boolean compileHeaderModules) { + // getInputs() method is overridden in this class so we pass a dummy empty + // list to the AbstractAction constructor in place of a real input collection. + super(owner, + Artifact.NO_ARTIFACTS, + CollectionUtils.asListWithoutNulls(outputFile, dotdFile.artifact(), + gcnoFile, dwoFile)); + this.configuration = configuration; + this.sourceLabel = sourceLabel; + this.outputFile = Preconditions.checkNotNull(outputFile); + this.dwoFile = dwoFile; + this.optionalSourceFile = optionalSourceFile; + this.context = context; + this.extraSystemIncludePrefixes = extraSystemIncludePrefixes; + this.enableLayeringCheck = enableLayeringCheck; + this.includeResolver = includeResolver; + this.cppConfiguration = cppConfiguration; + if (cppConfiguration != null && !cppConfiguration.shouldScanIncludes()) { + inputsKnown = true; + } + this.cppCompileCommandLine = new CppCompileCommandLine(sourceFile, dotdFile, + context.getCppModuleMap(), copts, coptsFilter, pluginOpts, + (gcnoFile != null), features, featureConfiguration, fdoBuildStamp); + this.actionContext = actionContext; + this.lipoScannables = lipoScannables; + this.actionClassId = actionClassId; + this.compileHeaderModules = compileHeaderModules; + + // We do not need to include the middleman artifact since it is a generated + // artifact and will definitely exist prior to this action execution. + this.mandatoryInputs = mandatoryInputs; + setInputs(createInputs(mandatoryInputs, context.getCompilationPrerequisites(), + optionalSourceFile)); + } + + private static NestedSet createInputs( + NestedSet mandatoryInputs, + Set prerequisites, Artifact optionalSourceFile) { + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + if (optionalSourceFile != null) { + builder.add(optionalSourceFile); + } + builder.addAll(prerequisites); + builder.addTransitive(mandatoryInputs); + return builder.build(); + } + + public boolean shouldScanIncludes() { + return cppConfiguration.shouldScanIncludes(); + } + + @Override + public List getBuiltInIncludeDirectories() { + return cppConfiguration.getBuiltInIncludeDirectories(); + } + + public String getHostSystemName() { + return cppConfiguration.getHostSystemName(); + } + + @Override + public NestedSet getMandatoryInputs() { + return mandatoryInputs; + } + + @Override + public boolean inputsKnown() { + return inputsKnown; + } + + /** + * Returns the list of additional inputs found by dependency discovery, during action preparation, + * and clears the stored list. {@link #prepare} must be called before this method is called, on + * each action execution. + */ + public Collection getAdditionalInputs() { + Collection result = Preconditions.checkNotNull(additionalInputs); + additionalInputs = null; + return result; + } + + @Override + public boolean discoversInputs() { + return true; + } + + @Override + public void discoverInputs(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + try { + this.additionalInputs = executor.getContext(CppCompileActionContext.class) + .findAdditionalInputs(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Include scanning of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + + @Override + public Artifact getPrimaryInput() { + return getSourceFile(); + } + + @Override + public Artifact getPrimaryOutput() { + return getOutputFile(); + } + + /** + * Returns the path of the c/cc source for gcc. + */ + public final Artifact getSourceFile() { + return cppCompileCommandLine.sourceFile; + } + + /** + * Returns the path where gcc should put its result. + */ + public Artifact getOutputFile() { + return outputFile; + } + + /** + * Returns the path of the debug info output file (when debug info is + * spliced out of the .o file via fission). + */ + @Nullable + Artifact getDwoFile() { + return dwoFile; + } + + protected PathFragment getInternalOutputFile() { + return outputFile.getExecPath(); + } + + @VisibleForTesting + public List getPluginOpts() { + return cppCompileCommandLine.pluginOpts; + } + + Collection getExtraSystemIncludePrefixes() { + return extraSystemIncludePrefixes; + } + + @Override + public Map getLegalGeneratedScannerFileMap() { + Map legalOuts = new HashMap<>(); + + for (Artifact a : context.getDeclaredIncludeSrcs()) { + if (!a.isSourceArtifact()) { + legalOuts.put(a, null); + } + } + for (Pair pregreppedSrcs : context.getPregreppedHeaders()) { + Artifact hdr = pregreppedSrcs.getFirst(); + Preconditions.checkState(!hdr.isSourceArtifact(), hdr); + legalOuts.put(hdr, pregreppedSrcs.getSecond().getPath()); + } + return Collections.unmodifiableMap(legalOuts); + } + + /** + * Returns the path where gcc should put the discovered dependency + * information. + */ + public DotdFile getDotdFile() { + return cppCompileCommandLine.dotdFile; + } + + protected boolean needsIncludeScanning(Executor executor) { + return executor.getContext(actionContext).needsIncludeScanning(); + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(actionContext).strategyLocality(); + } + + @VisibleForTesting + public CppCompilationContext getContext() { + return context; + } + + @Override + public List getQuoteIncludeDirs() { + return context.getQuoteIncludeDirs(); + } + + @Override + public List getIncludeDirs() { + ImmutableList.Builder result = ImmutableList.builder(); + result.addAll(context.getIncludeDirs()); + for (String opt : cppCompileCommandLine.copts) { + if (opt.startsWith("-I") && opt.length() > 2) { + // We insist on the combined form "-Idir". + result.add(new PathFragment(opt.substring(2))); + } + } + return result.build(); + } + + @Override + public List getSystemIncludeDirs() { + ImmutableList.Builder result = ImmutableList.builder(); + result.addAll(context.getSystemIncludeDirs()); + for (String opt : cppCompileCommandLine.copts) { + if (opt.startsWith("-isystem") && opt.length() > 8) { + // We insist on the combined form "-isystemdir". + result.add(new PathFragment(opt.substring(8))); + } + } + return result.build(); + } + + @Override + public List getCmdlineIncludes() { + ImmutableList.Builder cmdlineIncludes = ImmutableList.builder(); + List args = getArgv(); + for (Iterator argi = args.iterator(); argi.hasNext();) { + String arg = argi.next(); + if (arg.equals("-include") && argi.hasNext()) { + cmdlineIncludes.add(argi.next()); + } + } + return cmdlineIncludes.build(); + } + + @Override + public Collection getIncludeScannerSources() { + NestedSetBuilder builder = NestedSetBuilder.stableOrder(); + // For every header module we use for the build we need the set of sources that it can + // reference. + builder.addAll(context.getTransitiveHeaderModuleSrcs()); + if (CppFileTypes.CPP_MODULE_MAP.matches(getSourceFile().getPath())) { + // If this is an action that compiles the header module itself, the source we build is the + // module map, and we need to include-scan all headers that are referenced in the module map. + // We need to do include scanning as long as we want to support building code bases that are + // not fully strict layering clean. + builder.addAll(context.getHeaderModuleSrcs()); + } else { + builder.add(getSourceFile()); + } + return builder.build().toCollection(); + } + + @Override + public Iterable getAuxiliaryScannables() { + return lipoScannables; + } + + /** + * Returns the list of "-D" arguments that should be used by this gcc + * invocation. Only used for testing. + */ + @VisibleForTesting + public ImmutableCollection getDefines() { + return context.getDefines(); + } + + /** + * Returns an (immutable) map of environment key, value pairs to be + * provided to the C++ compiler. + */ + public ImmutableMap getEnvironment() { + Map environment = + new LinkedHashMap<>(configuration.getDefaultShellEnvironment()); + if (configuration.isCodeCoverageEnabled()) { + environment.put("PWD", "/proc/self/cwd"); + } + if (OS.getCurrent() == OS.WINDOWS) { + // TODO(bazel-team): Both GCC and clang rely on their execution directories being on + // PATH, otherwise they fail to find dependent DLLs (and they fail silently...). On + // the other hand, Windows documentation says that the directory of the executable + // is always searched for DLLs first. Not sure what to make of it. + // Other options are to forward the system path (brittle), or to add a PATH field to + // the crosstool file. + environment.put("PATH", cppConfiguration.getToolPathFragment(Tool.GCC).getParentDirectory() + .getPathString()); + } + return ImmutableMap.copyOf(environment); + } + + /** + * Returns a new, mutable list of command and arguments (argv) to be passed + * to the gcc subprocess. + */ + public final List getArgv() { + return getArgv(getInternalOutputFile()); + } + + protected final List getArgv(PathFragment outputFile) { + return cppCompileCommandLine.getArgv(outputFile); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + CppCompileInfo.Builder info = CppCompileInfo.newBuilder(); + info.setTool(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString()); + for (String option : getCompilerOptions()) { + info.addCompilerOption(option); + } + info.setOutputFile(outputFile.getExecPathString()); + info.setSourceFile(getSourceFile().getExecPathString()); + if (inputsKnown()) { + info.addAllSourcesAndHeaders(Artifact.toExecPaths(getInputs())); + } else { + info.addSourcesAndHeaders(getSourceFile().getExecPathString()); + info.addAllSourcesAndHeaders( + Artifact.toExecPaths(context.getDeclaredIncludeSrcs())); + } + + return super.getExtraActionInfo() + .setExtension(CppCompileInfo.cppCompileInfo, info.build()); + } + + /** + * Returns the compiler options. + */ + @VisibleForTesting + public List getCompilerOptions() { + return cppCompileCommandLine.getCompilerOptions(); + } + + /** + * Enforce that the includes actually visited during the compile were properly + * declared in the rules. + * + *

The technique is to walk through all of the reported includes that gcc + * emits into the .d file, and verify that they came from acceptable + * relative include directories. This is done in two steps: + * + *

First, each included file is stripped of any include path prefix from + * {@code quoteIncludeDirs} to produce an effective relative include dir+name. + * + *

Second, the remaining directory is looked up in {@code declaredIncludeDirs}, + * a list of acceptable dirs. This list contains a set of dir fragments that + * have been calculated by the configured target to be allowable for inclusion + * by this source. If no match is found, an error is reported and an exception + * is thrown. + * + * @throws ActionExecutionException iff there was an undeclared dependency + */ + @VisibleForTesting + public void validateInclusions( + MiddlemanExpander middlemanExpander, EventHandler eventHandler) + throws ActionExecutionException { + if (!cppConfiguration.shouldScanIncludes() || !inputsKnown()) { + return; + } + + IncludeProblems errors = new IncludeProblems(); + IncludeProblems warnings = new IncludeProblems(); + Set allowedIncludes = new HashSet<>(); + for (Artifact input : mandatoryInputs) { + if (input.isMiddlemanArtifact()) { + middlemanExpander.expand(input, allowedIncludes); + } + allowedIncludes.add(input); + } + + if (optionalSourceFile != null) { + allowedIncludes.add(optionalSourceFile); + } + List cxxSystemIncludeDirs = + cppConfiguration.getBuiltInIncludeDirectories(); + Iterable ignoreDirs = Iterables.concat(cxxSystemIncludeDirs, + extraSystemIncludePrefixes, context.getSystemIncludeDirs()); + + // Copy the sets to hash sets for fast contains checking. + // Avoid immutable sets here to limit memory churn. + Set declaredIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeDirs()); + Set warnIncludeDirs = Sets.newHashSet(context.getDeclaredIncludeWarnDirs()); + Set declaredIncludeSrcs = Sets.newHashSet(context.getDeclaredIncludeSrcs()); + for (Artifact input : getInputs()) { + if (context.getCompilationPrerequisites().contains(input) + || allowedIncludes.contains(input)) { + continue; // ignore our fixed source in mandatoryInput: we just want includes + } + // Ignore headers from built-in include directories. + if (FileSystemUtils.startsWithAny(input.getExecPath(), ignoreDirs)) { + continue; + } + if (!isDeclaredIn(input, declaredIncludeDirs, declaredIncludeSrcs)) { + // This call can never match the declared include sources (they would be matched above). + // There are no declared include sources we need to warn about, so use an empty set here. + if (isDeclaredIn(input, warnIncludeDirs, ImmutableSet.of())) { + warnings.add(input.getPath().toString()); + } else { + errors.add(input.getPath().toString()); + } + } + } + if (VALIDATION_DEBUG_WARN) { + synchronized (System.err) { + if (VALIDATION_DEBUG >= 2 || errors.hasProblems() || warnings.hasProblems()) { + if (errors.hasProblems()) { + System.err.println("ERROR: Include(s) were not in declared srcs:"); + } else if (warnings.hasProblems()) { + System.err.println("WARN: Include(s) were not in declared srcs:"); + } else { + System.err.println("INFO: Include(s) were OK for '" + getSourceFile() + + "', declared srcs:"); + } + for (Artifact a : context.getDeclaredIncludeSrcs()) { + System.err.println(" '" + a.toDetailString() + "'"); + } + System.err.println(" or under declared dirs:"); + for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeDirs())) { + System.err.println(" '" + f + "'"); + } + System.err.println(" or under declared warn dirs:"); + for (PathFragment f : Sets.newTreeSet(context.getDeclaredIncludeWarnDirs())) { + System.err.println(" '" + f + "'"); + } + System.err.println(" with prefixes:"); + for (PathFragment dirpath : context.getQuoteIncludeDirs()) { + System.err.println(" '" + dirpath + "'"); + } + } + } + } + + if (warnings.hasProblems()) { + eventHandler.handle( + new Event(EventKind.WARNING, + getOwner().getLocation(), warnings.getMessage(this, getSourceFile()), + Label.print(getOwner().getLabel()))); + } + errors.assertProblemFree(this, getSourceFile()); + } + + /** + * Returns true if an included artifact is declared in a set of allowed + * include directories. The simple case is that the artifact's parent + * directory is contained in the set, or is empty. + * + *

This check also supports a wildcard suffix of '**' for the cases where the + * calculations are inexact. + * + *

It also handles unseen non-nested-package subdirs by walking up the path looking + * for matches. + */ + private static boolean isDeclaredIn(Artifact input, Set declaredIncludeDirs, + Set declaredIncludeSrcs) { + // First check if it's listed in "srcs". If so, then its declared & OK. + if (declaredIncludeSrcs.contains(input)) { + return true; + } + // If it's a derived artifact, then it MUST be listed in "srcs" as checked above. + // We define derived here as being not source and not under the include link tree. + if (!input.isSourceArtifact() + && !input.getRoot().getExecPath().getBaseName().equals("include")) { + return false; + } + // Need to do dir/package matching: first try a quick exact lookup. + PathFragment includeDir = input.getRootRelativePath().getParentDirectory(); + if (includeDir.segmentCount() == 0 || declaredIncludeDirs.contains(includeDir)) { + return true; // OK: quick exact match. + } + // Not found in the quick lookup: try the wildcards. + for (PathFragment declared : declaredIncludeDirs) { + if (declared.getBaseName().equals("**")) { + if (includeDir.startsWith(declared.getParentDirectory())) { + return true; // OK: under a wildcard dir. + } + } + } + // Still not found: see if it is in a subdir of a declared package. + Path root = input.getRoot().getPath(); + for (Path dir = input.getPath().getParentDirectory();;) { + if (dir.getRelative("BUILD").exists()) { + return false; // Bad: this is a sub-package, not a subdir of a declared package. + } + dir = dir.getParentDirectory(); + if (dir.equals(root)) { + return false; // Bad: at the top, give up. + } + if (declaredIncludeDirs.contains(dir.relativeTo(root))) { + return true; // OK: found under a declared dir. + } + } + } + + /** + * Recalculates this action's live input collection, including sources, middlemen. + * + * @throws ActionExecutionException iff any errors happen during update. + */ + @VisibleForTesting + @ThreadCompatible + public final void updateActionInputs(Path execRoot, + ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply) + throws ActionExecutionException { + if (!cppConfiguration.shouldScanIncludes()) { + return; + } + inputsKnown = false; + NestedSetBuilder inputs = NestedSetBuilder.stableOrder(); + Profiler.instance().startTask(ProfilerTask.ACTION_UPDATE, this); + try { + inputs.addTransitive(mandatoryInputs); + if (optionalSourceFile != null) { + inputs.add(optionalSourceFile); + } + inputs.addAll(context.getCompilationPrerequisites()); + populateActionInputs(execRoot, artifactResolver, reply, inputs); + inputsKnown = true; + } finally { + Profiler.instance().completeTask(ProfilerTask.ACTION_UPDATE); + synchronized (this) { + setInputs(inputs.build()); + } + } + } + + private DependencySet processDepset(Path execRoot, CppCompileActionContext.Reply reply) + throws IOException { + DependencySet depSet = new DependencySet(execRoot); + + // artifact() is null if we are not using in-memory .d files. We also want to prepare for the + // case where we expected an in-memory .d file, but we did not get an appropriate response. + // Perhaps we produced the file locally. + if (getDotdFile().artifact() != null || reply == null) { + return depSet.read(getDotdFile().getPath()); + } else { + // This is an in-memory .d file. + return depSet.process(reply.getContents()); + } + } + + /** + * Populates the given ordered collection with additional input artifacts + * relevant to the specific action implementation. + * + *

The default implementation updates this Action's input set by reading + * dynamically-discovered dependency information out of the .d file. + * + *

Artifacts are considered inputs but not "mandatory" inputs. + * + * + * @param reply the reply from the compilation. + * @param inputs the ordered collection of inputs to append to + * @throws ActionExecutionException iff the .d is missing, malformed or has + * unresolvable included artifacts. + */ + @ThreadCompatible + private void populateActionInputs(Path execRoot, + ArtifactResolver artifactResolver, CppCompileActionContext.Reply reply, + NestedSetBuilder inputs) + throws ActionExecutionException { + try { + // Read .d file. + DependencySet depSet = processDepset(execRoot, reply); + + // Determine prefixes of allowed absolute inclusions. + CppConfiguration toolchain = cppConfiguration; + List systemIncludePrefixes = new ArrayList<>(); + for (PathFragment includePath : toolchain.getBuiltInIncludeDirectories()) { + if (includePath.isAbsolute()) { + systemIncludePrefixes.add(includePath); + } + } + systemIncludePrefixes.addAll(extraSystemIncludePrefixes); + + // Check inclusions. + IncludeProblems problems = new IncludeProblems(); + Map allowedDerivedInputsMap = getAllowedDerivedInputsMap(); + for (PathFragment execPath : depSet.getDependencies()) { + if (execPath.isAbsolute()) { + // Absolute includes from system paths are ignored. + if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) { + continue; + } + // Since gcc is given only relative paths on the command line, + // non-system include paths here should never be absolute. If they + // are, it's probably due to a non-hermetic #include, & we should stop + // the build with an error. + if (execPath.startsWith(execRoot.asFragment())) { + execPath = execPath.relativeTo(execRoot.asFragment()); // funky but tolerable path + } else { + problems.add(execPath.getPathString()); + continue; + } + } + Artifact artifact = allowedDerivedInputsMap.get(execPath); + if (artifact == null) { + artifact = artifactResolver.resolveSourceArtifact(execPath); + } + if (artifact != null) { + inputs.add(artifact); + // In some cases, execution backends need extra files for each included file. Add them + // to the set of actual inputs. + inputs.addAll(includeResolver.getInputsForIncludedFile(artifact, artifactResolver)); + } else { + // Abort if we see files that we can't resolve, likely caused by + // undeclared includes or illegal include constructs. + problems.add(execPath.getPathString()); + } + } + problems.assertProblemFree(this, getSourceFile()); + } catch (IOException e) { + // Some kind of IO or parse exception--wrap & rethrow it to stop the build. + throw new ActionExecutionException("error while parsing .d file", e, this, false); + } + } + + @Override + public void updateInputsFromCache( + ArtifactResolver artifactResolver, Collection inputPaths) { + // Note that this method may trigger a violation of the desirable invariant that getInputs() + // is a superset of getMandatoryInputs(). See bug about an "action not in canonical form" + // error message and the integration test test_crosstool_change_and_failure(). + + Map allowedDerivedInputsMap = getAllowedDerivedInputsMap(); + List inputs = new ArrayList<>(); + for (PathFragment execPath : inputPaths) { + // The artifact may be a derived artifact, and if it has been created already, then we still + // want to keep it to preserve incrementality. + Artifact artifact = allowedDerivedInputsMap.get(execPath); + if (artifact == null) { + artifact = artifactResolver.resolveSourceArtifact(execPath); + } + // If PathFragment cannot be resolved into the artifact - ignore it. This could happen if + // rule definition has changed and action no longer depends on, e.g., additional source file + // in the separate package and that package is no longer referenced anywhere else. + // It is safe to ignore such paths because dependency checker would identify change in inputs + // (ignored path was used before) and will force action execution. + if (artifact != null) { + inputs.add(artifact); + } + } + inputsKnown = true; + synchronized (this) { + setInputs(inputs); + } + } + + private Map getAllowedDerivedInputsMap() { + Map allowedDerivedInputMap = new HashMap<>(); + addToMap(allowedDerivedInputMap, mandatoryInputs); + addToMap(allowedDerivedInputMap, context.getDeclaredIncludeSrcs()); + addToMap(allowedDerivedInputMap, context.getCompilationPrerequisites()); + Artifact artifact = getSourceFile(); + if (!artifact.isSourceArtifact()) { + allowedDerivedInputMap.put(artifact.getExecPath(), artifact); + } + return allowedDerivedInputMap; + } + + private void addToMap(Map map, Iterable artifacts) { + for (Artifact artifact : artifacts) { + if (!artifact.isSourceArtifact()) { + map.put(artifact.getExecPath(), artifact); + } + } + } + + @Override + protected String getRawProgressMessage() { + return "Compiling " + getSourceFile().prettyPrint(); + } + + /** + * Return the directories in which to look for headers (pertains to headers + * not specifically listed in {@code declaredIncludeSrcs}). The return value + * may contain duplicate elements. + */ + public NestedSet getDeclaredIncludeDirs() { + return context.getDeclaredIncludeDirs(); + } + + /** + * Return the directories in which to look for headers and issue a warning. + * (pertains to headers not specifically listed in {@code + * declaredIncludeSrcs}). The return value may contain duplicate elements. + */ + public NestedSet getDeclaredIncludeWarnDirs() { + return context.getDeclaredIncludeWarnDirs(); + } + + /** + * Return explicit header files (i.e., header files explicitly listed). The + * return value may contain duplicate elements. + */ + public NestedSet getDeclaredIncludeSrcs() { + return context.getDeclaredIncludeSrcs(); + } + + /** + * Return explicit header files (i.e., header files explicitly listed) in an order + * that is stable between builds. + */ + protected final List getDeclaredIncludeSrcsInStableOrder() { + List paths = new ArrayList<>(); + for (Artifact declaredIncludeSrc : context.getDeclaredIncludeSrcs()) { + paths.add(declaredIncludeSrc.getExecPath()); + } + Collections.sort(paths); // Order is not important, but stability is. + return paths; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(actionContext).estimateResourceConsumption(this); + } + + @VisibleForTesting + public Class getActionContext() { + return actionContext; + } + + /** + * Estimate resource consumption when this action is executed locally. + */ + public ResourceSet estimateResourceConsumptionLocal() { + // We use a local compile, so much of the time is spent waiting for IO, + // but there is still significant CPU; hence we estimate 50% cpu usage. + return new ResourceSet(/*memoryMb=*/200, /*cpuUsage=*/0.5, /*ioUsage=*/0.0); + } + + @Override + public String computeKey() { + Fingerprint f = new Fingerprint(); + f.addUUID(actionClassId); + f.addStrings(getArgv()); + + /* + * getArgv() above captures all changes which affect the compilation + * command and hence the contents of the object file. But we need to + * also make sure that we reexecute the action if any of the fields + * that affect whether validateIncludes() will report an error or warning + * have changed, otherwise we might miss some errors. + */ + f.addPaths(context.getDeclaredIncludeDirs()); + f.addPaths(context.getDeclaredIncludeWarnDirs()); + f.addPaths(getDeclaredIncludeSrcsInStableOrder()); + f.addPaths(getExtraSystemIncludePrefixes()); + return f.hexDigestAndReset(); + } + + @Override + @ThreadCompatible + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + CppCompileActionContext.Reply reply; + try { + reply = executor.getContext(actionContext).execWithReply(this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("C++ compilation of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + ensureCoverageNotesFilesExist(); + IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class); + updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply); + reply = null; // Clear in-memory .d files early. + validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler()); + } + + /** + * Gcc only creates ".gcno" files if the compilation unit is non-empty. + * To ensure that the set of outputs for a CppCompileAction remains consistent + * and doesn't vary dynamically depending on the _contents_ of the input files, + * we create empty ".gcno" files if gcc didn't create them. + */ + private void ensureCoverageNotesFilesExist() throws ActionExecutionException { + for (Artifact output : getOutputs()) { + if (CppFileTypes.COVERAGE_NOTES.matches(output.getFilename()) // ".gcno" + && !output.getPath().exists()) { + try { + FileSystemUtils.createEmptyFile(output.getPath()); + } catch (IOException e) { + throw new ActionExecutionException( + "Error creating file '" + output.getPath() + "': " + e.getMessage(), e, this, false); + } + } + } + } + + /** + * Provides list of include files needed for performing extra actions on this action when run + * remotely. The list of include files is created by performing a header scan on the known input + * files. + */ + @Override + public Iterable getInputFilesForExtraAction( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Collection scannedIncludes = + actionExecutionContext.getExecutor().getContext(actionContext) + .getScannedIncludeFiles(this, actionExecutionContext); + // Use a set to eliminate duplicates. + ImmutableSet.Builder result = ImmutableSet.builder(); + return result.addAll(getInputs()).addAll(scannedIncludes).build(); + } + + @Override + public String getMnemonic() { return "CppCompile"; } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + message.append(getProgressMessage()); + message.append('\n'); + message.append(" Command: "); + message.append( + ShellEscaper.escapeString(cppConfiguration.getLdExecutable().getPathString())); + message.append('\n'); + // Outputting one argument per line makes it easier to diff the results. + for (String argument : ShellEscaper.escapeAll(getArgv())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + + for (PathFragment path : context.getDeclaredIncludeDirs()) { + message.append(" Declared include directory: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + + for (PathFragment path : getDeclaredIncludeSrcsInStableOrder()) { + message.append(" Declared include source: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + + for (PathFragment path : getExtraSystemIncludePrefixes()) { + message.append(" Extra system include prefix: "); + message.append(ShellEscaper.escapeString(path.getPathString())); + message.append('\n'); + } + return message.toString(); + } + + /** + * The compile command line for the enclosing C++ compile action. + */ + public final class CppCompileCommandLine { + private final Artifact sourceFile; + private final DotdFile dotdFile; + private final CppModuleMap cppModuleMap; + private final List copts; + private final Predicate coptsFilter; + private final List pluginOpts; + private final boolean isInstrumented; + private final Collection features; + private final FeatureConfiguration featureConfiguration; + + // The value of the BUILD_FDO_TYPE macro to be defined on command line + @Nullable private final String fdoBuildStamp; + + public CppCompileCommandLine(Artifact sourceFile, DotdFile dotdFile, CppModuleMap cppModuleMap, + ImmutableList copts, Predicate coptsFilter, + ImmutableList pluginOpts, boolean isInstrumented, + Collection features, FeatureConfiguration featureConfiguration, + @Nullable String fdoBuildStamp) { + this.sourceFile = Preconditions.checkNotNull(sourceFile); + this.dotdFile = Preconditions.checkNotNull(dotdFile); + this.cppModuleMap = cppModuleMap; + this.copts = Preconditions.checkNotNull(copts); + this.coptsFilter = coptsFilter; + this.pluginOpts = Preconditions.checkNotNull(pluginOpts); + this.isInstrumented = isInstrumented; + this.features = Preconditions.checkNotNull(features); + this.featureConfiguration = featureConfiguration; + this.fdoBuildStamp = fdoBuildStamp; + } + + protected List getArgv(PathFragment outputFile) { + List commandLine = new ArrayList<>(); + + // first: The command name. + commandLine.add(cppConfiguration.getToolPathFragment(Tool.GCC).getPathString()); + + // second: The compiler options. + commandLine.addAll(getCompilerOptions()); + + // third: The file to compile! + commandLine.add("-c"); + commandLine.add(sourceFile.getExecPathString()); + + // finally: The output file. (Prefixed with -o). + commandLine.add("-o"); + commandLine.add(outputFile.getPathString()); + + return commandLine; + } + + private String getActionName() { + PathFragment sourcePath = sourceFile.getExecPath(); + if (CppFileTypes.CPP_MODULE_MAP.matches(sourcePath)) { + return CPP_MODULE_COMPILE; + } else if (CppFileTypes.CPP_HEADER.matches(sourcePath)) { + // TODO(bazel-team): Handle C headers that probably don't work in C++ mode. + // TODO(bazel-team): Replace use of features.contains with featureConfiguration.isEnabled + // here (the other instances will need to stay with the current feature selection process + // until all crosstool configurations have been converted). + if (features.contains(CppRuleClasses.PARSE_HEADERS)) { + return CPP_HEADER_PARSING; + } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) { + return CPP_HEADER_PREPROCESSING; + } else { + // CcCommon.collectCAndCppSources() ensures we do not add headers to + // the compilation artifacts unless either 'parse_headers' or + // 'preprocess_headers' is set. + throw new IllegalStateException(); + } + } else if (CppFileTypes.C_SOURCE.matches(sourcePath)) { + return C_COMPILE; + } else if (CppFileTypes.CPP_SOURCE.matches(sourcePath)) { + return CPP_COMPILE; + } else if (CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR.matches(sourcePath)) { + return PREPROCESS_ASSEMBLE; + } + // CcLibraryHelper ensures CppCompileAction only gets instantiated for supported file types. + throw new IllegalStateException(); + } + + public List getCompilerOptions() { + List options = new ArrayList<>(); + + // TODO(bazel-team): Extract combinations of options into sections in the CROSSTOOL file. + if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFile.getExecPath())) { + options.add("-x"); + options.add("c++"); + } else if (CppFileTypes.CPP_HEADER.matches(sourceFile.getExecPath())) { + // TODO(bazel-team): Read the compiler flag settings out of the CROSSTOOL file. + // TODO(bazel-team): Handle C headers that probably don't work in C++ mode. + if (features.contains(CppRuleClasses.PARSE_HEADERS)) { + options.add("-x"); + options.add("c++-header"); + // Specifying -x c++-header will make clang/gcc create precompiled + // headers, which we suppress with -fsyntax-only. + options.add("-fsyntax-only"); + } else if (features.contains(CppRuleClasses.PREPROCESS_HEADERS)) { + options.add("-E"); + options.add("-x"); + options.add("c++"); + } else { + // CcCommon.collectCAndCppSources() ensures we do not add headers to + // the compilation artifacts unless either 'parse_headers' or + // 'preprocess_headers' is set. + throw new IllegalStateException(); + } + } + + for (PathFragment quoteIncludePath : context.getQuoteIncludeDirs()) { + // "-iquote" is a gcc-specific option. For C compilers that don't support "-iquote", + // we should instead use "-I". + options.add("-iquote"); + options.add(quoteIncludePath.getSafePathString()); + } + for (PathFragment includePath : context.getIncludeDirs()) { + options.add("-I" + includePath.getSafePathString()); + } + for (PathFragment systemIncludePath : context.getSystemIncludeDirs()) { + options.add("-isystem"); + options.add(systemIncludePath.getSafePathString()); + } + + CppConfiguration toolchain = cppConfiguration; + + // pluginOpts has to be added before defaultCopts because -fplugin must precede -plugin-arg. + options.addAll(pluginOpts); + addFilteredOptions(options, toolchain.getCompilerOptions(features)); + + // Enable instrumentation if requested. + if (isInstrumented) { + addFilteredOptions(options, ImmutableList.of("-fprofile-arcs", "-ftest-coverage")); + } + + String sourceFilename = sourceFile.getExecPathString(); + if (CppFileTypes.C_SOURCE.matches(sourceFilename)) { + addFilteredOptions(options, toolchain.getCOptions()); + } + if (CppFileTypes.CPP_SOURCE.matches(sourceFilename) + || CppFileTypes.CPP_HEADER.matches(sourceFilename) + || CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) { + addFilteredOptions(options, toolchain.getCxxOptions(features)); + } + + // Users don't expect the explicit copts to be filtered by coptsFilter, add them verbatim. + options.addAll(copts); + + for (String warn : cppConfiguration.getCWarns()) { + options.add("-W" + warn); + } + for (String define : context.getDefines()) { + options.add("-D" + define); + } + + // Stamp FDO builds with FDO subtype string + if (fdoBuildStamp != null) { + options.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\""); + } + + options.addAll(toolchain.getUnfilteredCompilerOptions(features)); + + // GCC gives randomized names to symbols which are defined in + // an anonymous namespace but have external linkage. To make + // computation of these deterministic, we want to override the + // default seed for the random number generator. It's safe to use + // any value which differs for all translation units; we use the + // path to the object file. + options.add("-frandom-seed=" + outputFile.getExecPathString()); + + // Add the options of --per_file_copt, if the label or the base name of the source file + // matches the specified regular expression filter. + for (PerLabelOptions perLabelOptions : cppConfiguration.getPerFileCopts()) { + if ((sourceLabel != null && perLabelOptions.isIncluded(sourceLabel)) + || perLabelOptions.isIncluded(sourceFile)) { + options.addAll(perLabelOptions.getOptions()); + } + } + + // Enable .d file generation. + if (dotdFile != null) { + // Gcc options: + // -MD turns on .d file output as a side-effect (doesn't imply -E) + // -MM[D] enables user includes only, not system includes + // -MF specifies the dotd file name + // Issues: + // -M[M] alone subverts actual .o output (implies -E) + // -M[M]D alone breaks some of the .d naming assumptions + // This combination gets user and system includes with specified name: + // -MD -MF + options.add("-MD"); + options.add("-MF"); + options.add(dotdFile.getSafeExecPath().getPathString()); + } + + if (cppModuleMap != null && (compileHeaderModules || enableLayeringCheck)) { + options.add("-Xclang-only=-fmodule-maps"); + options.add("-Xclang-only=-fmodule-name=" + cppModuleMap.getName()); + options.add("-Xclang-only=-fmodule-map-file=" + + cppModuleMap.getArtifact().getExecPathString()); + options.add("-Xclang=-fno-modules-implicit-maps"); + + if (compileHeaderModules) { + options.add("-Xclang-only=-fmodules"); + if (CppFileTypes.CPP_MODULE_MAP.matches(sourceFilename)) { + options.add("-Xclang=-emit-module"); + options.add("-Xcrosstool-module-compilation"); + } + // Select .pcm inputs to pass on the command line depending on whether + // we are in pic or non-pic mode. + // TODO(bazel-team): We want to add these to the compile even if the + // current target is not built as a module; currently that implies + // passing -fmodules to the compiler, which is experimental; thus, we + // do not use the header modules files for now if the current + // compilation is not modules enabled on its own. + boolean pic = copts.contains("-fPIC"); + for (Artifact source : context.getAdditionalInputs()) { + if ((pic && source.getFilename().endsWith(".pic.pcm")) || (!pic + && !source.getFilename().endsWith(".pic.pcm") + && source.getFilename().endsWith(".pcm"))) { + options.add("-Xclang=-fmodule-file=" + source.getExecPathString()); + } + } + } + if (enableLayeringCheck) { + options.add("-Xclang-only=-fmodules-strict-decluse"); + } + } + + if (FileType.contains(outputFile, CppFileTypes.ASSEMBLER, CppFileTypes.PIC_ASSEMBLER)) { + options.add("-S"); + } else if (FileType.contains(outputFile, CppFileTypes.PREPROCESSED_C, + CppFileTypes.PREPROCESSED_CPP, CppFileTypes.PIC_PREPROCESSED_C, + CppFileTypes.PIC_PREPROCESSED_CPP)) { + options.add("-E"); + } + + if (cppConfiguration.useFission()) { + options.add("-gsplit-dwarf"); + } + + options.addAll(featureConfiguration.getCommandLine(getActionName(), + ImmutableMultimap.of())); + return options; + } + + // For each option in 'in', add it to 'out' unless it is matched by the 'coptsFilter' regexp. + private void addFilteredOptions(List out, List in) { + Iterables.addAll(out, Iterables.filter(in, coptsFilter)); + } + } + + /** + * A reference to a .d file. There are two modes: + *
    + *
  1. an Artifact that represents a real on-disk file + *
  2. just an execPath that refers to a virtual .d file that is not written to disk + *
+ */ + public static class DotdFile { + private final Artifact artifact; + private final PathFragment execPath; + + public DotdFile(Artifact artifact) { + this.artifact = artifact; + this.execPath = null; + } + + public DotdFile(PathFragment execPath) { + this.artifact = null; + this.execPath = execPath; + } + + /** + * @return the Artifact or null + */ + public Artifact artifact() { + return artifact; + } + + /** + * @return Gets the execPath regardless of whether this is a real Artifact + */ + public PathFragment getSafeExecPath() { + return execPath == null ? artifact.getExecPath() : execPath; + } + + /** + * @return the on-disk location of the .d file or null + */ + public Path getPath() { + return artifact.getPath(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java new file mode 100644 index 0000000000..0d8da3e4a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java @@ -0,0 +1,439 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction.DotdFile; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction.IncludeResolver; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.FileType; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; + +/** + * Builder class to construct C++ compile actions. + */ +public class CppCompileActionBuilder { + public static final UUID GUID = UUID.fromString("cee5db0a-d2ad-4c69-9b81-97c936a29075"); + + private final ActionOwner owner; + private final List features = new ArrayList<>(); + private CcToolchainFeatures.FeatureConfiguration featureConfiguration; + private final Artifact sourceFile; + private final Label sourceLabel; + private final NestedSetBuilder mandatoryInputsBuilder; + private NestedSetBuilder pluginInputsBuilder; + private Artifact optionalSourceFile; + private Artifact outputFile; + private PathFragment tempOutputFile; + private DotdFile dotdFile; + private Artifact gcnoFile; + private final BuildConfiguration configuration; + private CppCompilationContext context = CppCompilationContext.EMPTY; + private final List copts = new ArrayList<>(); + private final List pluginOpts = new ArrayList<>(); + private final List nocopts = new ArrayList<>(); + private AnalysisEnvironment analysisEnvironment; + private ImmutableList extraSystemIncludePrefixes = ImmutableList.of(); + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + private String fdoBuildStamp; + private IncludeResolver includeResolver = CppCompileAction.VOID_INCLUDE_RESOLVER; + private UUID actionClassId = GUID; + private Class actionContext; + private CppConfiguration cppConfiguration; + private ImmutableMap lipoScannableMap; + + /** + * Creates a builder from a rule. This also uses the configuration and + * artifact factory from the rule. + */ + public CppCompileActionBuilder(RuleContext ruleContext, Artifact sourceFile, Label sourceLabel) { + this.owner = ruleContext.getActionOwner(); + this.actionContext = CppCompileActionContext.class; + this.cppConfiguration = ruleContext.getFragment(CppConfiguration.class); + this.analysisEnvironment = ruleContext.getAnalysisEnvironment(); + this.sourceFile = sourceFile; + this.sourceLabel = sourceLabel; + this.configuration = ruleContext.getConfiguration(); + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder(); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder(); + this.lipoScannableMap = getLipoScannableMap(ruleContext); + + features.addAll(ruleContext.getFeatures()); + } + + private static ImmutableMap getLipoScannableMap( + RuleContext ruleContext) { + if (!ruleContext.getFragment(CppConfiguration.class).isLipoOptimization()) { + return null; + } + + LipoContextProvider provider = ruleContext.getPrerequisite( + ":lipo_context_collector", Mode.DONT_CHECK, LipoContextProvider.class); + return provider.getIncludeScannables(); + } + + /** + * Creates a builder for an owner that is not required to be rule. + */ + public CppCompileActionBuilder( + ActionOwner owner, AnalysisEnvironment analysisEnvironment, Artifact sourceFile, + Label sourceLabel, BuildConfiguration configuration) { + this.owner = owner; + this.actionContext = CppCompileActionContext.class; + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.analysisEnvironment = analysisEnvironment; + this.sourceFile = sourceFile; + this.sourceLabel = sourceLabel; + this.configuration = configuration; + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder(); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder(); + this.lipoScannableMap = ImmutableMap.of(); + } + + /** + * Creates a builder that is a copy of another builder. + */ + public CppCompileActionBuilder(CppCompileActionBuilder other) { + this.owner = other.owner; + this.features.addAll(other.features); + this.featureConfiguration = other.featureConfiguration; + this.sourceFile = other.sourceFile; + this.sourceLabel = other.sourceLabel; + this.mandatoryInputsBuilder = NestedSetBuilder.stableOrder() + .addTransitive(other.mandatoryInputsBuilder.build()); + this.pluginInputsBuilder = NestedSetBuilder.stableOrder() + .addTransitive(other.pluginInputsBuilder.build()); + this.optionalSourceFile = other.optionalSourceFile; + this.outputFile = other.outputFile; + this.tempOutputFile = other.tempOutputFile; + this.dotdFile = other.dotdFile; + this.gcnoFile = other.gcnoFile; + this.configuration = other.configuration; + this.context = other.context; + this.copts.addAll(other.copts); + this.pluginOpts.addAll(other.pluginOpts); + this.nocopts.addAll(other.nocopts); + this.analysisEnvironment = other.analysisEnvironment; + this.extraSystemIncludePrefixes = ImmutableList.copyOf(other.extraSystemIncludePrefixes); + this.enableLayeringCheck = other.enableLayeringCheck; + this.compileHeaderModules = other.compileHeaderModules; + this.includeResolver = other.includeResolver; + this.actionClassId = other.actionClassId; + this.actionContext = other.actionContext; + this.cppConfiguration = other.cppConfiguration; + this.lipoScannableMap = other.lipoScannableMap; + } + + public PathFragment getTempOutputFile() { + return tempOutputFile; + } + + public Artifact getSourceFile() { + return sourceFile; + } + + public CppCompilationContext getContext() { + return context; + } + + public NestedSet getMandatoryInputs() { + return mandatoryInputsBuilder.build(); + } + + /** + * Returns the .dwo output file that matches the specified .o output file. If Fission mode + * isn't enabled for this build, this is null (we don't produce .dwo files in that case). + */ + private static Artifact getDwoFile(Artifact outputFile, AnalysisEnvironment artifactFactory, + CppConfiguration cppConfiguration) { + + // Only create .dwo's for .o compilations (i.e. not .ii or .S). + boolean isObjectOutput = CppFileTypes.OBJECT_FILE.matches(outputFile.getExecPath()) + || CppFileTypes.PIC_OBJECT_FILE.matches(outputFile.getExecPath()); + + // Note configurations can be null for tests. + if (cppConfiguration != null && cppConfiguration.useFission() && isObjectOutput) { + return artifactFactory.getDerivedArtifact( + FileSystemUtils.replaceExtension(outputFile.getRootRelativePath(), ".dwo"), + outputFile.getRoot()); + } else { + return null; + } + } + + private static Predicate getNocoptPredicate(Collection patterns) { + final ImmutableList finalPatterns = ImmutableList.copyOf(patterns); + if (finalPatterns.isEmpty()) { + return Predicates.alwaysTrue(); + } else { + return new Predicate() { + @Override + public boolean apply(String option) { + for (Pattern pattern : finalPatterns) { + if (pattern.matcher(option).matches()) { + return false; + } + } + + return true; + } + }; + } + } + + private Iterable getLipoScannables(NestedSet realMandatoryInputs) { + return lipoScannableMap == null ? ImmutableList.of() : Iterables.filter( + Iterables.transform( + Iterables.filter( + FileType.filter( + realMandatoryInputs, + CppFileTypes.C_SOURCE, CppFileTypes.CPP_SOURCE, + CppFileTypes.ASSEMBLER_WITH_C_PREPROCESSOR), + Predicates.not(Predicates.equalTo(getSourceFile()))), + Functions.forMap(lipoScannableMap, null)), + Predicates.notNull()); + } + + /** + * Builds the Action as configured and returns the to be generated Artifact. + * + *

This method may be called multiple times to create multiple compile + * actions (usually after calling some setters to modify the generated + * action). + */ + public CppCompileAction build() { + // Configuration can be null in tests. + NestedSetBuilder realMandatoryInputsBuilder = NestedSetBuilder.compileOrder(); + realMandatoryInputsBuilder.addTransitive(mandatoryInputsBuilder.build()); + if (tempOutputFile == null && configuration != null + && !configuration.getFragment(CppConfiguration.class).shouldScanIncludes()) { + realMandatoryInputsBuilder.addTransitive(context.getDeclaredIncludeSrcs()); + } + realMandatoryInputsBuilder.addTransitive(context.getAdditionalInputs()); + realMandatoryInputsBuilder.addTransitive(pluginInputsBuilder.build()); + realMandatoryInputsBuilder.add(sourceFile); + boolean fake = tempOutputFile != null; + + // Copying the collections is needed to make the builder reusable. + if (fake) { + return new FakeCppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration, + sourceFile, sourceLabel, realMandatoryInputsBuilder.build(), outputFile, tempOutputFile, + dotdFile, configuration, cppConfiguration, context, ImmutableList.copyOf(copts), + ImmutableList.copyOf(pluginOpts), getNocoptPredicate(nocopts), + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp); + } else { + NestedSet realMandatoryInputs = realMandatoryInputsBuilder.build(); + + return new CppCompileAction(owner, ImmutableList.copyOf(features), featureConfiguration, + sourceFile, sourceLabel, realMandatoryInputs, outputFile, dotdFile, + gcnoFile, getDwoFile(outputFile, analysisEnvironment, cppConfiguration), + optionalSourceFile, configuration, cppConfiguration, context, + actionContext, ImmutableList.copyOf(copts), + ImmutableList.copyOf(pluginOpts), + getNocoptPredicate(nocopts), + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, + includeResolver, getLipoScannables(realMandatoryInputs), actionClassId, + compileHeaderModules); + } + } + + /** + * Sets the feature configuration to be used for the action. + */ + public CppCompileActionBuilder setFeatureConfiguration( + FeatureConfiguration featureConfiguration) { + this.featureConfiguration = featureConfiguration; + return this; + } + + public CppCompileActionBuilder setIncludeResolver(IncludeResolver includeResolver) { + this.includeResolver = includeResolver; + return this; + } + + public CppCompileActionBuilder setCppConfiguration(CppConfiguration cppConfiguration) { + this.cppConfiguration = cppConfiguration; + return this; + } + + public CppCompileActionBuilder setActionContext( + Class actionContext) { + this.actionContext = actionContext; + return this; + } + + public CppCompileActionBuilder setActionClassId(UUID uuid) { + this.actionClassId = uuid; + return this; + } + + public CppCompileActionBuilder setExtraSystemIncludePrefixes( + Collection extraSystemIncludePrefixes) { + this.extraSystemIncludePrefixes = ImmutableList.copyOf(extraSystemIncludePrefixes); + return this; + } + + public CppCompileActionBuilder addPluginInput(Artifact artifact) { + pluginInputsBuilder.add(artifact); + return this; + } + + public CppCompileActionBuilder clearPluginInputs() { + pluginInputsBuilder = NestedSetBuilder.stableOrder(); + return this; + } + + /** + * Set an optional source file (usually with metadata of the main source file). The optional + * source file can only be set once, whether via this method or through the constructor + * {@link #CppCompileActionBuilder(CppCompileActionBuilder)}. + */ + public CppCompileActionBuilder addOptionalSourceFile(Artifact artifact) { + Preconditions.checkState(optionalSourceFile == null, "%s %s", optionalSourceFile, artifact); + optionalSourceFile = artifact; + return this; + } + + public CppCompileActionBuilder addMandatoryInputs(Iterable artifacts) { + mandatoryInputsBuilder.addAll(artifacts); + return this; + } + + public CppCompileActionBuilder addTransitiveMandatoryInputs(NestedSet artifacts) { + mandatoryInputsBuilder.addTransitive(artifacts); + return this; + } + + public CppCompileActionBuilder setOutputFile(Artifact outputFile) { + this.outputFile = outputFile; + return this; + } + + /** + * The temp output file is not an artifact, since it does not appear in the outputs of the + * action. + * + *

This is theoretically a problem if that file already existed before, since then Blaze + * does not delete it before executing the rule, but 1. that only applies for local + * execution which does not happen very often and 2. it is only a problem if the compiler is + * affected by the presence of this file, which it should not be. + */ + public CppCompileActionBuilder setTempOutputFile(PathFragment tempOutputFile) { + this.tempOutputFile = tempOutputFile; + return this; + } + + @VisibleForTesting + public CppCompileActionBuilder setDotdFileForTesting(Artifact dotdFile) { + this.dotdFile = new DotdFile(dotdFile); + return this; + } + + public CppCompileActionBuilder setDotdFile(PathFragment outputName, String extension, + RuleContext ruleContext) { + if (configuration.getFragment(CppConfiguration.class).getInmemoryDotdFiles()) { + // Just set the path, no artifact is constructed + PathFragment file = FileSystemUtils.replaceExtension(outputName, extension); + Root root = configuration.getBinDirectory(); + dotdFile = new DotdFile(root.getExecPath().getRelative(file)); + } else { + dotdFile = new DotdFile(ruleContext.getRelatedArtifact(outputName, extension)); + } + return this; + } + + public CppCompileActionBuilder setGcnoFile(Artifact gcnoFile) { + this.gcnoFile = gcnoFile; + return this; + } + + public CppCompileActionBuilder addCopt(String copt) { + copts.add(copt); + return this; + } + + public CppCompileActionBuilder addPluginOpt(String opt) { + pluginOpts.add(opt); + return this; + } + + public CppCompileActionBuilder clearPluginOpts() { + pluginOpts.clear(); + return this; + } + + public CppCompileActionBuilder addCopts(Iterable copts) { + Iterables.addAll(this.copts, copts); + return this; + } + + public CppCompileActionBuilder addCopts(int position, Iterable copts) { + this.copts.addAll(position, ImmutableList.copyOf(copts)); + return this; + } + + public CppCompileActionBuilder addNocopts(Pattern nocopts) { + this.nocopts.add(nocopts); + return this; + } + + public CppCompileActionBuilder setContext(CppCompilationContext context) { + this.context = context; + return this; + } + + public CppCompileActionBuilder setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * Sets whether the CompileAction should use header modules for its compilation. + */ + public CppCompileActionBuilder setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + public CppCompileActionBuilder setFdoBuildStamp(String fdoBuildStamp) { + this.fdoBuildStamp = fdoBuildStamp; + return this; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java new file mode 100644 index 0000000000..4bbfa44d61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionContext.java @@ -0,0 +1,84 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.io.IOException; +import java.util.Collection; + +import javax.annotation.Nullable; + +/** + * Context for compiling plain C++. + */ +@ActionContextMarker(name = "C++") +public interface CppCompileActionContext extends ActionContext { + /** + * Reply for the execution of a C++ compilation. + */ + public interface Reply { + /** + * Returns the contents of the .d file. + */ + byte[] getContents() throws IOException; + } + + /** Does include scanning to find the list of files needed to execute the action. */ + public Collection findAdditionalInputs(CppCompileAction action, + ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException, ActionExecutionException; + + /** + * Executes the given action and return the reply of the executor. + */ + Reply execWithReply(CppCompileAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException; + + /** + * Returns the executor reply from an exec exception, if available. + */ + @Nullable Reply getReplyFromException( + ExecException e, CppCompileAction action); + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(CppCompileAction action); + + /** + * Returns where the action actually runs. + */ + String strategyLocality(); + + /** + * Returns whether include scanning needs to be run. + */ + boolean needsIncludeScanning(); + + /** + * Returns the include files that should be shipped to the executor in addition the ones that + * were declared. + */ + Collection getScannedIncludeFiles( + CppCompileAction action, ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java new file mode 100644 index 0000000000..c5cc9a57fc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java @@ -0,0 +1,1691 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.common.collect.Iterables; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.ViewCreationFailedException; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.rules.cpp.CppConfigurationLoader.CppConfigurationParameters; +import com.google.devtools.build.lib.rules.cpp.FdoSupport.FdoException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.syntax.SkylarkCallable; +import com.google.devtools.build.lib.syntax.SkylarkModule; +import com.google.devtools.build.lib.util.IncludeScanningUtil; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LinkingModeFlags; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.build.skyframe.SkyFunction.Environment; +import com.google.devtools.common.options.OptionsParsingException; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipException; + +/** + * This class represents the C/C++ parts of the {@link BuildConfiguration}, + * including the host architecture, target architecture, compiler version, and + * a standard library version. It has information about the tools locations and + * the flags required for compiling. + */ +@SkylarkModule(name = "cpp", doc = "A configuration fragment for C++") +@Immutable +public class CppConfiguration extends BuildConfiguration.Fragment { + /** + * An enumeration of all the tools that comprise a toolchain. + */ + public enum Tool { + AR("ar"), + CPP("cpp"), + GCC("gcc"), + GCOV("gcov"), + GCOVTOOL("gcov-tool"), + LD("ld"), + NM("nm"), + OBJCOPY("objcopy"), + OBJDUMP("objdump"), + STRIP("strip"), + DWP("dwp"); + + private final String namePart; + + private Tool(String namePart) { + this.namePart = namePart; + } + + public String getNamePart() { + return namePart; + } + } + + /** + * Values for the --hdrs_check option. + */ + public static enum HeadersCheckingMode { + /** Legacy behavior: Silently allow undeclared headers. */ + LOOSE, + /** Warn about undeclared headers. */ + WARN, + /** Disallow undeclared headers. */ + STRICT + } + + /** + * --dynamic_mode parses to DynamicModeFlag, but AUTO will be translated based on platform, + * resulting in a DynamicMode value. + */ + public enum DynamicMode { OFF, DEFAULT, FULLY } + + /** + * This enumeration is used for the --strip option. + */ + public static enum StripMode { + + ALWAYS("always"), // Always strip. + SOMETIMES("sometimes"), // Strip iff compilationMode == FASTBUILD. + NEVER("never"); // Never strip. + + private final String mode; + + private StripMode(String mode) { + this.mode = mode; + } + + @Override + public String toString() { + return mode; + } + } + + /** Storage for the libc label, if given. */ + public static class LibcTop implements Serializable { + private final Label label; + + LibcTop(Label label) { + Preconditions.checkArgument(label != null); + this.label = label; + } + + public Label getLabel() { + return label; + } + + public PathFragment getSysroot() { + return label.getPackageFragment(); + } + + @Override + public String toString() { + return label.toString(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (other instanceof LibcTop) { + return label.equals(((LibcTop) other).label); + } else { + return false; + } + } + + @Override + public int hashCode() { + return label.hashCode(); + } + } + + /** + * This macro will be passed as a command-line parameter (eg. -DBUILD_FDO_TYPE="LIPO"). + * For possible values see {@code CppModel.getFdoBuildStamp()}. + */ + public static final String FDO_STAMP_MACRO = "BUILD_FDO_TYPE"; + + /** + * Represents an optional flag that can be toggled using the package features mechanism. + */ + @VisibleForTesting + static class OptionalFlag implements Serializable { + private final String name; + private final List flags; + + @VisibleForTesting + OptionalFlag(String name, List flags) { + this.name = name; + this.flags = flags; + } + + private List getFlags() { + return flags; + } + + private String getName() { + return name; + } + } + + @VisibleForTesting + static class FlagList implements Serializable { + private List prefixFlags; + private List optionalFlags; + private List suffixFlags; + + @VisibleForTesting + FlagList(List prefixFlags, + List optionalFlags, + List suffixFlags) { + this.prefixFlags = prefixFlags; + this.optionalFlags = optionalFlags; + this.suffixFlags = suffixFlags; + } + + @VisibleForTesting + List evaluate(Collection features) { + ImmutableList.Builder result = ImmutableList.builder(); + result.addAll(prefixFlags); + for (OptionalFlag optionalFlag : optionalFlags) { + // The flag is added if the default is true and the flag is not specified, + // or if the default is false and the flag is specified. + if (features.contains(optionalFlag.getName())) { + result.addAll(optionalFlag.getFlags()); + } + } + + result.addAll(suffixFlags); + return result.build(); + } + } + + private final Label crosstoolTop; + private final String hostSystemName; + private final String compiler; + private final String targetCpu; + private final String targetSystemName; + private final String targetLibc; + private final LipoMode lipoMode; + private final PathFragment crosstoolTopPathFragment; + + private final String abi; + private final String abiGlibcVersion; + + private final String toolchainIdentifier; + private final String cacheKey; + + private final CcToolchainFeatures toolchainFeatures; + private final boolean supportsGoldLinker; + private final boolean supportsThinArchives; + private final boolean supportsStartEndLib; + private final boolean supportsInterfaceSharedObjects; + private final boolean supportsEmbeddedRuntimes; + private final boolean supportsFission; + + // We encode three states with two booleans: + // (1) (false false) -> no pic code + // (2) (true false) -> shared libraries as pic, but not binaries + // (3) (true true) -> both shared libraries and binaries as pic + private final boolean toolchainNeedsPic; + private final boolean usePicForBinaries; + + private final FdoSupport fdoSupport; + + // TODO(bazel-team): All these labels (except for ccCompilerRuleLabel) can be removed once the + // transition to the cc_compiler rule is complete. + private final Label libcLabel; + private final Label staticRuntimeLibsLabel; + private final Label dynamicRuntimeLibsLabel; + private final Label ccToolchainLabel; + + private final PathFragment sysroot; + private final PathFragment runtimeSysroot; + private final List builtInIncludeDirectories; + + private final Map toolPaths; + private final PathFragment ldExecutable; + + // Only used during construction. + private final List commonLinkOptions; + private final ListMultimap linkOptionsFromCompilationMode; + private final ListMultimap linkOptionsFromLipoMode; + private final ListMultimap linkOptionsFromLinkingMode; + + private final FlagList compilerFlags; + private final FlagList cxxFlags; + private final FlagList unfilteredCompilerFlags; + private final List cOptions; + + private FlagList fullyStaticLinkFlags; + private FlagList mostlyStaticLinkFlags; + private FlagList mostlyStaticSharedLinkFlags; + private FlagList dynamicLinkFlags; + private FlagList dynamicLibraryLinkFlags; + private final List testOnlyLinkFlags; + + private final List linkOptions; + + private final List objcopyOptions; + private final List ldOptions; + private final List arOptions; + private final List arThinArchivesOptions; + + private final Map additionalMakeVariables; + + private final CppOptions cppOptions; + + // The dynamic mode for linking. + private final DynamicMode dynamicMode; + private final boolean stripBinaries; + private final ImmutableMap commandLineDefines; + private final String solibDirectory; + private final CompilationMode compilationMode; + private final Path execRoot; + /** + * If true, the ConfiguredTarget is only used to get the necessary cross-referenced + * CppCompilationContexts, but registering build actions is disabled. + */ + private final boolean lipoContextCollector; + private final Root greppedIncludesDirectory; + + protected CppConfiguration(CppConfigurationParameters params) + throws InvalidConfigurationException { + CrosstoolConfig.CToolchain toolchain = params.toolchain; + cppOptions = params.buildOptions.get(CppOptions.class); + this.hostSystemName = toolchain.getHostSystemName(); + this.compiler = toolchain.getCompiler(); + this.targetCpu = toolchain.getTargetCpu(); + this.lipoMode = cppOptions.getLipoMode(); + this.targetSystemName = toolchain.getTargetSystemName(); + this.targetLibc = toolchain.getTargetLibc(); + this.crosstoolTop = params.crosstoolTop; + this.ccToolchainLabel = params.ccToolchainLabel; + this.compilationMode = + params.buildOptions.get(BuildConfiguration.Options.class).compilationMode; + this.lipoContextCollector = cppOptions.lipoCollector; + this.execRoot = params.execRoot; + + // Note that the grepped includes directory is not configuration-specific; the paths of the + // files within that directory, however, are configuration-specific. + this.greppedIncludesDirectory = Root.asDerivedRoot(execRoot, + execRoot.getRelative(IncludeScanningUtil.GREPPED_INCLUDES)); + + this.crosstoolTopPathFragment = crosstoolTop.getPackageFragment(); + + try { + this.staticRuntimeLibsLabel = + crosstoolTop.getRelative(toolchain.hasStaticRuntimesFilegroup() ? + toolchain.getStaticRuntimesFilegroup() : "static-runtime-libs-" + targetCpu); + this.dynamicRuntimeLibsLabel = + crosstoolTop.getRelative(toolchain.hasDynamicRuntimesFilegroup() ? + toolchain.getDynamicRuntimesFilegroup() : "dynamic-runtime-libs-" + targetCpu); + } catch (SyntaxException e) { + // All of the above label.getRelative() calls are valid labels, and the crosstool_top + // was already checked earlier in the process. + throw new AssertionError(e); + } + + if (cppOptions.lipoMode == LipoMode.BINARY) { + // TODO(bazel-team): implement dynamic linking with LIPO + this.dynamicMode = DynamicMode.OFF; + } else { + switch (cppOptions.dynamicMode) { + case DEFAULT: + this.dynamicMode = DynamicMode.DEFAULT; break; + case OFF: this.dynamicMode = DynamicMode.OFF; break; + case FULLY: this.dynamicMode = DynamicMode.FULLY; break; + default: throw new IllegalStateException("Invalid dynamicMode."); + } + } + + this.fdoSupport = new FdoSupport( + params.buildOptions.get(CppOptions.class).fdoInstrument, params.fdoZip, + cppOptions.lipoMode, execRoot); + + this.stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS || + (cppOptions.stripBinaries == StripMode.SOMETIMES && + compilationMode == CompilationMode.FASTBUILD)); + + CrosstoolConfigurationIdentifier crosstoolConfig = + CrosstoolConfigurationIdentifier.fromToolchain(toolchain); + Preconditions.checkState(crosstoolConfig.getCpu().equals(targetCpu)); + Preconditions.checkState(crosstoolConfig.getCompiler().equals(compiler)); + Preconditions.checkState(crosstoolConfig.getLibc().equals(targetLibc)); + + this.solibDirectory = "_solib_" + targetCpu; + + this.toolchainIdentifier = toolchain.getToolchainIdentifier(); + this.cacheKey = this + ":" + crosstoolTop + ":" + params.cacheKeySuffix + ":" + + lipoContextCollector; + + this.toolchainFeatures = new CcToolchainFeatures(toolchain); + this.supportsGoldLinker = toolchain.getSupportsGoldLinker(); + this.supportsThinArchives = toolchain.getSupportsThinArchives(); + this.supportsStartEndLib = toolchain.getSupportsStartEndLib(); + this.supportsInterfaceSharedObjects = toolchain.getSupportsInterfaceSharedObjects(); + this.supportsEmbeddedRuntimes = toolchain.getSupportsEmbeddedRuntimes(); + this.supportsFission = toolchain.getSupportsFission(); + this.toolchainNeedsPic = toolchain.getNeedsPic(); + this.usePicForBinaries = + toolchain.getNeedsPic() && compilationMode != CompilationMode.OPT; + + this.toolPaths = Maps.newHashMap(); + for (CrosstoolConfig.ToolPath tool : toolchain.getToolPathList()) { + PathFragment path = new PathFragment(tool.getPath()); + if (!path.isNormalized()) { + throw new IllegalArgumentException("The include path '" + tool.getPath() + + "' is not normalized."); + } + toolPaths.put(tool.getName(), crosstoolTopPathFragment.getRelative(path)); + } + + if (toolPaths.isEmpty()) { + // If no paths are specified, we just use the names of the tools as the path. + for (Tool tool : Tool.values()) { + toolPaths.put(tool.getNamePart(), + crosstoolTopPathFragment.getRelative(tool.getNamePart())); + } + } else { + Iterable neededTools = Iterables.filter(EnumSet.allOf(Tool.class), + new Predicate() { + @Override + public boolean apply(Tool tool) { + if (tool == Tool.DWP) { + // When fission is unsupported, don't check for the dwp tool. + return supportsFission(); + } else if (tool == Tool.GCOVTOOL) { + // gcov-tool is optional, don't check whether it's present + return false; + } else { + return true; + } + } + }); + for (Tool tool : neededTools) { + if (!toolPaths.containsKey(tool.getNamePart())) { + throw new IllegalArgumentException("Tool path for '" + tool.getNamePart() + + "' is missing"); + } + } + } + + // We can't use an ImmutableMap.Builder here; we need the ability (at least + // in tests) to add entries with keys that are already in the map, and only + // HashMap supports this (by replacing the existing entry under the key). + Map commandLineDefinesBuilder = new HashMap<>(); + for (Map.Entry define : cppOptions.commandLineDefinedVariables) { + commandLineDefinesBuilder.put(define.getKey(), define.getValue()); + } + commandLineDefines = ImmutableMap.copyOf(commandLineDefinesBuilder); + + ListMultimap cFlags = ArrayListMultimap.create(); + ListMultimap cxxFlags = ArrayListMultimap.create(); + linkOptionsFromCompilationMode = ArrayListMultimap.create(); + for (CrosstoolConfig.CompilationModeFlags flags : toolchain.getCompilationModeFlagsList()) { + // Remove this when CROSSTOOL files no longer contain 'coverage'. + if (flags.getMode() == CrosstoolConfig.CompilationMode.COVERAGE) { + continue; + } + CompilationMode realmode = importCompilationMode(flags.getMode()); + cFlags.putAll(realmode, flags.getCompilerFlagList()); + cxxFlags.putAll(realmode, flags.getCxxFlagList()); + linkOptionsFromCompilationMode.putAll(realmode, flags.getLinkerFlagList()); + } + + ListMultimap lipoCFlags = ArrayListMultimap.create(); + ListMultimap lipoCxxFlags = ArrayListMultimap.create(); + linkOptionsFromLipoMode = ArrayListMultimap.create(); + for (CrosstoolConfig.LipoModeFlags flags : toolchain.getLipoModeFlagsList()) { + LipoMode realmode = flags.getMode(); + lipoCFlags.putAll(realmode, flags.getCompilerFlagList()); + lipoCxxFlags.putAll(realmode, flags.getCxxFlagList()); + linkOptionsFromLipoMode.putAll(realmode, flags.getLinkerFlagList()); + } + + linkOptionsFromLinkingMode = ArrayListMultimap.create(); + for (LinkingModeFlags flags : toolchain.getLinkingModeFlagsList()) { + LinkingMode realmode = importLinkingMode(flags.getMode()); + linkOptionsFromLinkingMode.putAll(realmode, flags.getLinkerFlagList()); + } + + this.commonLinkOptions = ImmutableList.copyOf(toolchain.getLinkerFlagList()); + dynamicLibraryLinkFlags = new FlagList( + ImmutableList.copyOf(toolchain.getDynamicLibraryLinkerFlagList()), + convertOptionalOptions(toolchain.getOptionalDynamicLibraryLinkerFlagList()), + Collections.emptyList()); + this.objcopyOptions = ImmutableList.copyOf(toolchain.getObjcopyEmbedFlagList()); + this.ldOptions = ImmutableList.copyOf(toolchain.getLdEmbedFlagList()); + this.arOptions = copyOrDefaultIfEmpty(toolchain.getArFlagList(), "rcsD"); + this.arThinArchivesOptions = copyOrDefaultIfEmpty( + toolchain.getArThinArchivesFlagList(), "rcsDT"); + + this.abi = toolchain.getAbiVersion(); + this.abiGlibcVersion = toolchain.getAbiLibcVersion(); + + // The default value for optional string attributes is the empty string. + PathFragment defaultSysroot = toolchain.getBuiltinSysroot().length() == 0 + ? null + : new PathFragment(toolchain.getBuiltinSysroot()); + if ((defaultSysroot != null) && !defaultSysroot.isNormalized()) { + throw new IllegalArgumentException("The built-in sysroot '" + defaultSysroot + + "' is not normalized."); + } + + if ((cppOptions.libcTop != null) && (defaultSysroot == null)) { + throw new InvalidConfigurationException("The selected toolchain " + toolchainIdentifier + + " does not support setting --grte_top."); + } + LibcTop libcTop = cppOptions.libcTop; + if ((libcTop == null) && !toolchain.getDefaultGrteTop().isEmpty()) { + try { + libcTop = new CppOptions.LibcTopConverter().convert(toolchain.getDefaultGrteTop()); + } catch (OptionsParsingException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } + } + if ((libcTop != null) && (libcTop.getLabel() != null)) { + libcLabel = libcTop.getLabel(); + } else { + libcLabel = null; + } + + ImmutableList.Builder builtInIncludeDirectoriesBuilder + = ImmutableList.builder(); + sysroot = libcTop == null ? defaultSysroot : libcTop.getSysroot(); + for (String s : toolchain.getCxxBuiltinIncludeDirectoryList()) { + builtInIncludeDirectoriesBuilder.add( + resolveIncludeDir(s, sysroot, crosstoolTopPathFragment)); + } + builtInIncludeDirectories = builtInIncludeDirectoriesBuilder.build(); + + // The runtime sysroot should really be set from --grte_top. However, currently libc has no + // way to set the sysroot. The CROSSTOOL file does set the runtime sysroot, in the + // builtin_sysroot field. This implies that you can not arbitrarily mix and match Crosstool + // and libc versions, you must always choose compatible ones. + runtimeSysroot = defaultSysroot; + + String sysrootFlag; + if (sysroot != null && !sysroot.equals(defaultSysroot)) { + // Only specify the --sysroot option if it is different from the built-in one. + sysrootFlag = "--sysroot=" + sysroot; + } else { + sysrootFlag = null; + } + + ImmutableList.Builder unfilteredCoptsBuilder = ImmutableList.builder(); + if (sysrootFlag != null) { + unfilteredCoptsBuilder.add(sysrootFlag); + } + unfilteredCoptsBuilder.addAll(toolchain.getUnfilteredCxxFlagList()); + unfilteredCompilerFlags = new FlagList( + unfilteredCoptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalUnfilteredCxxFlagList()), + Collections.emptyList()); + + ImmutableList.Builder linkoptsBuilder = ImmutableList.builder(); + linkoptsBuilder.addAll(cppOptions.linkoptList); + if (cppOptions.experimentalOmitfp) { + linkoptsBuilder.add("-Wl,--eh-frame-hdr"); + } + if (sysrootFlag != null) { + linkoptsBuilder.add(sysrootFlag); + } + this.linkOptions = linkoptsBuilder.build(); + + ImmutableList.Builder coptsBuilder = ImmutableList.builder() + .addAll(toolchain.getCompilerFlagList()) + .addAll(cFlags.get(compilationMode)) + .addAll(lipoCFlags.get(cppOptions.getLipoMode())); + if (cppOptions.experimentalOmitfp) { + coptsBuilder.add("-fomit-frame-pointer"); + coptsBuilder.add("-fasynchronous-unwind-tables"); + coptsBuilder.add("-DNO_FRAME_POINTER"); + } + this.compilerFlags = new FlagList( + coptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalCompilerFlagList()), + cppOptions.coptList); + + this.cOptions = ImmutableList.copyOf(cppOptions.conlyoptList); + + ImmutableList.Builder cxxOptsBuilder = ImmutableList.builder() + .addAll(toolchain.getCxxFlagList()) + .addAll(cxxFlags.get(compilationMode)) + .addAll(lipoCxxFlags.get(cppOptions.getLipoMode())); + + this.cxxFlags = new FlagList( + cxxOptsBuilder.build(), + convertOptionalOptions(toolchain.getOptionalCxxFlagList()), + cppOptions.cxxoptList); + + this.ldExecutable = getToolPathFragment(CppConfiguration.Tool.LD); + + boolean stripBinaries = (cppOptions.stripBinaries == StripMode.ALWAYS) || + ((cppOptions.stripBinaries == StripMode.SOMETIMES) && + (compilationMode == CompilationMode.FASTBUILD)); + + fullyStaticLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.FULLY_STATIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.emptyList()); + mostlyStaticLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.MOSTLY_STATIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.emptyList()); + mostlyStaticSharedLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, + LinkingMode.MOSTLY_STATIC_LIBRARIES, ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.emptyList()); + dynamicLinkFlags = new FlagList( + configureLinkerOptions(compilationMode, lipoMode, LinkingMode.DYNAMIC, + ldExecutable, stripBinaries), + convertOptionalOptions(toolchain.getOptionalLinkerFlagList()), + Collections.emptyList()); + testOnlyLinkFlags = ImmutableList.copyOf(toolchain.getTestOnlyLinkerFlagList()); + + Map makeVariablesBuilder = new HashMap<>(); + // The following are to be used to allow some build rules to avoid the limits on stack frame + // sizes and variable-length arrays. Ensure that these are always set. + makeVariablesBuilder.put("STACK_FRAME_UNLIMITED", ""); + makeVariablesBuilder.put("CC_FLAGS", ""); + for (CrosstoolConfig.MakeVariable variable : toolchain.getMakeVariableList()) { + makeVariablesBuilder.put(variable.getName(), variable.getValue()); + } + if (sysrootFlag != null) { + String ccFlags = makeVariablesBuilder.get("CC_FLAGS"); + ccFlags = ccFlags.isEmpty() ? sysrootFlag : ccFlags + " " + sysrootFlag; + makeVariablesBuilder.put("CC_FLAGS", ccFlags); + } + this.additionalMakeVariables = ImmutableMap.copyOf(makeVariablesBuilder); + } + + private List convertOptionalOptions( + List optionalFlagList) + throws IllegalArgumentException { + List result = new ArrayList<>(); + + for (CrosstoolConfig.CToolchain.OptionalFlag crosstoolOptionalFlag : optionalFlagList) { + String name = crosstoolOptionalFlag.getDefaultSettingName(); + result.add(new OptionalFlag( + name, + ImmutableList.copyOf(crosstoolOptionalFlag.getFlagList()))); + } + + return result; + } + + private static ImmutableList copyOrDefaultIfEmpty(List list, + String defaultValue) { + return list.isEmpty() ? ImmutableList.of(defaultValue) : ImmutableList.copyOf(list); + } + + @VisibleForTesting + static CompilationMode importCompilationMode(CrosstoolConfig.CompilationMode mode) { + return CompilationMode.valueOf(mode.name()); + } + + @VisibleForTesting + static LinkingMode importLinkingMode(CrosstoolConfig.LinkingMode mode) { + return LinkingMode.valueOf(mode.name()); + } + + private static final PathFragment SYSROOT_FRAGMENT = new PathFragment("%sysroot%"); + + /** + * Resolve the given include directory. If it is not absolute, it is + * interpreted relative to the crosstool top. If it starts with %sysroot%/, + * that part is replaced with the actual sysroot. + */ + static PathFragment resolveIncludeDir(String s, PathFragment sysroot, + PathFragment crosstoolTopPathFragment) { + PathFragment path = new PathFragment(s); + if (!path.isNormalized()) { + throw new IllegalArgumentException("The include path '" + s + "' is not normalized."); + } + if (path.startsWith(SYSROOT_FRAGMENT)) { + if (sysroot == null) { + throw new IllegalArgumentException("A %sysroot% prefix is only allowed if the " + + "default_sysroot option is set"); + } + return sysroot.getRelative(path.relativeTo(SYSROOT_FRAGMENT)); + } else { + return crosstoolTopPathFragment.getRelative(path); + } + } + + /** + * Returns the configuration-independent grepped-includes directory. + */ + public Root getGreppedIncludesDirectory() { + return greppedIncludesDirectory; + } + + @VisibleForTesting + List configureLinkerOptions( + CompilationMode compilationMode, LipoMode lipoMode, LinkingMode linkingMode, + PathFragment ldExecutable, boolean stripBinaries) { + List result = new ArrayList<>(); + result.addAll(commonLinkOptions); + + result.add("-B" + ldExecutable.getParentDirectory().getPathString()); + if (stripBinaries) { + result.add("-Wl,-S"); + } + + result.addAll(linkOptionsFromCompilationMode.get(compilationMode)); + result.addAll(linkOptionsFromLipoMode.get(lipoMode)); + result.addAll(linkOptionsFromLinkingMode.get(linkingMode)); + return ImmutableList.copyOf(result); + } + + /** + * Returns the toolchain identifier, which uniquely identifies the compiler + * version, target libc version, target cpu, and LIPO linkage. + */ + public String getToolchainIdentifier() { + return toolchainIdentifier; + } + + /** + * Returns the system name which is required by the toolchain to run. + */ + public String getHostSystemName() { + return hostSystemName; + } + + @Override + public String toString() { + return toolchainIdentifier; + } + + /** + * Returns the compiler version string (e.g. "gcc-4.1.1"). + */ + @SkylarkCallable(name = "compiler", structField = true, doc = "C++ compiler.") + public String getCompiler() { + return compiler; + } + + /** + * Returns the libc version string (e.g. "glibc-2.2.2"). + */ + public String getTargetLibc() { + return targetLibc; + } + + /** + * Returns the target architecture using blaze-specific constants (e.g. "piii"). + */ + @SkylarkCallable(name = "cpu", structField = true, doc = "Target CPU of the C++ toolchain.") + public String getTargetCpu() { + return targetCpu; + } + + /** + * Returns the path fragment that is either absolute or relative to the + * execution root that can be used to execute the given tool. + * + *

Note that you must not use this method to get the linker location, but + * use {@link #getLdExecutable} instead! + */ + public PathFragment getToolPathFragment(CppConfiguration.Tool tool) { + return toolPaths.get(tool.getNamePart()); + } + + /** + * Returns a label that forms a dependency to the files required for the + * sysroot that is used. + */ + public Label getLibcLabel() { + return libcLabel; + } + + /** + * Returns a label that references the library files needed to statically + * link the C++ runtime (i.e. libgcc.a, libgcc_eh.a, libstdc++.a) for the + * target architecture. + */ + public Label getStaticRuntimeLibsLabel() { + return supportsEmbeddedRuntimes() ? staticRuntimeLibsLabel : null; + } + + /** + * Returns a label that references the library files needed to dynamically + * link the C++ runtime (i.e. libgcc_s.so, libstdc++.so) for the target + * architecture. + */ + public Label getDynamicRuntimeLibsLabel() { + return supportsEmbeddedRuntimes() ? dynamicRuntimeLibsLabel : null; + } + + /** + * Returns the label of the cc_compiler rule for the C++ configuration. + */ + public Label getCcToolchainRuleLabel() { + return ccToolchainLabel; + } + + /** + * Returns the abi we're using, which is a gcc version. E.g.: "gcc-3.4". + * Note that in practice we might be using gcc-3.4 as ABI even when compiling + * with gcc-4.1.0, because ABIs are backwards compatible. + */ + // TODO(bazel-team): The javadoc should clarify how this is used in Blaze. + public String getAbi() { + return abi; + } + + /** + * Returns the glibc version used by the abi we're using. This is a + * glibc version number (e.g., "2.2.2"). Note that in practice we + * might be using glibc 2.2.2 as ABI even when compiling with + * gcc-4.2.2, gcc-4.3.1, or gcc-4.4.0 (which use glibc 2.3.6), + * because ABIs are backwards compatible. + */ + // TODO(bazel-team): The javadoc should clarify how this is used in Blaze. + public String getAbiGlibcVersion() { + return abiGlibcVersion; + } + + /** + * Returns the configured features of the toolchain. Rules should not call this directly, but + * instead use {@code CcToolchainProvider.getFeatures}. + */ + public CcToolchainFeatures getFeatures() { + return toolchainFeatures; + } + + /** + * Returns whether the toolchain supports the gold linker. + */ + public boolean supportsGoldLinker() { + return supportsGoldLinker; + } + + /** + * Returns whether the toolchain supports thin archives. + */ + public boolean supportsThinArchives() { + return supportsThinArchives; + } + + /** + * Returns whether the toolchain supports the --start-lib/--end-lib options. + */ + public boolean supportsStartEndLib() { + return supportsStartEndLib; + } + + /** + * Returns whether build_interface_so can build interface shared objects for this toolchain. + * Should be true if this toolchain generates ELF objects. + */ + public boolean supportsInterfaceSharedObjects() { + return supportsInterfaceSharedObjects; + } + + /** + * Returns whether the toolchain supports linking C/C++ runtime libraries + * supplied inside the toolchain distribution. + */ + public boolean supportsEmbeddedRuntimes() { + return supportsEmbeddedRuntimes; + } + + /** + * Returns whether the toolchain supports EXEC_ORIGIN libraries resolution. + */ + public boolean supportsExecOrigin() { + // We're rolling out support for this in the same release that also supports embedded runtimes. + return supportsEmbeddedRuntimes; + } + + /** + * Returns whether the toolchain supports "Fission" C++ builds, i.e. builds + * where compilation partitions object code and debug symbols into separate + * output files. + */ + public boolean supportsFission() { + return supportsFission; + } + + /** + * Returns whether shared libraries must be compiled with position + * independent code on this platform. + */ + public boolean toolchainNeedsPic() { + return toolchainNeedsPic; + } + + /** + * Returns whether binaries must be compiled with position independent code. + */ + public boolean usePicForBinaries() { + return usePicForBinaries; + } + + /** + * Returns the type of archives being used. + */ + public Link.ArchiveType archiveType() { + if (useStartEndLib()) { + return Link.ArchiveType.START_END_LIB; + } + if (useThinArchives()) { + return Link.ArchiveType.THIN; + } + return Link.ArchiveType.FAT; + } + + /** + * Returns the ar flags to be used. + */ + public List getArFlags(boolean thinArchives) { + return thinArchives ? arThinArchivesOptions : arOptions; + } + + /** + * Returns a string that uniquely identifies the toolchain. + */ + @Override + public String cacheKey() { + return cacheKey; + } + + /** + * Returns the built-in list of system include paths for the toolchain + * compiler. All paths in this list should be relative to the exec directory. + * They may be absolute if they are also installed on the remote build nodes or + * for local compilation. + */ + public List getBuiltInIncludeDirectories() { + return builtInIncludeDirectories; + } + + /** + * Returns the sysroot to be used. If the toolchain compiler does not support + * different sysroots, or the sysroot is the same as the default sysroot, then + * this method returns null. + */ + public PathFragment getSysroot() { + return sysroot; + } + + /** + * Returns the run time sysroot, which is where the dynamic linker + * and system libraries are found at runtime. This is usually an absolute path. If the + * toolchain compiler does not support sysroots, then this method returns null. + */ + public PathFragment getRuntimeSysroot() { + return runtimeSysroot; + } + + /** + * Returns the default options to use for compiling C, C++, and assembler. + * This is just the options that should be used for all three languages. + * There may be additional C-specific or C++-specific options that should be used, + * in addition to the ones returned by this method; + */ + public List getCompilerOptions(Collection features) { + return compilerFlags.evaluate(features); + } + + /** + * Returns the list of additional C-specific options to use for compiling + * C. These should be go on the command line after the common options + * returned by {@link #getCompilerOptions}. + */ + public List getCOptions() { + return cOptions; + } + + /** + * Returns the list of additional C++-specific options to use for compiling + * C++. These should be go on the command line after the common options + * returned by {@link #getCompilerOptions}. + */ + public List getCxxOptions(Collection features) { + return cxxFlags.evaluate(features); + } + + /** + * Returns the default list of options which cannot be filtered by BUILD + * rules. These should be appended to the command line after filtering. + */ + public List getUnfilteredCompilerOptions(Collection features) { + return unfilteredCompilerFlags.evaluate(features); + } + + /** + * Returns the set of command-line linker options, including any flags + * inferred from the command-line options. + * + * @see Link + */ + // TODO(bazel-team): Clean up the linker options computation! + public List getLinkOptions() { + return linkOptions; + } + + /** + * Returns the immutable list of linker options for fully statically linked + * outputs. Does not include command-line options passed via --linkopt or + * --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List getFullyStaticLinkOptions(Collection features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions(mostlyStaticLinkFlags, features); + } else { + return fullyStaticLinkFlags.evaluate(features); + } + } + + /** + * Returns the immutable list of linker options for mostly statically linked + * outputs. Does not include command-line options passed via --linkopt or + * --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List getMostlyStaticLinkOptions(Collection features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions( + supportsEmbeddedRuntimes ? mostlyStaticSharedLinkFlags : dynamicLinkFlags, + features); + } else { + return mostlyStaticLinkFlags.evaluate(features); + } + } + + /** + * Returns the immutable list of linker options for artifacts that are not + * fully or mostly statically linked. Does not include command-line options + * passed via --linkopt or --linkopts. + * + * @param features default settings affecting this link + * @param sharedLib true if the output is a shared lib, false if it's an executable + */ + public List getDynamicLinkOptions(Collection features, + boolean sharedLib) { + if (sharedLib) { + return getSharedLibraryLinkOptions(dynamicLinkFlags, features); + } else { + return dynamicLinkFlags.evaluate(features); + } + } + + /** + * Returns link options for the specified flag list, combined with universal options + * for all shared libraries (regardless of link staticness). + */ + private List getSharedLibraryLinkOptions(FlagList flags, + Collection features) { + return ImmutableList.builder() + .addAll(flags.evaluate(features)) + .addAll(dynamicLibraryLinkFlags.evaluate(features)) + .build(); + } + + /** + * Returns test-only link options such that certain test-specific features can be configured + * separately (e.g. lazy binding). + */ + public List getTestOnlyLinkOptions() { + return testOnlyLinkFlags; + } + + + /** + * Returns the list of options to be used with 'objcopy' when converting + * binary files to object files, or {@code null} if this operation is not + * supported. + */ + public List getObjCopyOptionsForEmbedding() { + return objcopyOptions; + } + + /** + * Returns the list of options to be used with 'ld' when converting + * binary files to object files, or {@code null} if this operation is not + * supported. + */ + public List getLdOptionsForEmbedding() { + return ldOptions; + } + + /** + * Returns a map of additional make variables for use by {@link + * BuildConfiguration}. These are to used to allow some build rules to + * avoid the limits on stack frame sizes and variable-length arrays. + * + *

The returned map must contain an entry for {@code STACK_FRAME_UNLIMITED}, + * though the entry may be an empty string. + */ + @VisibleForTesting + public Map getAdditionalMakeVariables() { + return additionalMakeVariables; + } + + /** + * Returns the execution path to the linker binary to use for this build. + * Relative paths are relative to the execution root. + */ + public PathFragment getLdExecutable() { + return ldExecutable; + } + + /** + * Returns the dynamic linking mode (full, off, or default). + */ + public DynamicMode getDynamicMode() { + return dynamicMode; + } + + /* + * If true then the directory name for non-LIPO targets will have a '-lipodata' suffix in + * AutoFDO mode. + */ + public boolean getAutoFdoLipoData() { + return cppOptions.autoFdoLipoData; + } + + /** + * Returns the STL label if given on the command line. {@code null} + * otherwise. + */ + public Label getStl() { + return cppOptions.stl; + } + + /* + * Returns the command-line "Make" variable overrides. + */ + @Override + public ImmutableMap getCommandLineDefines() { + return commandLineDefines; + } + + /** + * Returns the command-line override value for the specified "Make" variable + * for this configuration, or null if none. + */ + public String getMakeVariableOverride(String var) { + return commandLineDefines.get(var); + } + + public boolean shouldScanIncludes() { + return cppOptions.scanIncludes; + } + + /** + * Returns the currently active LIPO compilation mode. + */ + public LipoMode getLipoMode() { + return cppOptions.lipoMode; + } + + public boolean isFdo() { + return cppOptions.isFdo(); + } + + public boolean isLipoOptimization() { + // The LIPO optimization bits are set in the LIPO context collector configuration, too. + return cppOptions.isLipoOptimization() && !isLipoContextCollector(); + } + + public boolean isLipoOptimizationOrInstrumentation() { + return cppOptions.isLipoOptimizationOrInstrumentation(); + } + + /** + * Returns true if it is AutoFDO LIPO build. + */ + public boolean isAutoFdoLipo() { + return cppOptions.fdoOptimize != null && FdoSupport.isAutoFdo(cppOptions.fdoOptimize) + && getLipoMode() != LipoMode.OFF; + } + + /** + * Returns the default header check mode. + */ + public HeadersCheckingMode getHeadersCheckingMode() { + return cppOptions.headersCheckingMode; + } + + /** + * Returns whether or not to strip the binaries. + */ + public boolean shouldStripBinaries() { + return stripBinaries; + } + + /** + * Returns the additional options to pass to strip when generating a + * {@code .stripped} binary by this build. + */ + public List getStripOpts() { + return cppOptions.stripoptList; + } + + /** + * Returns whether temporary outputs from gcc will be saved. + */ + public boolean getSaveTemps() { + return cppOptions.saveTemps; + } + + /** + * Returns the {@link PerLabelOptions} to apply to the gcc command line, if + * the label of the compiled file matches the regular expression. + */ + public List getPerFileCopts() { + return cppOptions.perFileCopts; + } + + public Label getLipoContextLabel() { + return cppOptions.getLipoContextLabel(); + } + + /** + * Returns the custom malloc library label. + */ + public Label customMalloc() { + return cppOptions.customMalloc; + } + + /** + * Returns the extra warnings enabled for C compilation. + */ + public List getCWarns() { + return cppOptions.cWarns; + } + + /** + * Returns true if mostly-static C++ binaries should be skipped. + */ + public boolean skipStaticOutputs() { + return cppOptions.skipStaticOutputs; + } + + /** + * Returns true if Fission is specified for this build and supported by the crosstool. + */ + public boolean useFission() { + return cppOptions.fissionModes.contains(compilationMode) && supportsFission(); + } + + /** + * Returns true if all C++ compilations should produce position-independent code, links should + * produce position-independent executables, and dependencies with equivalent pre-built pic and + * nopic versions should apply the pic versions. Returns false if default settings should be + * applied (i.e. make no special provisions for pic code). + */ + public boolean forcePic() { + return cppOptions.forcePic; + } + + public boolean useStartEndLib() { + return cppOptions.useStartEndLib && supportsStartEndLib(); + } + + public boolean useThinArchives() { + return cppOptions.useThinArchives && supportsThinArchives(); + } + + /** + * Returns true if interface shared objects should be used. + */ + public boolean useInterfaceSharedObjects() { + return supportsInterfaceSharedObjects() && cppOptions.useInterfaceSharedObjects; + } + + public boolean forceIgnoreDashStatic() { + return cppOptions.forceIgnoreDashStatic; + } + + /** + * Returns true iff this build configuration requires inclusion extraction + * (for include scanning) in the action graph. + */ + public boolean needsIncludeScanning() { + return cppOptions.extractInclusions; + } + + public boolean createCppModuleMaps() { + return cppOptions.cppModuleMaps; + } + + /** + * Returns true if shared libraries must be compiled with position independent code + * on this platform or in this configuration. + */ + public boolean needsPic() { + return forcePic() || toolchainNeedsPic(); + } + + /** + * Returns true iff we should use ".pic.o" files when linking executables. + */ + public boolean usePicObjectsForBinaries() { + return forcePic() || usePicForBinaries(); + } + + public boolean legacyWholeArchive() { + return cppOptions.legacyWholeArchive; + } + + public boolean getSymbolCounts() { + return cppOptions.symbolCounts; + } + + public boolean getInmemoryDotdFiles() { + return cppOptions.inmemoryDotdFiles; + } + + public boolean useIsystemForIncludes() { + return cppOptions.useIsystemForIncludes; + } + + public LibcTop getLibcTop() { + return cppOptions.libcTop; + } + + public boolean getUseInterfaceSharedObjects() { + return cppOptions.useInterfaceSharedObjects; + } + + /** + * Returns the FDO support object. + */ + public FdoSupport getFdoSupport() { + return fdoSupport; + } + + /** + * Return the name of the directory (relative to the bin directory) that + * holds mangled links to shared libraries. This name is always set to + * the '{@code _solib_}. + */ + public String getSolibDirectory() { + return solibDirectory; + } + + /** + * Returns the path to the GNU binutils 'objcopy' binary to use for this + * build. (Corresponds to $(OBJCOPY) in make-dbg.) Relative paths are + * relative to the execution root. + */ + public PathFragment getObjCopyExecutable() { + return getToolPathFragment(CppConfiguration.Tool.OBJCOPY); + } + + /** + * Returns the path to the GNU binutils 'gcc' binary that should be used + * by this build. This binary should support compilation of both C (*.c) + * and C++ (*.cc) files. Relative paths are relative to the execution root. + */ + public PathFragment getCppExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCC); + } + + /** + * Returns the path to the GNU binutils 'g++' binary that should be used + * by this build. This binary should support linking of both C (*.c) + * and C++ (*.cc) files. Relative paths are relative to the execution root. + */ + public PathFragment getCppLinkExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCC); + } + + /** + * Returns the path to the GNU binutils 'cpp' binary that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getCpreprocessorExecutable() { + return getToolPathFragment(CppConfiguration.Tool.CPP); + } + + /** + * Returns the path to the GNU binutils 'gcov' binary that should be used + * by this build to analyze C++ coverage data. Relative paths are relative to + * the execution root. + */ + public PathFragment getGcovExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCOV); + } + + /** + * Returns the path to the 'gcov-tool' executable that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getGcovToolExecutable() { + return getToolPathFragment(CppConfiguration.Tool.GCOVTOOL); + } + + /** + * Returns the path to the GNU binutils 'nm' executable that should be used + * by this build. Used only for testing. Relative paths are relative to the + * execution root. + */ + public PathFragment getNmExecutable() { + return getToolPathFragment(CppConfiguration.Tool.NM); + } + + /** + * Returns the path to the GNU binutils 'objdump' executable that should be + * used by this build. Used only for testing. Relative paths are relative to + * the execution root. + */ + public PathFragment getObjdumpExecutable() { + return getToolPathFragment(CppConfiguration.Tool.OBJDUMP); + } + + /** + * Returns the path to the GNU binutils 'ar' binary to use for this build. + * Relative paths are relative to the execution root. + */ + public PathFragment getArExecutable() { + return getToolPathFragment(CppConfiguration.Tool.AR); + } + + /** + * Returns the path to the GNU binutils 'strip' executable that should be used + * by this build. Relative paths are relative to the execution root. + */ + public PathFragment getStripExecutable() { + return getToolPathFragment(CppConfiguration.Tool.STRIP); + } + + /** + * Returns the path to the GNU binutils 'dwp' binary that should be used by this + * build to combine debug info output from individual C++ compilations (i.e. .dwo + * files) into aggregate target-level debug packages. Relative paths are relative to the + * execution root. See https://gcc.gnu.org/wiki/DebugFission . + */ + public PathFragment getDwpExecutable() { + return getToolPathFragment(CppConfiguration.Tool.DWP); + } + + /** + * Returns the GNU System Name + */ + public String getTargetGnuSystemName() { + return targetSystemName; + } + + /** + * Returns the architecture component of the GNU System Name + */ + public String getGnuSystemArch() { + if (targetSystemName.indexOf('-') == -1) { + return targetSystemName; + } + return targetSystemName.substring(0, targetSystemName.indexOf('-')); + } + + /** + * Returns whether the configuration's purpose is only to collect LIPO-related data. + */ + public boolean isLipoContextCollector() { + return lipoContextCollector; + } + + @Override + public String getName() { + return "cpp"; + } + + @Override + public void reportInvalidOptions(EventHandler reporter, BuildOptions buildOptions) { + CppOptions cppOptions = buildOptions.get(CppOptions.class); + if (stripBinaries) { + boolean warn = cppOptions.coptList.contains("-g"); + for (PerLabelOptions opt : cppOptions.perFileCopts) { + warn |= opt.getOptions().contains("-g"); + } + if (warn) { + reporter.handle(Event.warn("Stripping enabled, but '--copt=-g' (or --per_file_copt=...@-g) " + + "specified. Debug information will be generated and then stripped away. This is " + + "probably not what you want! Use '-c dbg' for debug mode, or use '--strip=never' " + + "to disable stripping")); + } + } + + if (cppOptions.fdoInstrument != null && cppOptions.fdoOptimize != null) { + reporter.handle(Event.error("Cannot instrument and optimize for FDO at the same time. " + + "Remove one of the '--fdo_instrument' and '--fdo_optimize' options")); + } + + if (cppOptions.lipoContext != null) { + if (cppOptions.lipoMode != LipoMode.BINARY || cppOptions.fdoOptimize == null) { + reporter.handle(Event.warn("The --lipo_context option can only be used together with " + + "--fdo_optimize= and --lipo=binary. LIPO context will be ignored.")); + } + } else { + if (cppOptions.lipoMode == LipoMode.BINARY && cppOptions.fdoOptimize != null) { + reporter.handle(Event.error("The --lipo_context option must be specified when using " + + "--fdo_optimize= and --lipo=binary")); + } + } + if (cppOptions.lipoMode == LipoMode.BINARY && + compilationMode != CompilationMode.OPT) { + reporter.handle(Event.error( + "'--lipo=binary' can only be used with '--compilation_mode=opt' (or '-c opt')")); + } + + if (cppOptions.fissionModes.contains(compilationMode) && !supportsFission()) { + reporter.handle( + Event.warn("Fission is not supported by this crosstool. Please use a supporting " + + "crosstool to enable fission")); + } + } + + @Override + public void addGlobalMakeVariables(Builder globalMakeEnvBuilder) { + // hardcoded CC->gcc setting for unit tests + globalMakeEnvBuilder.put("CC", getCppExecutable().getPathString()); + + // Make variables provided by crosstool/gcc compiler suite. + globalMakeEnvBuilder.put("AR", getArExecutable().getPathString()); + globalMakeEnvBuilder.put("NM", getNmExecutable().getPathString()); + globalMakeEnvBuilder.put("OBJCOPY", getObjCopyExecutable().getPathString()); + globalMakeEnvBuilder.put("STRIP", getStripExecutable().getPathString()); + + PathFragment gcovtool = getGcovToolExecutable(); + if (gcovtool != null) { + // gcov-tool is optional in Crosstool + globalMakeEnvBuilder.put("GCOVTOOL", gcovtool.getPathString()); + } + + if (getTargetLibc().startsWith("glibc-")) { + globalMakeEnvBuilder.put("GLIBC_VERSION", + getTargetLibc().substring("glibc-".length())); + } else { + globalMakeEnvBuilder.put("GLIBC_VERSION", getTargetLibc()); + } + + globalMakeEnvBuilder.put("C_COMPILER", getCompiler()); + globalMakeEnvBuilder.put("TARGET_CPU", getTargetCpu()); + + // Deprecated variables + + // TODO(bazel-team): (2009) These variables are so rarely used we should try to eliminate + // them entirely. see: "cs -f=BUILD -noi GNU_TARGET" and "cs -f=build_defs -noi + // GNU_TARGET" + globalMakeEnvBuilder.put("CROSSTOOLTOP", crosstoolTopPathFragment.getPathString()); + globalMakeEnvBuilder.put("GLIBC", getTargetLibc()); + globalMakeEnvBuilder.put("GNU_TARGET", targetSystemName); + + globalMakeEnvBuilder.putAll(getAdditionalMakeVariables()); + + globalMakeEnvBuilder.put("ABI_GLIBC_VERSION", getAbiGlibcVersion()); + globalMakeEnvBuilder.put("ABI", abi); + } + + @Override + public void addImplicitLabels(Multimap implicitLabels) { + if (getLibcLabel() != null) { + implicitLabels.put("crosstool", getLibcLabel()); + } + + implicitLabels.put("crosstool", crosstoolTop); + } + + @Override + public void prepareHook(Path execRoot, ArtifactFactory artifactFactory, PathFragment genfilesPath, + PackageRootResolver resolver) throws ViewCreationFailedException { + try { + getFdoSupport().prepareToBuild(execRoot, genfilesPath, artifactFactory, resolver); + } catch (ZipException e) { + throw new ViewCreationFailedException("Error reading provided FDO zip file", e); + } catch (FdoException | IOException e) { + throw new ViewCreationFailedException("Error while initializing FDO support", e); + } + } + + @Override + public void declareSkyframeDependencies(Environment env) { + getFdoSupport().declareSkyframeDependencies(env, execRoot); + } + + @Override + public void addRoots(List roots) { + // Fdo root can only exist for the target configuration. + FdoSupport fdoSupport = getFdoSupport(); + if (fdoSupport.getFdoRoot() != null) { + roots.add(fdoSupport.getFdoRoot()); + } + + // Grepped header includes; this root is not configuration specific. + roots.add(getGreppedIncludesDirectory()); + } + + @Override + public Map getCoverageEnvironment() { + ImmutableMap.Builder env = ImmutableMap.builder(); + env.put("COVERAGE_GCOV_PATH", getGcovExecutable().getPathString()); + PathFragment fdoInstrument = getFdoSupport().getFdoInstrument(); + if (fdoInstrument != null) { + env.put("FDO_DIR", fdoInstrument.getPathString()); + } + return env.build(); + } + + @Override + public ImmutableList

This class can be used only after the loading phase. + */ +public class CppHelper { + // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES? + public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY); + + private static final FileTypeSet CPP_FILETYPES = FileTypeSet.of( + CppFileTypes.CPP_HEADER, + CppFileTypes.CPP_SOURCE); + + private CppHelper() { + // prevents construction + } + + /** + * Merges the STL and toolchain contexts into context builder. The STL is automatically determined + * using the ":stl" attribute. + */ + public static void mergeToolchainDependentContext(RuleContext ruleContext, + Builder contextBuilder) { + TransitiveInfoCollection stl = ruleContext.getPrerequisite(":stl", Mode.TARGET); + if (stl != null) { + // TODO(bazel-team): Clean this up. + contextBuilder.addSystemIncludeDir(stl.getLabel().getPackageFragment().getRelative("gcc3")); + contextBuilder.mergeDependentContext(stl.getProvider(CppCompilationContext.class)); + } + CcToolchainProvider toolchain = getToolchain(ruleContext); + if (toolchain != null) { + contextBuilder.mergeDependentContext(toolchain.getCppCompilationContext()); + } + } + + /** + * Returns the malloc implementation for the given target. + */ + public static TransitiveInfoCollection mallocForTarget(RuleContext ruleContext) { + if (ruleContext.getFragment(CppConfiguration.class).customMalloc() != null) { + return ruleContext.getPrerequisite(":default_malloc", Mode.TARGET); + } else { + return ruleContext.getPrerequisite("malloc", Mode.TARGET); + } + } + + /** + * Expands Make variables in a list of string and tokenizes the result. If the package feature + * no_copts_tokenization is set, tokenize only items consisting of a single make variable. + * + * @param ruleContext the ruleContext to be used as the context of Make variable expansion + * @param attributeName the name of the attribute to use in error reporting + * @param input the list of strings to expand + * @return a list of strings containing the expanded and tokenized values for the + * attribute + */ + // TODO(bazel-team): Move to CcCommon; refactor CcPlugin to use either CcLibraryHelper or + // CcCommon. + static List expandMakeVariables( + RuleContext ruleContext, String attributeName, List input) { + boolean tokenization = + !ruleContext.getFeatures().contains("no_copts_tokenization"); + + List tokens = new ArrayList<>(); + for (String token : input) { + try { + // Legacy behavior: tokenize all items. + if (tokenization) { + ruleContext.tokenizeAndExpandMakeVars(tokens, attributeName, token); + } else { + String exp = ruleContext.expandSingleMakeVariable(attributeName, token); + if (exp != null) { + ShellUtils.tokenize(tokens, exp); + } else { + tokens.add(ruleContext.expandMakeVariables(attributeName, token)); + } + } + } catch (ShellUtils.TokenizationException e) { + ruleContext.attributeError(attributeName, e.getMessage()); + } + } + return ImmutableList.copyOf(tokens); + } + + /** + * Appends the tokenized values of the copts attribute to copts. + */ + public static ImmutableList getAttributeCopts(RuleContext ruleContext, String attr) { + Preconditions.checkArgument(ruleContext.getRule().isAttrDefined(attr, Type.STRING_LIST)); + List unexpanded = ruleContext.attributes().get(attr, Type.STRING_LIST); + + return ImmutableList.copyOf(expandMakeVariables(ruleContext, attr, unexpanded)); + } + + /** + * Expands attribute value either using label expansion + * (if attemptLabelExpansion == {@code true} and it does not look like make + * variable or flag) or tokenizes and expands make variables. + */ + public static void expandAttribute(RuleContext ruleContext, + List values, String attrName, String attrValue, boolean attemptLabelExpansion) { + if (attemptLabelExpansion && CppHelper.isLinkoptLabel(attrValue)) { + if (!CppHelper.expandLabel(ruleContext, values, attrValue)) { + ruleContext.attributeError(attrName, "could not resolve label '" + attrValue + "'"); + } + } else { + ruleContext.tokenizeAndExpandMakeVars(values, attrName, attrValue); + } + } + + /** + * Determines if a linkopt can be a label. Linkopts come in 2 varieties: + * literals -- flags like -Xl and makefile vars like $(LD) -- and labels, + * which we should expand into filenames. + * + * @param linkopt the link option to test. + * @return true if the linkopt is not a flag (starting with "-") or a makefile + * variable (starting with "$"); + */ + private static boolean isLinkoptLabel(String linkopt) { + return !linkopt.startsWith("$") && !linkopt.startsWith("-"); + } + + /** + * Expands a label against the target's deps, adding the expanded path strings + * to the linkopts. + * + * @param linkopts the linkopts to add the expanded label to + * @param labelName the name of the label to expand + * @return true if the label was expanded successfully, false otherwise + */ + private static boolean expandLabel(RuleContext ruleContext, List linkopts, + String labelName) { + try { + Label label = ruleContext.getLabel().getRelative(labelName); + for (FileProvider target : ruleContext + .getPrerequisites("deps", Mode.TARGET, FileProvider.class)) { + if (target.getLabel().equals(label)) { + for (Artifact artifact : target.getFilesToBuild()) { + linkopts.add(artifact.getExecPathString()); + } + return true; + } + } + } catch (SyntaxException e) { + // Quietly ignore and fall through. + } + linkopts.add(labelName); + return false; + } + + /** + * This almost trivial method looks up the :cc_toolchain attribute on the rule context, makes sure + * that it refers to a rule that has a {@link CcToolchainProvider} (gives an error otherwise), and + * returns a reference to that {@link CcToolchainProvider}. The method only returns {@code null} + * if there is no such attribute (this is currently not an error). + */ + @Nullable public static CcToolchainProvider getToolchain(RuleContext ruleContext) { + if (ruleContext.attributes().getAttributeDefinition(":cc_toolchain") == null) { + // TODO(bazel-team): Report an error or throw an exception in this case. + return null; + } + TransitiveInfoCollection dep = ruleContext.getPrerequisite(":cc_toolchain", Mode.TARGET); + return getToolchain(ruleContext, dep); + } + + /** + * This almost trivial method makes sure that the given info collection has a {@link + * CcToolchainProvider} (gives an error otherwise), and returns a reference to that {@link + * CcToolchainProvider}. The method never returns {@code null}, even if there is no toolchain. + */ + public static CcToolchainProvider getToolchain(RuleContext ruleContext, + TransitiveInfoCollection dep) { + // TODO(bazel-team): Consider checking this generally at the attribute level. + if ((dep == null) || (dep.getProvider(CcToolchainProvider.class) == null)) { + ruleContext.ruleError("The selected C++ toolchain is not a cc_toolchain rule"); + return CcToolchainProvider.EMPTY_TOOLCHAIN_IS_ERROR; + } + return dep.getProvider(CcToolchainProvider.class); + } + + /** + * Returns the directory where object files are created. + */ + public static PathFragment getObjDirectory(Label ruleLabel) { + return AnalysisUtils.getUniqueDirectory(ruleLabel, new PathFragment("_objs")); + } + + /** + * Creates a grep-includes ExtractInclusions action for generated sources/headers in the + * needsIncludeScanning() BuildConfiguration case. Returns a map from original header + * Artifact to the output Artifact of grepping over it. The return value only includes + * entries for generated sources or headers when --extract_generated_inclusions is enabled. + * + *

Previously, incremental rebuilds redid all include scanning work + * for a given .cc source in serial. For high-latency file systems, this could cause + * performance problems if many headers are generated. + */ + @Nullable + public static final Map createExtractInclusions(RuleContext ruleContext, + Iterable prerequisites) { + Map extractions = new HashMap<>(); + for (Artifact prerequisite : prerequisites) { + Artifact scanned = createExtractInclusions(ruleContext, prerequisite); + if (scanned != null) { + extractions.put(prerequisite, scanned); + } + } + return extractions; + } + + /** + * Creates a grep-includes ExtractInclusions action for generated sources/headers in the + * needsIncludeScanning() BuildConfiguration case. + * + *

Previously, incremental rebuilds redid all include scanning work for a given + * .cc source in serial. For high-latency file systems, this could cause + * performance problems if many headers are generated. + */ + private static final Artifact createExtractInclusions(RuleContext ruleContext, + Artifact prerequisite) { + if (ruleContext != null && + ruleContext.getFragment(CppConfiguration.class).needsIncludeScanning() && + !prerequisite.isSourceArtifact() && + CPP_FILETYPES.matches(prerequisite.getFilename())) { + Artifact scanned = getIncludesOutput(ruleContext, prerequisite); + ruleContext.registerAction( + new ExtractInclusionAction(ruleContext.getActionOwner(), prerequisite, scanned)); + return scanned; + } + return null; + } + + private static Artifact getIncludesOutput(RuleContext ruleContext, Artifact src) { + Root root = ruleContext.getFragment(CppConfiguration.class).getGreppedIncludesDirectory(); + PathFragment relOut = IncludeScanningUtil.getRootRelativeOutputPath(src.getExecPath()); + return ruleContext.getAnalysisEnvironment().getDerivedArtifact(relOut, root); + } + + /** + * Returns the workspace-relative filename for the linked artifact. + */ + public static PathFragment getLinkedFilename(RuleContext ruleContext, + LinkTargetType linkType) { + PathFragment relativePath = Util.getWorkspaceRelativePath(ruleContext.getTarget()); + PathFragment linkedFileName = (linkType == LinkTargetType.EXECUTABLE) ? + relativePath : + relativePath.replaceName("lib" + relativePath.getBaseName() + linkType.getExtension()); + return linkedFileName; + } + + /** + * Resolves the linkstamp collection from the {@code CcLinkParams} into a map. + * + *

Emits a warning on the rule if there are identical linkstamp artifacts with different + * compilation contexts. + */ + public static Map> resolveLinkstamps(RuleContext ruleContext, + CcLinkParams linkParams) { + Map> result = new LinkedHashMap<>(); + for (Linkstamp pair : linkParams.getLinkstamps()) { + Artifact artifact = pair.getArtifact(); + if (result.containsKey(artifact)) { + ruleContext.ruleWarning("rule inherits the '" + artifact.toDetailString() + + "' linkstamp file from more than one cc_library rule"); + } + result.put(artifact, pair.getDeclaredIncludeSrcs()); + } + return result; + } + + public static void addTransitiveLipoInfoForCommonAttributes( + RuleContext ruleContext, + CcCompilationOutputs outputs, + NestedSetBuilder scannableBuilder) { + + TransitiveLipoInfoProvider stl = null; + if (ruleContext.getRule().getAttributeDefinition(":stl") != null && + ruleContext.getPrerequisite(":stl", Mode.TARGET) != null) { + // If the attribute is defined, it is never null. + stl = ruleContext.getPrerequisite(":stl", Mode.TARGET) + .getProvider(TransitiveLipoInfoProvider.class); + } + if (stl != null) { + scannableBuilder.addTransitive(stl.getTransitiveIncludeScannables()); + } + + for (TransitiveLipoInfoProvider dep : + ruleContext.getPrerequisites("deps", Mode.TARGET, TransitiveLipoInfoProvider.class)) { + scannableBuilder.addTransitive(dep.getTransitiveIncludeScannables()); + } + + if (ruleContext.getRule().getRuleClassObject().hasAttr("malloc", Type.LABEL)) { + TransitiveInfoCollection malloc = mallocForTarget(ruleContext); + TransitiveLipoInfoProvider provider = malloc.getProvider(TransitiveLipoInfoProvider.class); + if (provider != null) { + scannableBuilder.addTransitive(provider.getTransitiveIncludeScannables()); + } + } + + for (IncludeScannable scannable : outputs.getLipoScannables()) { + Preconditions.checkState(scannable.getIncludeScannerSources().size() == 1); + scannableBuilder.add(scannable); + } + } + + // TODO(bazel-team): figure out a way to merge these 2 methods. See the Todo in + // CcCommonConfiguredTarget.noCoptsMatches(). + /** + * Determines if we should apply -fPIC for this rule's C++ compilations. This determination + * is generally made by the global C++ configuration settings "needsPic" and + * and "usePicForBinaries". However, an individual rule may override these settings by applying + * -fPIC" to its "nocopts" attribute. This allows incompatible rules to "opt out" of global PIC + * settings (see bug: "Provide a way to turn off -fPIC for targets that can't be built that way"). + * + * @param ruleContext the context of the rule to check + * @param forBinary true if compiling for a binary, false if for a shared library + * @return true if this rule's compilations should apply -fPIC, false otherwise + */ + public static boolean usePic(RuleContext ruleContext, boolean forBinary) { + if (CcCommon.noCoptsMatches("-fPIC", ruleContext)) { + return false; + } + CppConfiguration config = ruleContext.getFragment(CppConfiguration.class); + return forBinary ? config.usePicObjectsForBinaries() : config.needsPic(); + } + + /** + * Returns the LIPO context provider for configured target, + * or null if such a provider doesn't exist. + */ + public static LipoContextProvider getLipoContextProvider(RuleContext ruleContext) { + if (ruleContext.getRule().getAttributeDefinition(":lipo_context_collector") == null) { + return null; + } + + TransitiveInfoCollection dep = + ruleContext.getPrerequisite(":lipo_context_collector", Mode.DONT_CHECK); + return (dep != null) ? dep.getProvider(LipoContextProvider.class) : null; + } + + // Creates CppModuleMap object, and adds it to C++ compilation context. + public static CppModuleMap addCppModuleMapToContext(RuleContext ruleContext, + CppCompilationContext.Builder contextBuilder) { + if (!ruleContext.getFragment(CppConfiguration.class).createCppModuleMaps()) { + return null; + } + if (getToolchain(ruleContext).getCppCompilationContext().getCppModuleMap() == null) { + return null; + } + // Create the module map artifact as a genfile. + PathFragment mapPath = FileSystemUtils.appendExtension(ruleContext.getLabel().toPathFragment(), + Iterables.getOnlyElement(CppFileTypes.CPP_MODULE_MAP.getExtensions())); + Artifact mapFile = ruleContext.getAnalysisEnvironment().getDerivedArtifact(mapPath, + ruleContext.getConfiguration().getGenfilesDirectory()); + CppModuleMap moduleMap = + new CppModuleMap(mapFile, ruleContext.getLabel().toString()); + contextBuilder.setCppModuleMap(moduleMap); + return moduleMap; + } + + /** + * Returns a middleman for all files to build for the given configured target, + * substituting shared library artifacts with corresponding solib symlinks. If + * multiple calls are made, then it returns the same artifact for configurations + * with the same internal directory. + * + *

The resulting middleman only aggregates the inputs and must be expanded + * before populating the set of files necessary to execute an action. + */ + static List getAggregatingMiddlemanForCppRuntimes(RuleContext ruleContext, + String purpose, TransitiveInfoCollection dep, String solibDirOverride, + BuildConfiguration configuration) { + return getMiddlemanInternal( + ruleContext.getAnalysisEnvironment(), ruleContext, ruleContext.getActionOwner(), purpose, + dep, true, true, solibDirOverride, configuration); + } + + @VisibleForTesting + public static List getAggregatingMiddlemanForTesting(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner owner, String purpose, TransitiveInfoCollection dep, + boolean useSolibSymlinks, BuildConfiguration configuration) { + return getMiddlemanInternal( + env, ruleContext, owner, purpose, dep, useSolibSymlinks, false, null, configuration); + } + + /** + * Internal implementation for getAggregatingMiddlemanForCppRuntimes. + */ + private static List getMiddlemanInternal(AnalysisEnvironment env, + RuleContext ruleContext, ActionOwner actionOwner, String purpose, + TransitiveInfoCollection dep, boolean useSolibSymlinks, boolean isCppRuntime, + String solibDirOverride, BuildConfiguration configuration) { + if (dep == null) { + return ImmutableList.of(); + } + MiddlemanFactory factory = env.getMiddlemanFactory(); + Iterable artifacts = dep.getProvider(FileProvider.class).getFilesToBuild(); + if (useSolibSymlinks) { + List symlinkedArtifacts = new ArrayList<>(); + for (Artifact artifact : artifacts) { + symlinkedArtifacts.add(solibArtifactMaybe( + ruleContext, artifact, isCppRuntime, solibDirOverride, configuration)); + } + artifacts = symlinkedArtifacts; + purpose += "_with_solib"; + } + return ImmutableList.of(factory.createMiddlemanAllowMultiple( + env, actionOwner, purpose, artifacts, configuration.getMiddlemanDirectory())); + } + + /** + * If the artifact is a shared library, returns the solib symlink artifact associated with it. + * + * @param ruleContext the context of the rule that creates the symlink + * @param artifact the library the solib symlink should point to + * @param isCppRuntime whether the library is a C++ runtime + * @param solibDirOverride if not null, forces the solib symlink to be in this directory + */ + private static Artifact solibArtifactMaybe(RuleContext ruleContext, Artifact artifact, + boolean isCppRuntime, String solibDirOverride, BuildConfiguration configuration) { + if (SHARED_LIBRARY_FILETYPES.matches(artifact.getFilename())) { + return isCppRuntime + ? SolibSymlinkAction.getCppRuntimeSymlink( + ruleContext, artifact, solibDirOverride, configuration) + .getArtifact() + : SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, artifact, false, true, configuration) + .getArtifact(); + } else { + return artifact; + } + } + + /** + * Returns the type of archives being used. + */ + public static Link.ArchiveType archiveType(BuildConfiguration config) { + CppConfiguration cppConfig = config.getFragment(CppConfiguration.class); + return cppConfig.archiveType(); + } + + /** + * Returns the FDO build subtype. + */ + public static String getFdoBuildStamp(CppConfiguration cppConfiguration) { + if (cppConfiguration.getFdoSupport().isAutoFdoEnabled()) { + return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "ALIPO" : "AFDO"; + } + if (cppConfiguration.isFdo()) { + return (cppConfiguration.getLipoMode() == LipoMode.BINARY) ? "LIPO" : "FDO"; + } + return null; + } + + /** + * Returns a relative path to the bin directory for data in AutoFDO LIPO mode. + */ + public static PathFragment getLipoDataBinFragment(BuildConfiguration configuration) { + PathFragment parent = configuration.getBinFragment().getParentDirectory(); + return parent.replaceName(parent.getBaseName() + "-lipodata") + .getChild(configuration.getBinFragment().getBaseName()); + } + + /** + * Returns a relative path to the genfiles directory for data in AutoFDO LIPO mode. + */ + public static PathFragment getLipoDataGenfilesFragment(BuildConfiguration configuration) { + PathFragment parent = configuration.getGenfilesFragment().getParentDirectory(); + return parent.replaceName(parent.getBaseName() + "-lipodata") + .getChild(configuration.getGenfilesFragment().getBaseName()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java new file mode 100644 index 0000000000..ecf3431b4f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkAction.java @@ -0,0 +1,1074 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ParameterFile; +import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.extra.CppLinkInfo; +import com.google.devtools.build.lib.actions.extra.ExtraActionInfo; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.ImmutableIterable; +import com.google.devtools.build.lib.collect.IterablesChain; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Action that represents an ELF linking step. + */ +@ThreadCompatible +public final class CppLinkAction extends AbstractAction { + private static final String LINK_GUID = "58ec78bd-1176-4e36-8143-439f656b181d"; + private static final String FAKE_LINK_GUID = "da36f819-5a15-43a9-8a45-e01b60e10c8b"; + + private final CppConfiguration cppConfiguration; + private final LibraryToLink outputLibrary; + private final LibraryToLink interfaceOutputLibrary; + + private final LinkCommandLine linkCommandLine; + + /** True for cc_fake_binary targets. */ + private final boolean fake; + + private final Iterable mandatoryInputs; + + // Linking uses a lot of memory; estimate 1 MB per input file, min 1.5 Gib. + // It is vital to not underestimate too much here, + // because running too many concurrent links can + // thrash the machine to the point where it stops + // responding to keystrokes or mouse clicks. + // CPU and IO do not scale similarly and still use the static minimum estimate. + public static final ResourceSet LINK_RESOURCES_PER_INPUT = new ResourceSet(1, 0, 0); + + // This defines the minimum of each resource that will be reserved. + public static final ResourceSet MIN_STATIC_LINK_RESOURCES = new ResourceSet(1536, 1, 0.3); + + // Dynamic linking should be cheaper than static linking. + public static final ResourceSet MIN_DYNAMIC_LINK_RESOURCES = new ResourceSet(1024, 0.3, 0.2); + + /** + * Use {@link Builder} to create instances of this class. Also see there for + * the documentation of all parameters. + * + *

This constructor is intentionally private and is only to be called from + * {@link Builder#build()}. + */ + private CppLinkAction(ActionOwner owner, + Iterable inputs, + ImmutableList outputs, + CppConfiguration cppConfiguration, + LibraryToLink outputLibrary, + LibraryToLink interfaceOutputLibrary, + boolean fake, + LinkCommandLine linkCommandLine) { + super(owner, inputs, outputs); + this.mandatoryInputs = inputs; + this.cppConfiguration = cppConfiguration; + this.outputLibrary = outputLibrary; + this.interfaceOutputLibrary = interfaceOutputLibrary; + this.fake = fake; + + this.linkCommandLine = linkCommandLine; + } + + private static Iterable filterLinkerInputs(Iterable inputs) { + return Iterables.filter(inputs, new Predicate() { + @Override + public boolean apply(LinkerInput input) { + return Link.VALID_LINKER_INPUTS.matches(input.getArtifact().getFilename()); + } + }); + } + + private static Iterable filterLinkerInputArtifacts(Iterable inputs) { + return Iterables.filter(inputs, new Predicate() { + @Override + public boolean apply(Artifact input) { + return Link.VALID_LINKER_INPUTS.matches(input.getFilename()); + } + }); + } + + private CppConfiguration getCppConfiguration() { + return cppConfiguration; + } + + @VisibleForTesting + public String getTargetCpu() { + return getCppConfiguration().getTargetCpu(); + } + + public String getHostSystemName() { + return getCppConfiguration().getHostSystemName(); + } + + /** + * Returns the link configuration; for correctness you should not call this method during + * execution - only the argv is part of the action cache key, and we therefore don't guarantee + * that the action will be re-executed if the contents change in a way that does not affect the + * argv. + */ + @VisibleForTesting + public LinkCommandLine getLinkCommandLine() { + return linkCommandLine; + } + + public LibraryToLink getOutputLibrary() { + return outputLibrary; + } + + public LibraryToLink getInterfaceOutputLibrary() { + return interfaceOutputLibrary; + } + + /** + * Returns the path to the output artifact produced by the linker. + */ + public Path getOutputFile() { + return outputLibrary.getArtifact().getPath(); + } + + @VisibleForTesting + public List getRawLinkArgv() { + return linkCommandLine.getRawLinkArgv(); + } + + @VisibleForTesting + public List getArgv() { + return linkCommandLine.arguments(); + } + + /** + * Prepares and returns the command line specification for this link. + * Splits appropriate parts into a .params file and adds any required + * linkstamp compilation steps. + * + * @return a finalized command line suitable for execution + */ + public final List prepareCommandLine(Path execRoot, List inputFiles) + throws ExecException { + List commandlineArgs; + // Try to shorten the command line by use of a parameter file. + // This makes the output with --subcommands (et al) more readable. + if (linkCommandLine.canBeSplit()) { + PathFragment paramExecPath = ParameterFile.derivePath( + outputLibrary.getArtifact().getExecPath()); + Pair, List> split = linkCommandLine.splitCommandline(paramExecPath); + commandlineArgs = split.first; + writeToParamFile(execRoot, paramExecPath, split.second); + if (inputFiles != null) { + inputFiles.add(paramExecPath.getPathString()); + } + } else { + commandlineArgs = linkCommandLine.getRawLinkArgv(); + } + return linkCommandLine.finalizeWithLinkstampCommands(commandlineArgs); + } + + private static void writeToParamFile(Path workingDir, PathFragment paramExecPath, + List paramFileArgs) throws ExecException { + // Create parameter file. + ParameterFile paramFile = new ParameterFile(workingDir, paramExecPath, ISO_8859_1, + ParameterFileType.UNQUOTED); + Path paramFilePath = paramFile.getPath(); + try { + // writeContent() fails for existing files that are marked readonly. + paramFilePath.delete(); + } catch (IOException e) { + throw new EnvironmentalExecException("could not delete file '" + paramFilePath + "'", e); + } + paramFile.writeContent(paramFileArgs); + + // Normally Blaze chmods all output files automatically (see + // SkyframeActionExecutor#setOutputsReadOnlyAndExecutable), but this params file is created + // out-of-band and is not declared as an output. By chmodding the file, other processes + // can observe this file being created. + try { + paramFilePath.setWritable(false); + paramFilePath.setExecutable(true); // for consistency with other action outputs + } catch (IOException e) { + throw new EnvironmentalExecException("could not chmod param file '" + paramFilePath + "'", e); + } + } + + @Override + @ThreadCompatible + public void execute( + ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + if (fake) { + executeFake(); + } else { + Executor executor = actionExecutionContext.getExecutor(); + + try { + executor.getContext(CppLinkActionContext.class).exec( + this, actionExecutionContext); + } catch (ExecException e) { + throw e.toActionExecutionException("Linking of rule '" + getOwner().getLabel() + "'", + executor.getVerboseFailures(), this); + } + } + } + + @Override + public String describeStrategy(Executor executor) { + return fake + ? "fake,local" + : executor.getContext(CppLinkActionContext.class).strategyLocality(this); + } + + // Don't forget to update FAKE_LINK_GUID if you modify this method. + @ThreadCompatible + private void executeFake() + throws ActionExecutionException { + // The uses of getLinkConfiguration in this method may not be consistent with the computed key. + // I.e., this may be incrementally incorrect. + final Collection linkstampOutputs = getLinkCommandLine().getLinkstamps().values(); + + // Prefix all fake output files in the command line with $TEST_TMPDIR/. + final String outputPrefix = "$TEST_TMPDIR/"; + List escapedLinkArgv = escapeLinkArgv(linkCommandLine.getRawLinkArgv(), + linkstampOutputs, outputPrefix); + // Write the commands needed to build the real target to the fake target + // file. + StringBuilder s = new StringBuilder(); + Joiner.on('\n').appendTo(s, + "# This is a fake target file, automatically generated.", + "# Do not edit by hand!", + "echo $0 is a fake target file and not meant to be executed.", + "exit 0", + "EOS", + "", + "makefile_dir=.", + ""); + + try { + // Concatenate all the (fake) .o files into the result. + for (LinkerInput linkerInput : getLinkCommandLine().getLinkerInputs()) { + Artifact objectFile = linkerInput.getArtifact(); + if (CppFileTypes.OBJECT_FILE.matches(objectFile.getFilename()) + && linkerInput.isFake()) { + s.append(FileSystemUtils.readContentAsLatin1(objectFile.getPath())); // (IOException) + } + } + + s.append(getOutputFile().getBaseName()).append(": "); + for (Artifact linkstamp : linkstampOutputs) { + s.append("mkdir -p " + outputPrefix + + linkstamp.getExecPath().getParentDirectory() + " && "); + } + Joiner.on(' ').appendTo(s, + ShellEscaper.escapeAll(linkCommandLine.finalizeAlreadyEscapedWithLinkstampCommands( + escapedLinkArgv, outputPrefix))); + s.append('\n'); + if (getOutputFile().exists()) { + getOutputFile().setWritable(true); // (IOException) + } + FileSystemUtils.writeContent(getOutputFile(), ISO_8859_1, s.toString()); + getOutputFile().setExecutable(true); // (IOException) + for (Artifact linkstamp : linkstampOutputs) { + FileSystemUtils.touchFile(linkstamp.getPath()); + } + } catch (IOException e) { + throw new ActionExecutionException("failed to create fake link command for rule '" + + getOwner().getLabel() + ": " + e.getMessage(), + this, false); + } + } + + /** + * Shell-escapes the raw link command line. + * + * @param rawLinkArgv raw link command line + * @param linkstampOutputs linkstamp artifacts + * @param outputPrefix to be prepended to any outputs + * @return escaped link command line + */ + private List escapeLinkArgv(List rawLinkArgv, + final Collection linkstampOutputs, final String outputPrefix) { + final List linkstampExecPaths = Artifact.asExecPaths(linkstampOutputs); + ImmutableList.Builder escapedArgs = ImmutableList.builder(); + for (String rawArg : rawLinkArgv) { + String escapedArg; + if (rawArg.equals(getPrimaryOutput().getExecPathString()) + || linkstampExecPaths.contains(rawArg)) { + escapedArg = outputPrefix + ShellEscaper.escapeString(rawArg); + } else if (rawArg.startsWith(Link.FAKE_OBJECT_PREFIX)) { + escapedArg = outputPrefix + ShellEscaper.escapeString( + rawArg.substring(Link.FAKE_OBJECT_PREFIX.length())); + } else { + escapedArg = ShellEscaper.escapeString(rawArg); + } + escapedArgs.add(escapedArg); + } + return escapedArgs.build(); + } + + @Override + public ExtraActionInfo.Builder getExtraActionInfo() { + // The uses of getLinkConfiguration in this method may not be consistent with the computed key. + // I.e., this may be incrementally incorrect. + CppLinkInfo.Builder info = CppLinkInfo.newBuilder(); + info.addAllInputFile(Artifact.toExecPaths( + LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getLinkerInputs()))); + info.addAllInputFile(Artifact.toExecPaths( + LinkerInputs.toLibraryArtifacts(getLinkCommandLine().getRuntimeInputs()))); + info.setOutputFile(getPrimaryOutput().getExecPathString()); + if (interfaceOutputLibrary != null) { + info.setInterfaceOutputFile(interfaceOutputLibrary.getArtifact().getExecPathString()); + } + info.setLinkTargetType(getLinkCommandLine().getLinkTargetType().name()); + info.setLinkStaticness(getLinkCommandLine().getLinkStaticness().name()); + info.addAllLinkStamp(Artifact.toExecPaths(getLinkCommandLine().getLinkstamps().values())); + info.addAllBuildInfoHeaderArtifact( + Artifact.toExecPaths(getLinkCommandLine().getBuildInfoHeaderArtifacts())); + info.addAllLinkOpt(getLinkCommandLine().getLinkopts()); + + return super.getExtraActionInfo() + .setExtension(CppLinkInfo.cppLinkInfo, info.build()); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(fake ? FAKE_LINK_GUID : LINK_GUID); + f.addString(getCppConfiguration().getLdExecutable().getPathString()); + f.addStrings(linkCommandLine.arguments()); + // TODO(bazel-team): For correctness, we need to ensure the invariant that all values accessed + // during the execution phase are also covered by the key. Above, we add the argv to the key, + // which covers most cases. Unfortunately, the extra action and fake support methods above also + // sometimes directly access settings from the link configuration that may or may not affect the + // key. We either need to change the code to cover them in the key computation, or change the + // LinkConfiguration to disallow the combinations where the value of a setting does not affect + // the argv. + f.addBoolean(linkCommandLine.isNativeDeps()); + f.addBoolean(linkCommandLine.useTestOnlyFlags()); + if (linkCommandLine.getRuntimeSolibDir() != null) { + f.addPath(linkCommandLine.getRuntimeSolibDir()); + } + return f.hexDigestAndReset(); + } + + @Override + public String describeKey() { + StringBuilder message = new StringBuilder(); + if (fake) { + message.append("Fake "); + } + message.append(getProgressMessage()); + message.append('\n'); + message.append(" Command: "); + message.append(ShellEscaper.escapeString( + getCppConfiguration().getLdExecutable().getPathString())); + message.append('\n'); + // Outputting one argument per line makes it easier to diff the results. + for (String argument : ShellEscaper.escapeAll(linkCommandLine.arguments())) { + message.append(" Argument: "); + message.append(argument); + message.append('\n'); + } + return message.toString(); + } + + @Override + public String getMnemonic() { return "CppLink"; } + + @Override + protected String getRawProgressMessage() { + return "Linking " + outputLibrary.getArtifact().prettyPrint(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(CppLinkActionContext.class).estimateResourceConsumption(this); + } + + /** + * Estimate the resources consumed when this action is run locally. + */ + public ResourceSet estimateResourceConsumptionLocal() { + // It's ok if this behaves differently even if the key is identical. + ResourceSet minLinkResources = + getLinkCommandLine().getLinkStaticness() == Link.LinkStaticness.DYNAMIC + ? MIN_DYNAMIC_LINK_RESOURCES + : MIN_STATIC_LINK_RESOURCES; + + final int inputSize = Iterables.size(getLinkCommandLine().getLinkerInputs()) + + Iterables.size(getLinkCommandLine().getRuntimeInputs()); + + return new ResourceSet( + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getMemoryMb(), + minLinkResources.getMemoryMb()), + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getCpuUsage(), + minLinkResources.getCpuUsage()), + Math.max(inputSize * LINK_RESOURCES_PER_INPUT.getIoUsage(), + minLinkResources.getIoUsage()) + ); + } + + @Override + public Iterable getMandatoryInputs() { + return mandatoryInputs; + } + + /** + * Determines whether or not this link should output a symbol counts file. + */ + private static boolean enableSymbolsCounts(CppConfiguration cppConfiguration, boolean fake, + LinkTargetType linkType) { + return cppConfiguration.getSymbolCounts() + && cppConfiguration.supportsGoldLinker() + && linkType == LinkTargetType.EXECUTABLE + && !fake; + } + + /** + * Builder class to construct {@link CppLinkAction}s. + */ + public static class Builder { + // Builder-only + private final RuleContext ruleContext; + private final AnalysisEnvironment analysisEnvironment; + private final PathFragment outputPath; + private final CcToolchainProvider toolchain; + private PathFragment interfaceOutputPath; + private PathFragment runtimeSolibDir; + protected final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + + // Morally equivalent with {@link Context}, except these are mutable. + // Keep these in sync with {@link Context}. + private final Set nonLibraries = new LinkedHashSet<>(); + private final NestedSetBuilder libraries = NestedSetBuilder.linkOrder(); + private NestedSet crosstoolInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private Artifact runtimeMiddleman; + private NestedSet runtimeInputs = NestedSetBuilder.emptySet(Order.STABLE_ORDER); + private final NestedSetBuilder compilationInputs = NestedSetBuilder.stableOrder(); + private final Set linkstamps = new LinkedHashSet<>(); + private List linkstampOptions = new ArrayList<>(); + private final List linkopts = new ArrayList<>(); + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC; + private boolean fake; + private boolean isNativeDeps; + private boolean useTestOnlyFlags; + private boolean wholeArchive; + private boolean supportsParamFiles = true; + + /** + * Creates a builder that builds {@link CppLinkAction} instances. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + */ + public Builder(RuleContext ruleContext, PathFragment outputPath) { + this(ruleContext, outputPath, ruleContext.getConfiguration(), + ruleContext.getAnalysisEnvironment(), CppHelper.getToolchain(ruleContext)); + } + + /** + * Creates a builder that builds {@link CppLinkAction} instances. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + */ + public Builder(RuleContext ruleContext, PathFragment outputPath, + BuildConfiguration configuration, CcToolchainProvider toolchain) { + this(ruleContext, outputPath, configuration, + ruleContext.getAnalysisEnvironment(), toolchain); + } + + /** + * Creates a builder that builds {@link CppLinkAction}s. + * + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + * @param configuration the configuration used to determine the tool chain + * and the default link options + */ + private Builder(RuleContext ruleContext, PathFragment outputPath, + BuildConfiguration configuration, AnalysisEnvironment analysisEnvironment, + CcToolchainProvider toolchain) { + this.ruleContext = ruleContext; + this.analysisEnvironment = Preconditions.checkNotNull(analysisEnvironment); + this.outputPath = Preconditions.checkNotNull(outputPath); + this.configuration = Preconditions.checkNotNull(configuration); + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.toolchain = toolchain; + + // The toolchain != null is here for CppLinkAction.createTestBuilder(). Meh. + if (cppConfiguration.supportsEmbeddedRuntimes() && toolchain != null) { + runtimeSolibDir = toolchain.getDynamicRuntimeSolibDir(); + } + if (toolchain != null) { + supportsParamFiles = toolchain.supportsParamFiles(); + } + } + + /** + * Given a Context, creates a Builder that builds {@link CppLinkAction}s. + * Note well: Keep the Builder->Context and Context->Builder transforms consistent! + * @param ruleContext the rule that owns the action + * @param outputPath the path of the ELF file to be created, relative to the + * 'bin' directory + * @param linkContext an immutable CppLinkAction.Context from the original builder + */ + public Builder(RuleContext ruleContext, PathFragment outputPath, Context linkContext, + BuildConfiguration configuration) { + // These Builder-only fields get set in the constructor: + // ruleContext, analysisEnvironment, outputPath, configuration, runtimeSolibDir + this(ruleContext, outputPath, configuration, ruleContext.getAnalysisEnvironment(), + CppHelper.getToolchain(ruleContext)); + Preconditions.checkNotNull(linkContext); + + // All linkContext fields should be transferred to this Builder. + this.nonLibraries.addAll(linkContext.nonLibraries); + this.libraries.addTransitive(linkContext.libraries); + this.crosstoolInputs = linkContext.crosstoolInputs; + this.runtimeMiddleman = linkContext.runtimeMiddleman; + this.runtimeInputs = linkContext.runtimeInputs; + this.compilationInputs.addTransitive(linkContext.compilationInputs); + this.linkstamps.addAll(linkContext.linkstamps); + this.linkopts.addAll(linkContext.linkopts); + this.linkType = linkContext.linkType; + this.linkStaticness = linkContext.linkStaticness; + this.fake = linkContext.fake; + this.isNativeDeps = linkContext.isNativeDeps; + this.useTestOnlyFlags = linkContext.useTestOnlyFlags; + } + + /** + * Builds the Action as configured and returns it. + * + *

This method may only be called once. + */ + public CppLinkAction build() { + if (interfaceOutputPath != null && (fake || linkType != LinkTargetType.DYNAMIC_LIBRARY)) { + throw new RuntimeException("Interface output can only be used " + + "with non-fake DYNAMIC_LIBRARY targets"); + } + + final Artifact output = createArtifact(outputPath); + final Artifact interfaceOutput = (interfaceOutputPath != null) + ? createArtifact(interfaceOutputPath) + : null; + + final ImmutableList buildInfoHeaderArtifacts = !linkstamps.isEmpty() + ? ruleContext.getAnalysisEnvironment().getBuildInfo(ruleContext, CppBuildInfo.KEY) + : ImmutableList.of(); + + final Artifact symbolCountOutput = enableSymbolsCounts(cppConfiguration, fake, linkType) + ? createArtifact(output.getRootRelativePath().replaceName( + output.getExecPath().getBaseName() + ".sc")) + : null; + + boolean needWholeArchive = wholeArchive || needWholeArchive( + linkStaticness, linkType, linkopts, isNativeDeps, cppConfiguration); + + NestedSet uniqueLibraries = libraries.build(); + final Iterable filteredNonLibraryArtifacts = filterLinkerInputArtifacts( + LinkerInputs.toLibraryArtifacts(nonLibraries)); + final Iterable linkerInputs = IterablesChain.builder() + .add(ImmutableList.copyOf(filterLinkerInputs(nonLibraries))) + .add(ImmutableIterable.from(Link.mergeInputsCmdLine( + uniqueLibraries, needWholeArchive, cppConfiguration.archiveType()))) + .build(); + + // ruleContext can only be null during testing. This is kind of ugly. + final ImmutableSet features = (ruleContext == null) + ? ImmutableSet.of() + : ruleContext.getFeatures(); + + final LibraryToLink outputLibrary = + LinkerInputs.newInputLibrary(output, filteredNonLibraryArtifacts); + final LibraryToLink interfaceOutputLibrary = interfaceOutput == null ? null : + LinkerInputs.newInputLibrary(interfaceOutput, filteredNonLibraryArtifacts); + + final ImmutableMap linkstampMap = + mapLinkstampsToOutputs(linkstamps, ruleContext, output); + + final ImmutableList actionOutputs = constructOutputs( + outputLibrary.getArtifact(), + linkstampMap.values(), + interfaceOutputLibrary == null ? null : interfaceOutputLibrary.getArtifact(), + symbolCountOutput); + + LinkCommandLine linkCommandLine = new LinkCommandLine.Builder(configuration, getOwner()) + .setOutput(outputLibrary.getArtifact()) + .setInterfaceOutput(interfaceOutput) + .setSymbolCountsOutput(symbolCountOutput) + .setBuildInfoHeaderArtifacts(buildInfoHeaderArtifacts) + .setLinkerInputs(linkerInputs) + .setRuntimeInputs(ImmutableList.copyOf(LinkerInputs.simpleLinkerInputs(runtimeInputs))) + .setLinkTargetType(linkType) + .setLinkStaticness(linkStaticness) + .setLinkopts(ImmutableList.copyOf(linkopts)) + .setFeatures(features) + .setLinkstamps(linkstampMap) + .addLinkstampCompileOptions(linkstampOptions) + .setRuntimeSolibDir(linkType.isStaticLibraryLink() ? null : runtimeSolibDir) + .setNativeDeps(isNativeDeps) + .setUseTestOnlyFlags(useTestOnlyFlags) + .setNeedWholeArchive(needWholeArchive) + .setInterfaceSoBuilder(getInterfaceSoBuilder()) + .setSupportsParamFiles(supportsParamFiles) + .build(); + + // Compute the set of inputs - we only need stable order here. + NestedSetBuilder dependencyInputsBuilder = NestedSetBuilder.stableOrder(); + dependencyInputsBuilder.addAll(buildInfoHeaderArtifacts); + dependencyInputsBuilder.addAll(linkstamps); + dependencyInputsBuilder.addTransitive(crosstoolInputs); + if (runtimeMiddleman != null) { + dependencyInputsBuilder.add(runtimeMiddleman); + } + dependencyInputsBuilder.addTransitive(compilationInputs.build()); + + Iterable expandedInputs = + LinkerInputs.toLibraryArtifacts(Link.mergeInputsDependencies(uniqueLibraries, + needWholeArchive, cppConfiguration.archiveType())); + // getPrimaryInput returns the first element, and that is a public interface - therefore the + // order here is important. + Iterable inputs = IterablesChain.builder() + .add(ImmutableList.copyOf(LinkerInputs.toLibraryArtifacts(nonLibraries))) + .add(dependencyInputsBuilder.build()) + .add(ImmutableIterable.from(expandedInputs)) + .deduplicate() + .build(); + + return new CppLinkAction( + getOwner(), + inputs, + actionOutputs, + cppConfiguration, + outputLibrary, + interfaceOutputLibrary, + fake, + linkCommandLine); + } + + /** + * The default heuristic on whether we need to use whole-archive for the link. + */ + private static boolean needWholeArchive(LinkStaticness staticness, + LinkTargetType type, Collection linkopts, boolean isNativeDeps, + CppConfiguration cppConfig) { + boolean fullyStatic = (staticness == LinkStaticness.FULLY_STATIC); + boolean mostlyStatic = (staticness == LinkStaticness.MOSTLY_STATIC); + boolean sharedLinkopts = type == LinkTargetType.DYNAMIC_LIBRARY + || linkopts.contains("-shared") + || cppConfig.getLinkOptions().contains("-shared"); + return (isNativeDeps || cppConfig.legacyWholeArchive()) + && (fullyStatic || mostlyStatic) + && sharedLinkopts; + } + + private static ImmutableList constructOutputs(Artifact primaryOutput, + Collection outputList, Artifact... outputs) { + return new ImmutableList.Builder() + .add(primaryOutput) + .addAll(outputList) + .addAll(CollectionUtils.asListWithoutNulls(outputs)) + .build(); + } + + /** + * Translates a collection of linkstamp source files to an immutable + * mapping from source files to object files. In other words, given a + * set of source files, this method determines the output path to which + * each file should be compiled. + * + * @param linkstamps collection of linkstamp source files + * @param ruleContext the rule for which this link is being performed + * @param outputBinary the binary output path for this link + * @return an immutable map that pairs each source file with the + * corresponding object file that should be fed into the link + */ + public static ImmutableMap mapLinkstampsToOutputs( + Collection linkstamps, RuleContext ruleContext, Artifact outputBinary) { + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + + PathFragment outputBinaryPath = outputBinary.getRootRelativePath(); + PathFragment stampOutputDirectory = outputBinaryPath.getParentDirectory(). + getRelative("_objs").getRelative(outputBinaryPath.getBaseName()); + + for (Artifact linkstamp : linkstamps) { + PathFragment stampOutputPath = stampOutputDirectory.getRelative( + FileSystemUtils.replaceExtension(linkstamp.getRootRelativePath(), ".o")); + mapBuilder.put(linkstamp, + ruleContext.getAnalysisEnvironment().getDerivedArtifact( + stampOutputPath, outputBinary.getRoot())); + } + return mapBuilder.build(); + } + + protected ActionOwner getOwner() { + return ruleContext.getActionOwner(); + } + + protected Artifact createArtifact(PathFragment path) { + return analysisEnvironment.getDerivedArtifact(path, configuration.getBinDirectory()); + } + + protected Artifact getInterfaceSoBuilder() { + return analysisEnvironment.getEmbeddedToolArtifact(CppRuleClasses.BUILD_INTERFACE_SO); + } + + /** + * Set the crosstool inputs required for the action. + */ + public Builder setCrosstoolInputs(NestedSet inputs) { + this.crosstoolInputs = inputs; + return this; + } + + /** + * Sets the C++ runtime library inputs for the action. + */ + public Builder setRuntimeInputs(Artifact middleman, NestedSet inputs) { + Preconditions.checkArgument((middleman == null) == inputs.isEmpty()); + this.runtimeMiddleman = middleman; + this.runtimeInputs = inputs; + return this; + } + + /** + * Sets the interface output of the link. A non-null argument can + * only be provided if the link type is {@code DYNAMIC_LIBRARY} + * and fake is false. + */ + public Builder setInterfaceOutputPath(PathFragment path) { + this.interfaceOutputPath = path; + return this; + } + + /** + * Add additional inputs needed for the linkstamp compilation that is being done as part of the + * link. + */ + public Builder addCompilationInputs(Iterable inputs) { + this.compilationInputs.addAll(inputs); + return this; + } + + public Builder addTransitiveCompilationInputs(NestedSet inputs) { + this.compilationInputs.addTransitive(inputs); + return this; + } + + private void addNonLibraryInput(LinkerInput input) { + String name = input.getArtifact().getFilename(); + Preconditions.checkArgument( + !Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) + && !Link.SHARED_LIBRARY_FILETYPES.matches(name), + "'%s' is a library file", input); + this.nonLibraries.add(input); + } + /** + * Adds a single artifact to the set of inputs (C++ source files, header files, etc). Artifacts + * that are not of recognized types will be used for dependency checking but will not be passed + * to the linker. The artifact must not be an archive or a shared library. + */ + public Builder addNonLibraryInput(Artifact input) { + addNonLibraryInput(LinkerInputs.simpleLinkerInput(input)); + return this; + } + + /** + * Adds multiple artifacts to the set of inputs (C++ source files, header files, etc). + * Artifacts that are not of recognized types will be used for dependency checking but will + * not be passed to the linker. The artifacts must not be archives or shared libraries. + */ + public Builder addNonLibraryInputs(Iterable inputs) { + for (Artifact input : inputs) { + addNonLibraryInput(LinkerInputs.simpleLinkerInput(input)); + } + return this; + } + + public Builder addFakeNonLibraryInputs(Iterable inputs) { + for (Artifact input : inputs) { + addNonLibraryInput(LinkerInputs.fakeLinkerInput(input)); + } + return this; + } + + private void checkLibrary(LibraryToLink input) { + String name = input.getArtifact().getFilename(); + Preconditions.checkArgument( + Link.ARCHIVE_LIBRARY_FILETYPES.matches(name) || + Link.SHARED_LIBRARY_FILETYPES.matches(name), + "'%s' is not a library file", input); + } + + /** + * Adds a single artifact to the set of inputs. The artifact must be an archive or a shared + * library. Note that all directly added libraries are implicitly ordered before all nested + * sets added with {@link #addLibraries}, even if added in the opposite order. + */ + public Builder addLibrary(LibraryToLink input) { + checkLibrary(input); + libraries.add(input); + return this; + } + + /** + * Adds multiple artifact to the set of inputs. The artifacts must be archives or shared + * libraries. + */ + public Builder addLibraries(NestedSet inputs) { + for (LibraryToLink input : inputs) { + checkLibrary(input); + } + this.libraries.addTransitive(inputs); + return this; + } + + /** + * Sets the type of ELF file to be created (.a, .so, .lo, executable). The + * default is {@link LinkTargetType#STATIC_LIBRARY}. + */ + public Builder setLinkType(LinkTargetType linkType) { + this.linkType = linkType; + return this; + } + + /** + * Sets the degree of "staticness" of the link: fully static (static binding + * of all symbols), mostly static (use dynamic binding only for symbols from + * glibc), dynamic (use dynamic binding wherever possible). The default is + * {@link LinkStaticness#FULLY_STATIC}. + */ + public Builder setLinkStaticness(LinkStaticness linkStaticness) { + this.linkStaticness = linkStaticness; + return this; + } + + /** + * Adds a C++ source file which will be compiled at link time. This is used + * to embed various values from the build system into binaries to identify + * their provenance. + * + *

Link stamps are also automatically added to the inputs. + */ + public Builder addLinkstamps(Map> linkstamps) { + this.linkstamps.addAll(linkstamps.keySet()); + // Add inputs for linkstamping. + if (!linkstamps.isEmpty()) { + // This will just be the compiler unless include scanning is disabled, in which case it will + // include all header files. Since we insist that linkstamps declare all their headers, all + // header files would be overkill, but that only happens when include scanning is disabled. + addTransitiveCompilationInputs(toolchain.getCompile()); + for (Map.Entry> entry : linkstamps.entrySet()) { + addCompilationInputs(entry.getValue()); + } + } + return this; + } + + public Builder addLinkstampCompilerOptions(ImmutableList linkstampOptions) { + this.linkstampOptions = linkstampOptions; + return this; + } + + /** + * Adds an additional linker option. + */ + public Builder addLinkopt(String linkopt) { + this.linkopts.add(linkopt); + return this; + } + + /** + * Adds multiple linker options at once. + * + * @see #addLinkopt(String) + */ + public Builder addLinkopts(Collection linkopts) { + this.linkopts.addAll(linkopts); + return this; + } + + /** + * Sets whether this link action will be used for a cc_fake_binary; false by + * default. + */ + public Builder setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * Sets whether this link action is used for a native dependency library. + */ + public Builder setNativeDeps(boolean isNativeDeps) { + this.isNativeDeps = isNativeDeps; + return this; + } + + /** + * Setting this to true overrides the default whole-archive computation and force-enables + * whole archives for every archive in the link. This is only necessary for linking executable + * binaries that are supposed to export symbols. + * + *

Usually, the link action while use whole archives for dynamic libraries that are native + * deps (or the legacy whole archive flag is enabled), and that are not dynamically linked. + * + *

(Note that it is possible to build dynamic libraries with cc_binary rules by specifying + * linkshared = 1, and giving the rule a name that matches the pattern {@code + * lib<name>.so}.) + */ + public Builder setWholeArchive(boolean wholeArchive) { + this.wholeArchive = wholeArchive; + return this; + } + + /** + * Sets whether this link action should use test-specific flags (e.g. $EXEC_ORIGIN instead of + * $ORIGIN for the solib search path or lazy binding); false by default. + */ + public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) { + this.useTestOnlyFlags = useTestOnlyFlags; + return this; + } + + /** + * Sets the name of the directory where the solib symlinks for the dynamic runtime libraries + * live. This is usually automatically set from the cc_toolchain. + */ + public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) { + this.runtimeSolibDir = runtimeSolibDir; + return this; + } + + /** + * Creates a builder without the need for a {@link RuleContext}. + * This is to be used exclusively for testing purposes. + * + *

Link stamping is not supported if using this method. + */ + @VisibleForTesting + public static Builder createTestBuilder( + final ActionOwner owner, final AnalysisEnvironment analysisEnvironment, + final PathFragment outputPath, BuildConfiguration config) { + return new Builder(null, outputPath, config, analysisEnvironment, null) { + @Override + protected Artifact createArtifact(PathFragment path) { + return new Artifact(configuration.getBinDirectory().getPath().getRelative(path), + configuration.getBinDirectory(), configuration.getBinFragment().getRelative(path), + analysisEnvironment.getOwner()); + } + @Override + protected ActionOwner getOwner() { + return owner; + } + }; + } + } + + /** + * Immutable ELF linker context, suitable for serialization. + */ + @Immutable @ThreadSafe + public static final class Context implements TransitiveInfoProvider { + // Morally equivalent with {@link Builder}, except these are immutable. + // Keep these in sync with {@link Builder}. + private final ImmutableSet nonLibraries; + private final NestedSet libraries; + private final NestedSet crosstoolInputs; + private final Artifact runtimeMiddleman; + private final NestedSet runtimeInputs; + private final NestedSet compilationInputs; + private final ImmutableSet linkstamps; + private final ImmutableList linkopts; + private final LinkTargetType linkType; + private final LinkStaticness linkStaticness; + private final boolean fake; + private final boolean isNativeDeps; + private final boolean useTestOnlyFlags; + + /** + * Given a {@link Builder}, creates a {@code Context} to pass to another target. + * Note well: Keep the Builder->Context and Context->Builder transforms consistent! + * @param builder a mutable {@link CppLinkAction.Builder} to clone from + */ + public Context(Builder builder) { + this.nonLibraries = ImmutableSet.copyOf(builder.nonLibraries); + this.libraries = NestedSetBuilder.linkOrder() + .addTransitive(builder.libraries.build()).build(); + this.crosstoolInputs = + NestedSetBuilder.stableOrder().addTransitive(builder.crosstoolInputs).build(); + this.runtimeMiddleman = builder.runtimeMiddleman; + this.runtimeInputs = + NestedSetBuilder.stableOrder().addTransitive(builder.runtimeInputs).build(); + this.compilationInputs = NestedSetBuilder.stableOrder() + .addTransitive(builder.compilationInputs.build()).build(); + this.linkstamps = ImmutableSet.copyOf(builder.linkstamps); + this.linkopts = ImmutableList.copyOf(builder.linkopts); + this.linkType = builder.linkType; + this.linkStaticness = builder.linkStaticness; + this.fake = builder.fake; + this.isNativeDeps = builder.isNativeDeps; + this.useTestOnlyFlags = builder.useTestOnlyFlags; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java new file mode 100644 index 0000000000..24a936b281 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionContext.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionContextMarker; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceSet; + +/** + * Context for executing {@link CppLinkAction}s. + */ +@ActionContextMarker(name = "C++ link") +public interface CppLinkActionContext extends ActionContext { + /** + * Returns where the action actually runs. + */ + String strategyLocality(CppLinkAction action); + + /** + * Returns the estimated resource consumption of the action. + */ + ResourceSet estimateResourceConsumption(CppLinkAction action); + + /** + * Executes the specified action. + */ + void exec(CppLinkAction action, + ActionExecutionContext actionExecutionContext) + throws ExecException, ActionExecutionException, InterruptedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java new file mode 100644 index 0000000000..44258a5ab1 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModel.java @@ -0,0 +1,707 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.rules.cpp.CcCompilationOutputs.Builder; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.RegexFilter; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * Representation of a C/C++ compilation. Its purpose is to share the code that creates compilation + * actions between all classes that need to do so. It follows the builder pattern - load up the + * necessary settings and then call {@link #createCcCompileActions}. + * + *

This class is not thread-safe, and it should only be used once for each set of source files, + * i.e. calling {@link #createCcCompileActions} will throw an Exception if called twice. + */ +public final class CppModel { + private final CppSemantics semantics; + private final RuleContext ruleContext; + private final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + + // compile model + private CppCompilationContext context; + private final List> sourceFiles = new ArrayList<>(); + private final List copts = new ArrayList<>(); + private final List additionalIncludes = new ArrayList<>(); + @Nullable private Pattern nocopts; + private boolean fake; + private boolean maySaveTemps; + private boolean onlySingleOutput; + private CcCompilationOutputs compilationOutputs; + private boolean enableLayeringCheck; + private boolean compileHeaderModules; + + // link model + private final List linkopts = new ArrayList<>(); + private LinkTargetType linkType = LinkTargetType.STATIC_LIBRARY; + private boolean neverLink; + private boolean allowInterfaceSharedObjects; + private boolean createDynamicLibrary = true; + private PathFragment soImplFilename; + private FeatureConfiguration featureConfiguration; + + public CppModel(RuleContext ruleContext, CppSemantics semantics) { + this.ruleContext = ruleContext; + this.semantics = semantics; + configuration = ruleContext.getConfiguration(); + cppConfiguration = configuration.getFragment(CppConfiguration.class); + } + + /** + * If the cpp compilation is a fake, then it creates only a single compile action without PIC. + * Defaults to false. + */ + public CppModel setFake(boolean fake) { + this.fake = fake; + return this; + } + + /** + * If set, the CppModel only creates a single .o output that can be linked into a dynamic library, + * i.e., it never generates both PIC and non-PIC outputs. Otherwise it creates outputs that can be + * linked into both static binaries and dynamic libraries (if both require PIC or both require + * non-PIC, then it still only creates a single output). Defaults to false. + */ + public CppModel setOnlySingleOutput(boolean onlySingleOutput) { + this.onlySingleOutput = onlySingleOutput; + return this; + } + + /** + * If set, use compiler flags to enable compiler based layering checks. + */ + public CppModel setEnableLayeringCheck(boolean enableLayeringCheck) { + this.enableLayeringCheck = enableLayeringCheck; + return this; + } + + /** + * If set, add actions that compile header modules to the build. + * See http://clang.llvm.org/docs/Modules.html for more information. + */ + public CppModel setCompileHeaderModules(boolean compileHeaderModules) { + this.compileHeaderModules = compileHeaderModules; + return this; + } + + /** + * Whether to create actions for temps. This defaults to false. + */ + public CppModel setSaveTemps(boolean maySaveTemps) { + this.maySaveTemps = maySaveTemps; + return this; + } + + /** + * Sets the compilation context, i.e. include directories and allowed header files inclusions. + */ + public CppModel setContext(CppCompilationContext context) { + this.context = context; + return this; + } + + /** + * Adds a single source file to be compiled. Note that this should only be called for primary + * compilation units, not for header files or files that are otherwise included. + */ + public CppModel addSources(Iterable sourceFiles, Label sourceLabel) { + for (Artifact sourceFile : sourceFiles) { + this.sourceFiles.add(Pair.of(sourceFile, sourceLabel)); + } + return this; + } + + /** + * Adds all the source files. Note that this should only be called for primary compilation units, + * not for header files or files that are otherwise included. + */ + public CppModel addSources(Iterable> sources) { + Iterables.addAll(this.sourceFiles, sources); + return this; + } + + /** + * Adds the given copts. + */ + public CppModel addCopts(Collection copts) { + this.copts.addAll(copts); + return this; + } + + /** + * Sets the nocopts pattern. This is used to filter out flags from the system defined set of + * flags. By default no filter is applied. + */ + public CppModel setNoCopts(@Nullable Pattern nocopts) { + this.nocopts = nocopts; + return this; + } + + /** + * This can be used to specify additional include directories, without modifying the compilation + * context. + */ + public CppModel addAdditionalIncludes(Collection additionalIncludes) { + // TODO(bazel-team): Maybe this could be handled by the compilation context instead? + this.additionalIncludes.addAll(additionalIncludes); + return this; + } + + /** + * Adds the given linkopts to the optional dynamic library link command. + */ + public CppModel addLinkopts(Collection linkopts) { + this.linkopts.addAll(linkopts); + return this; + } + + /** + * Sets the link type used for the link actions. Note that only static links are supported at this + * time. + */ + public CppModel setLinkTargetType(LinkTargetType linkType) { + this.linkType = linkType; + return this; + } + + public CppModel setNeverLink(boolean neverLink) { + this.neverLink = neverLink; + return this; + } + + /** + * Whether to allow interface dynamic libraries. Note that setting this to true only has an effect + * if the configuration allows it. Defaults to false. + */ + public CppModel setAllowInterfaceSharedObjects(boolean allowInterfaceSharedObjects) { + // TODO(bazel-team): Set the default to true, and require explicit action to disable it. + this.allowInterfaceSharedObjects = allowInterfaceSharedObjects; + return this; + } + + public CppModel setCreateDynamicLibrary(boolean createDynamicLibrary) { + this.createDynamicLibrary = createDynamicLibrary; + return this; + } + + public CppModel setDynamicLibraryPath(PathFragment soImplFilename) { + this.soImplFilename = soImplFilename; + return this; + } + + /** + * Sets the feature configuration to be used for C/C++ actions. + */ + public CppModel setFeatureConfiguration(FeatureConfiguration featureConfiguration) { + this.featureConfiguration = featureConfiguration; + return this; + } + + /** + * @return the non-pic header module artifact for the current target. + */ + public Artifact getHeaderModule(Artifact moduleMapArtifact) { + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + PathFragment outputName = objectDir.getRelative( + semantics.getEffectiveSourcePath(moduleMapArtifact)); + return ruleContext.getRelatedArtifact(outputName, ".pcm"); + } + + /** + * @return the pic header module artifact for the current target. + */ + public Artifact getPicHeaderModule(Artifact moduleMapArtifact) { + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + PathFragment outputName = objectDir.getRelative( + semantics.getEffectiveSourcePath(moduleMapArtifact)); + return ruleContext.getRelatedArtifact(outputName, ".pic.pcm"); + } + + /** + * @return whether this target needs to generate pic actions. + */ + public boolean getGeneratePicActions() { + return CppHelper.usePic(ruleContext, false); + } + + /** + * @return whether this target needs to generate non-pic actions. + */ + public boolean getGenerateNoPicActions() { + return + // If we always need pic for everything, then don't bother to create a no-pic action. + (!CppHelper.usePic(ruleContext, true) || !CppHelper.usePic(ruleContext, false)) + // onlySingleOutput guarantees that the code is only ever linked into a dynamic library - so + // we don't need a no-pic action even if linking into a binary would require it. + && !((onlySingleOutput && getGeneratePicActions())); + } + + /** + * @return whether this target needs to generate a pic header module. + */ + public boolean getGeneratesPicHeaderModule() { + // TODO(bazel-team): Make sure cc_fake_binary works with header module support. + return compileHeaderModules && !fake && getGeneratePicActions(); + } + + /** + * @return whether this target needs to generate a non-pic header module. + */ + public boolean getGeratesNoPicHeaderModule() { + return compileHeaderModules && !fake && getGenerateNoPicActions(); + } + + /** + * Returns a {@code CppCompileActionBuilder} with the common fields for a C++ compile action + * being initialized. + */ + private CppCompileActionBuilder initializeCompileAction(Artifact sourceArtifact, + Label sourceLabel) { + CppCompileActionBuilder builder = createCompileActionBuilder(sourceArtifact, sourceLabel); + if (nocopts != null) { + builder.addNocopts(nocopts); + } + + builder.setEnableLayeringCheck(enableLayeringCheck); + builder.setCompileHeaderModules(compileHeaderModules); + builder.setExtraSystemIncludePrefixes(additionalIncludes); + builder.setFdoBuildStamp(CppHelper.getFdoBuildStamp(cppConfiguration)); + builder.setFeatureConfiguration(featureConfiguration); + return builder; + } + + /** + * Constructs the C++ compiler actions. It generally creates one action for every specified source + * file. It takes into account LIPO, fake-ness, coverage, and PIC, in addition to using the + * settings specified on the current object. This method should only be called once. + */ + public CcCompilationOutputs createCcCompileActions() { + CcCompilationOutputs.Builder result = new CcCompilationOutputs.Builder(); + Preconditions.checkNotNull(context); + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + PathFragment objectDir = CppHelper.getObjDirectory(ruleContext.getLabel()); + + if (compileHeaderModules) { + Artifact moduleMapArtifact = context.getCppModuleMap().getArtifact(); + Label moduleMapLabel = Label.parseAbsoluteUnchecked(context.getCppModuleMap().getName()); + PathFragment outputName = getObjectOutputPath(moduleMapArtifact, objectDir); + CppCompileActionBuilder builder = initializeCompileAction(moduleMapArtifact, moduleMapLabel); + + // A header module compile action is just like a normal compile action, but: + // - the compiled source file is the module map + // - it creates a header module (.pcm file). + createSourceAction(outputName, result, env, moduleMapArtifact, builder, ".pcm"); + } + + for (Pair source : sourceFiles) { + Artifact sourceArtifact = source.getFirst(); + Label sourceLabel = source.getSecond(); + PathFragment outputName = getObjectOutputPath(sourceArtifact, objectDir); + CppCompileActionBuilder builder = initializeCompileAction(sourceArtifact, sourceLabel); + + if (CppFileTypes.CPP_HEADER.matches(source.first.getExecPath())) { + createHeaderAction(outputName, result, env, builder); + } else { + createSourceAction(outputName, result, env, sourceArtifact, builder, ".o"); + } + } + + compilationOutputs = result.build(); + return compilationOutputs; + } + + private void createHeaderAction(PathFragment outputName, Builder result, AnalysisEnvironment env, + CppCompileActionBuilder builder) { + builder.setOutputFile(ruleContext.getRelatedArtifact(outputName, ".h.processed")).setDotdFile( + outputName, ".h.d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction compileAction = builder.build(); + env.registerAction(compileAction); + Artifact tokenFile = compileAction.getOutputFile(); + result.addHeaderTokenFile(tokenFile); + } + + private void createSourceAction(PathFragment outputName, + CcCompilationOutputs.Builder result, + AnalysisEnvironment env, + Artifact sourceArtifact, + CppCompileActionBuilder builder, + String outputExtension) { + PathFragment ccRelativeName = semantics.getEffectiveSourcePath(sourceArtifact); + LipoContextProvider lipoProvider = null; + if (cppConfiguration.isLipoOptimization()) { + // TODO(bazel-team): we shouldn't be needing this, merging context with the binary + // is a superset of necessary information. + lipoProvider = Preconditions.checkNotNull(CppHelper.getLipoContextProvider(ruleContext), + outputName); + builder.setContext(CppCompilationContext.mergeForLipo(lipoProvider.getLipoContext(), + context)); + } + if (fake) { + // For cc_fake_binary, we only create a single fake compile action. It's + // not necessary to use -fPIC for negative compilation tests, and using + // .pic.o files in cc_fake_binary would break existing uses of + // cc_fake_binary. + Artifact outputFile = ruleContext.getRelatedArtifact(outputName, outputExtension); + PathFragment tempOutputName = + FileSystemUtils.replaceExtension(outputFile.getExecPath(), ".temp" + outputExtension); + builder + .setOutputFile(outputFile) + .setDotdFile(outputName, ".d", ruleContext) + .setTempOutputFile(tempOutputName); + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction action = builder.build(); + env.registerAction(action); + result.addObjectFile(action.getOutputFile()); + } else { + boolean generatePicAction = getGeneratePicActions(); + // If we always need pic for everything, then don't bother to create a no-pic action. + boolean generateNoPicAction = getGenerateNoPicActions(); + Preconditions.checkState(generatePicAction || generateNoPicAction); + + // Create PIC compile actions (same as non-PIC, but use -fPIC and + // generate .pic.o, .pic.d, .pic.gcno instead of .o, .d, .gcno.) + if (generatePicAction) { + CppCompileActionBuilder picBuilder = copyAsPicBuilder(builder, outputName, outputExtension); + cppConfiguration.getFdoSupport().configureCompilation(picBuilder, ruleContext, env, + ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/true, + lipoProvider); + + if (maySaveTemps) { + result.addTemps( + createTempsActions(sourceArtifact, outputName, picBuilder, /*usePic=*/true)); + } + + if (isCodeCoverageEnabled()) { + picBuilder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".pic.gcno")); + } + + semantics.finalizeCompileActionBuilder(ruleContext, picBuilder); + CppCompileAction picAction = picBuilder.build(); + env.registerAction(picAction); + result.addPicObjectFile(picAction.getOutputFile()); + if (picAction.getDwoFile() != null) { + // Host targets don't produce .dwo files. + result.addPicDwoFile(picAction.getDwoFile()); + } + if (cppConfiguration.isLipoContextCollector() && !generateNoPicAction) { + result.addLipoScannable(picAction); + } + } + + if (generateNoPicAction) { + builder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, outputExtension)) + .setDotdFile(outputName, ".d", ruleContext); + // Create non-PIC compile actions + cppConfiguration.getFdoSupport().configureCompilation(builder, ruleContext, env, + ruleContext.getLabel(), ccRelativeName, nocopts, /*usePic=*/false, + lipoProvider); + + if (maySaveTemps) { + result.addTemps( + createTempsActions(sourceArtifact, outputName, builder, /*usePic=*/false)); + } + + if (!cppConfiguration.isLipoOptimization() && isCodeCoverageEnabled()) { + builder.setGcnoFile(ruleContext.getRelatedArtifact(outputName, ".gcno")); + } + + semantics.finalizeCompileActionBuilder(ruleContext, builder); + CppCompileAction compileAction = builder.build(); + env.registerAction(compileAction); + Artifact objectFile = compileAction.getOutputFile(); + result.addObjectFile(objectFile); + if (compileAction.getDwoFile() != null) { + // Host targets don't produce .dwo files. + result.addDwoFile(compileAction.getDwoFile()); + } + if (cppConfiguration.isLipoContextCollector()) { + result.addLipoScannable(compileAction); + } + } + } + } + + /** + * Constructs the C++ linker actions. It generally generates two actions, one for a static library + * and one for a dynamic library. If PIC is required for shared libraries, but not for binaries, + * it additionally creates a third action to generate a PIC static library. + * + *

For dynamic libraries, this method can additionally create an interface shared library that + * can be used for linking, but doesn't contain any executable code. This increases the number of + * cache hits for link actions. Call {@link #setAllowInterfaceSharedObjects(boolean)} to enable + * this behavior. + */ + public CcLinkingOutputs createCcLinkActions(CcCompilationOutputs ccOutputs) { + // For now only handle static links. Note that the dynamic library link below ignores linkType. + // TODO(bazel-team): Either support non-static links or move this check to setLinkType(). + Preconditions.checkState(linkType.isStaticLibraryLink(), "can only handle static links"); + + CcLinkingOutputs.Builder result = new CcLinkingOutputs.Builder(); + if (cppConfiguration.isLipoContextCollector()) { + // Don't try to create LIPO link actions in collector mode, + // because it needs some data that's not available at this point. + return result.build(); + } + + AnalysisEnvironment env = ruleContext.getAnalysisEnvironment(); + boolean usePicForBinaries = CppHelper.usePic(ruleContext, true); + boolean usePicForSharedLibs = CppHelper.usePic(ruleContext, false); + + // Create static library (.a). The linkType only reflects whether the library is alwayslink or + // not. The PIC-ness is determined by whether we need to use PIC or not. There are three cases + // for (usePicForSharedLibs usePicForBinaries): + // + // (1) (false false) -> no pic code + // (2) (true false) -> shared libraries as pic, but not binaries + // (3) (true true) -> both shared libraries and binaries as pic + // + // In case (3), we always need PIC, so only create one static library containing the PIC object + // files. The name therefore does not match the content. + // + // Presumably, it is done this way because the .a file is an implicit output of every cc_library + // rule, so we can't use ".pic.a" that in the always-PIC case. + PathFragment linkedFileName = CppHelper.getLinkedFilename(ruleContext, linkType); + CppLinkAction maybePicAction = newLinkActionBuilder(linkedFileName) + .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForBinaries)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(linkType) + .setLinkStaticness(LinkStaticness.FULLY_STATIC) + .build(); + env.registerAction(maybePicAction); + result.addStaticLibrary(maybePicAction.getOutputLibrary()); + + // Create a second static library (.pic.a). Only in case (2) do we need both PIC and non-PIC + // static libraries. In that case, the first static library contains the non-PIC code, and this + // one contains the PIC code, so the names match the content. + if (!usePicForBinaries && usePicForSharedLibs) { + LinkTargetType picLinkType = (linkType == LinkTargetType.ALWAYS_LINK_STATIC_LIBRARY) + ? LinkTargetType.ALWAYS_LINK_PIC_STATIC_LIBRARY + : LinkTargetType.PIC_STATIC_LIBRARY; + + PathFragment picFileName = CppHelper.getLinkedFilename(ruleContext, picLinkType); + CppLinkAction picAction = newLinkActionBuilder(picFileName) + .addNonLibraryInputs(ccOutputs.getObjectFiles(true)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(picLinkType) + .setLinkStaticness(LinkStaticness.FULLY_STATIC) + .build(); + env.registerAction(picAction); + result.addPicStaticLibrary(picAction.getOutputLibrary()); + } + + if (!createDynamicLibrary) { + return result.build(); + } + + // Create dynamic library. + if (soImplFilename == null) { + soImplFilename = CppHelper.getLinkedFilename(ruleContext, LinkTargetType.DYNAMIC_LIBRARY); + } + List sonameLinkopts = ImmutableList.of(); + PathFragment soInterfaceFilename = null; + if (cppConfiguration.useInterfaceSharedObjects() && allowInterfaceSharedObjects) { + soInterfaceFilename = + CppHelper.getLinkedFilename(ruleContext, LinkTargetType.INTERFACE_DYNAMIC_LIBRARY); + Artifact dynamicLibrary = env.getDerivedArtifact( + soImplFilename, configuration.getBinDirectory()); + sonameLinkopts = ImmutableList.of("-Wl,-soname=" + + SolibSymlinkAction.getDynamicLibrarySoname(dynamicLibrary.getRootRelativePath(), false)); + } + + // Should we also link in any libraries that this library depends on? + // That is required on some systems... + CppLinkAction action = newLinkActionBuilder(soImplFilename) + .setInterfaceOutputPath(soInterfaceFilename) + .addNonLibraryInputs(ccOutputs.getObjectFiles(usePicForSharedLibs)) + .addNonLibraryInputs(ccOutputs.getHeaderTokenFiles()) + .setLinkType(LinkTargetType.DYNAMIC_LIBRARY) + .setLinkStaticness(LinkStaticness.DYNAMIC) + .addLinkopts(linkopts) + .addLinkopts(sonameLinkopts) + .setRuntimeInputs( + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkMiddleman(), + CppHelper.getToolchain(ruleContext).getDynamicRuntimeLinkInputs()) + .build(); + env.registerAction(action); + + LibraryToLink dynamicLibrary = action.getOutputLibrary(); + LibraryToLink interfaceLibrary = action.getInterfaceOutputLibrary(); + if (interfaceLibrary == null) { + interfaceLibrary = dynamicLibrary; + } + + // If shared library has neverlink=1, then leave it untouched. Otherwise, + // create a mangled symlink for it and from now on reference it through + // mangled name only. + if (neverLink) { + result.addDynamicLibrary(interfaceLibrary); + result.addExecutionDynamicLibrary(dynamicLibrary); + } else { + LibraryToLink libraryLink = SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, interfaceLibrary.getArtifact(), false, false, + ruleContext.getConfiguration()); + result.addDynamicLibrary(libraryLink); + LibraryToLink implLibraryLink = SolibSymlinkAction.getDynamicLibrarySymlink( + ruleContext, dynamicLibrary.getArtifact(), false, false, + ruleContext.getConfiguration()); + result.addExecutionDynamicLibrary(implLibraryLink); + } + return result.build(); + } + + private CppLinkAction.Builder newLinkActionBuilder(PathFragment outputPath) { + return new CppLinkAction.Builder(ruleContext, outputPath) + .setCrosstoolInputs(CppHelper.getToolchain(ruleContext).getLink()) + .addNonLibraryInputs(context.getCompilationPrerequisites()); + } + + /** + * Returns the output artifact path relative to the object directory. + */ + private PathFragment getObjectOutputPath(Artifact source, PathFragment objectDirectory) { + return objectDirectory.getRelative(semantics.getEffectiveSourcePath(source)); + } + + /** + * Creates a basic cpp compile action builder for source file. Configures options, + * crosstool inputs, output and dotd file names, compilation context and copts. + */ + private CppCompileActionBuilder createCompileActionBuilder( + Artifact source, Label label) { + CppCompileActionBuilder builder = new CppCompileActionBuilder( + ruleContext, source, label); + + builder + .setContext(context) + .addCopts(copts); + return builder; + } + + /** + * Creates cpp PIC compile action builder from the given builder by adding necessary copt and + * changing output and dotd file names. + */ + private CppCompileActionBuilder copyAsPicBuilder(CppCompileActionBuilder builder, + PathFragment outputName, String outputExtension) { + CppCompileActionBuilder picBuilder = new CppCompileActionBuilder(builder); + picBuilder.addCopt("-fPIC") + .setOutputFile(ruleContext.getRelatedArtifact(outputName, ".pic" + outputExtension)) + .setDotdFile(outputName, ".pic.d", ruleContext); + return picBuilder; + } + + /** + * Create the actions for "--save_temps". + */ + private ImmutableList createTempsActions(Artifact source, PathFragment outputName, + CppCompileActionBuilder builder, boolean usePic) { + if (!cppConfiguration.getSaveTemps()) { + return ImmutableList.of(); + } + + String path = source.getFilename(); + boolean isCFile = CppFileTypes.C_SOURCE.matches(path); + boolean isCppFile = CppFileTypes.CPP_SOURCE.matches(path); + + if (!isCFile && !isCppFile) { + return ImmutableList.of(); + } + + String iExt = isCFile ? ".i" : ".ii"; + String picExt = usePic ? ".pic" : ""; + CppCompileActionBuilder dBuilder = new CppCompileActionBuilder(builder); + CppCompileActionBuilder sdBuilder = new CppCompileActionBuilder(builder); + + dBuilder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + iExt)) + .setDotdFile(outputName, picExt + iExt + ".d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, dBuilder); + CppCompileAction dAction = dBuilder.build(); + ruleContext.registerAction(dAction); + + sdBuilder + .setOutputFile(ruleContext.getRelatedArtifact(outputName, picExt + ".s")) + .setDotdFile(outputName, picExt + ".s.d", ruleContext); + semantics.finalizeCompileActionBuilder(ruleContext, sdBuilder); + CppCompileAction sdAction = sdBuilder.build(); + ruleContext.registerAction(sdAction); + return ImmutableList.of( + dAction.getOutputFile(), + sdAction.getOutputFile()); + } + + /** + * Returns true iff code coverage is enabled for the given target. + */ + private boolean isCodeCoverageEnabled() { + if (configuration.isCodeCoverageEnabled()) { + final RegexFilter filter = configuration.getInstrumentationFilter(); + // If rule is matched by the instrumentation filter, enable instrumentation + if (filter.isIncluded(ruleContext.getLabel().toString())) { + return true; + } + // At this point the rule itself is not matched by the instrumentation filter. However, we + // might still want to instrument C++ rules if one of the targets listed in "deps" is + // instrumented and, therefore, can supply header files that we would want to collect code + // coverage for. For example, think about cc_test rule that tests functionality defined in a + // header file that is supplied by the cc_library. + // + // Note that we only check direct prerequisites and not the transitive closure. This is done + // for two reasons: + // a) It is a good practice to declare libraries which you directly rely on. Including headers + // from a library hidden deep inside the transitive closure makes build dependencies less + // readable and can lead to unexpected breakage. + // b) Traversing the transitive closure for each C++ compile action would require more complex + // implementation (with caching results of this method) to avoid O(N^2) slowdown. + if (ruleContext.getRule().isAttrDefined("deps", Type.LABEL_LIST)) { + for (TransitiveInfoCollection dep : ruleContext.getPrerequisites("deps", Mode.TARGET)) { + if (dep.getProvider(CppCompilationContext.class) != null + && filter.isIncluded(dep.getLabel().toString())) { + return true; + } + } + } + } + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java new file mode 100644 index 0000000000..bb272098a4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMap.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * Structure for C++ module maps. Stores the name of the module and a .cppmap artifact. + */ +@Immutable +public class CppModuleMap { + private final Artifact artifact; + private final String name; + + public CppModuleMap(Artifact artifact, String name) { + this.artifact = artifact; + this.name = name; + } + + public Artifact getArtifact() { + return artifact; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return name + "@" + artifact; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java new file mode 100644 index 0000000000..a350fc4618 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppModuleMapAction.java @@ -0,0 +1,185 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Creates C++ module map artifact genfiles. These are then passed to Clang to + * do dependency checking. + */ +public class CppModuleMapAction extends AbstractFileWriteAction { + + private static final String GUID = "4f407081-1951-40c1-befc-d6b4daff5de3"; + + // C++ module map of the current target + private final CppModuleMap cppModuleMap; + + /** + * If set, the paths in the module map are relative to the current working directory instead + * of relative to the module map file's location. + */ + private final boolean moduleMapHomeIsCwd; + + // Headers and dependencies list + private final ImmutableList privateHeaders; + private final ImmutableList publicHeaders; + private final ImmutableList dependencies; + private final ImmutableList additionalExportedHeaders; + private final boolean compiledModule; + + public CppModuleMapAction(ActionOwner owner, CppModuleMap cppModuleMap, + Iterable privateHeaders, Iterable publicHeaders, + Iterable dependencies, Iterable additionalExportedHeaders, + boolean compiledModule, boolean moduleMapHomeIsCwd) { + super(owner, ImmutableList.of(), cppModuleMap.getArtifact(), + /*makeExecutable=*/false); + this.cppModuleMap = cppModuleMap; + this.moduleMapHomeIsCwd = moduleMapHomeIsCwd; + this.privateHeaders = ImmutableList.copyOf(privateHeaders); + this.publicHeaders = ImmutableList.copyOf(publicHeaders); + this.dependencies = ImmutableList.copyOf(dependencies); + this.additionalExportedHeaders = ImmutableList.copyOf(additionalExportedHeaders); + this.compiledModule = compiledModule; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) { + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + StringBuilder content = new StringBuilder(); + PathFragment fragment = cppModuleMap.getArtifact().getExecPath(); + int segmentsToExecPath = fragment.segmentCount() - 1; + + // For details about the different header types, see: + // http://clang.llvm.org/docs/Modules.html#header-declaration + String leadingPeriods = moduleMapHomeIsCwd ? "" : Strings.repeat("../", segmentsToExecPath); + content.append("module \"").append(cppModuleMap.getName()).append("\" {\n"); + content.append(" export *\n"); + for (Artifact artifact : privateHeaders) { + appendHeader(content, "private", artifact.getExecPath(), leadingPeriods, + /*canCompile=*/true); + } + for (Artifact artifact : publicHeaders) { + appendHeader(content, "", artifact.getExecPath(), leadingPeriods, /*canCompile=*/true); + } + for (PathFragment additionalExportedHeader : additionalExportedHeaders) { + appendHeader(content, "", additionalExportedHeader, leadingPeriods, /*canCompile*/false); + } + for (CppModuleMap dep : dependencies) { + content.append(" use \"").append(dep.getName()).append("\"\n"); + } + content.append("}"); + for (CppModuleMap dep : dependencies) { + content.append("\nextern module \"") + .append(dep.getName()) + .append("\" \"") + .append(leadingPeriods) + .append(dep.getArtifact().getExecPath()) + .append("\""); + } + out.write(content.toString().getBytes(StandardCharsets.ISO_8859_1)); + } + }; + } + + private void appendHeader(StringBuilder content, String visibilitySpecifier, PathFragment path, + String leadingPeriods, boolean canCompile) { + content.append(" "); + if (!visibilitySpecifier.isEmpty()) { + content.append(visibilitySpecifier).append(" "); + } + if (!canCompile || !shouldCompileHeader(path)) { + content.append("textual "); + } + content.append("header \"").append(leadingPeriods).append(path).append("\"\n"); + } + + private boolean shouldCompileHeader(PathFragment path) { + return compiledModule && !CppFileTypes.CPP_TEXTUAL_INCLUDE.matches(path); + } + + @Override + public String getMnemonic() { + return "CppModuleMap"; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addInt(privateHeaders.size()); + for (Artifact artifact : privateHeaders) { + f.addPath(artifact.getRootRelativePath()); + } + f.addInt(publicHeaders.size()); + for (Artifact artifact : publicHeaders) { + f.addPath(artifact.getRootRelativePath()); + } + f.addInt(dependencies.size()); + for (CppModuleMap dep : dependencies) { + f.addPath(dep.getArtifact().getExecPath()); + } + f.addPath(cppModuleMap.getArtifact().getExecPath()); + f.addString(cppModuleMap.getName()); + return f.hexDigestAndReset(); + } + + @Override + public ResourceSet estimateResourceConsumptionLocal() { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.02); + } + + @VisibleForTesting + public Collection getPublicHeaders() { + return publicHeaders; + } + + @VisibleForTesting + public Collection getPrivateHeaders() { + return privateHeaders; + } + + @VisibleForTesting + public ImmutableList getAdditionalExportedHeaders() { + return additionalExportedHeaders; + } + + @VisibleForTesting + public Collection getDependencyArtifacts() { + List artifacts = new ArrayList<>(); + for (CppModuleMap map : dependencies) { + artifacts.add(map.getArtifact()); + } + return artifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java new file mode 100644 index 0000000000..b0f2e820a5 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppOptions.java @@ -0,0 +1,646 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelConverter; +import com.google.devtools.build.lib.analysis.config.CompilationMode; +import com.google.devtools.build.lib.analysis.config.FragmentOptions; +import com.google.devtools.build.lib.analysis.config.PerLabelOptions; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.HeadersCheckingMode; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.LibcTop; +import com.google.devtools.build.lib.rules.cpp.CppConfiguration.StripMode; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsParsingException; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Command-line options for C++. + */ +public class CppOptions extends FragmentOptions { + /** + * Label of a filegroup that contains all crosstool files for all configurations. + */ + @VisibleForTesting + public static final String DEFAULT_CROSSTOOL_TARGET = "//tools/cpp:toolchain"; + + + /** + * Converter for --cwarn flag + */ + public static class GccWarnConverter implements Converter { + @Override + public String convert(String input) throws OptionsParsingException { + if (input.startsWith("no-") || input.startsWith("-W")) { + throw new OptionsParsingException("Not a valid gcc warning to enable"); + } + return input; + } + + @Override + public String getTypeDescription() { + return "A gcc warning to enable"; + } + } + + /** + * Converts a comma-separated list of compilation mode settings to a properly typed List. + */ + public static class FissionOptionConverter implements Converter> { + @Override + public List convert(String input) throws OptionsParsingException { + ImmutableSet.Builder modes = ImmutableSet.builder(); + if (input.equals("yes")) { // Special case: enable all modes. + modes.add(CompilationMode.values()); + } else if (!input.equals("no")) { // "no" is another special case that disables all modes. + CompilationMode.Converter modeConverter = new CompilationMode.Converter(); + for (String mode : Splitter.on(',').split(input)) { + modes.add(modeConverter.convert(mode)); + } + } + return modes.build().asList(); + } + + @Override + public String getTypeDescription() { + return "a set of compilation modes"; + } + } + + /** + * The same as DynamicMode, but on command-line we also allow AUTO. + */ + public enum DynamicModeFlag { OFF, DEFAULT, FULLY, AUTO } + + /** + * Converter for DynamicModeFlag + */ + public static class DynamicModeConverter extends EnumConverter { + public DynamicModeConverter() { + super(DynamicModeFlag.class, "dynamic mode"); + } + } + + /** + * Converter for the --strip option. + */ + public static class StripModeConverter extends EnumConverter { + public StripModeConverter() { + super(StripMode.class, "strip mode"); + } + } + + private static final String LIBC_RELATIVE_LABEL = ":everything"; + + /** + * Converts a String, which is an absolute path or label into a LibcTop + * object. + */ + public static class LibcTopConverter implements Converter { + @Override + public LibcTop convert(String input) throws OptionsParsingException { + if (!input.startsWith("//")) { + throw new OptionsParsingException("Not a label"); + } + try { + Label label = Label.parseAbsolute(input).getRelative(LIBC_RELATIVE_LABEL); + return new LibcTop(label); + } catch (SyntaxException e) { + throw new OptionsParsingException(e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a label"; + } + } + + /** + * Converter for the --hdrs_check option. + */ + public static class HdrsCheckConverter extends EnumConverter { + public HdrsCheckConverter() { + super(HeadersCheckingMode.class, "Headers check mode"); + } + } + + /** + * Checks whether a string is a valid regex pattern and compiles it. + */ + public static class NullableRegexPatternConverter implements Converter { + + @Override + public Pattern convert(String input) throws OptionsParsingException { + if (input.isEmpty()) { + return null; + } + try { + return Pattern.compile(input); + } catch (PatternSyntaxException e) { + throw new OptionsParsingException("Not a valid regular expression: " + e.getMessage()); + } + } + + @Override + public String getTypeDescription() { + return "a valid Java regular expression"; + } + } + + /** + * Converter for the --lipo option. + */ + public static class LipoModeConverter extends EnumConverter { + public LipoModeConverter() { + super(LipoMode.class, "LIPO mode"); + } + } + + @Option(name = "lipo input collector", + defaultValue = "false", + category = "undocumented", + help = "Internal flag, only used to create configurations with the LIPO-collector flag set.") + public boolean lipoCollector; + + @Option(name = "crosstool_top", + defaultValue = CppOptions.DEFAULT_CROSSTOOL_TARGET, + category = "version", + converter = LabelConverter.class, + help = "The label of the crosstool package to be used for compiling C++ code.") + public Label crosstoolTop; + + @Option(name = "compiler", + defaultValue = "null", + category = "version", + help = "The C++ compiler to use for compiling the target.") + public String cppCompiler; + + @Option(name = "glibc", + defaultValue = "null", + category = "version", + help = "The version of glibc the target should be linked against. " + + "By default, a suitable version is chosen based on --cpu.") + public String glibc; + + @Option(name = "thin_archives", + defaultValue = "false", + category = "strategy", // but also adds edges to the action graph + help = "Pass the 'T' flag to ar if supported by the toolchain. " + + "All supported toolchains support this setting.") + public boolean useThinArchives; + + // O intrepid reaper of unused options: Be warned that the [no]start_end_lib + // option, however tempting to remove, has a use case. Look in our telemetry data. + @Option(name = "start_end_lib", + defaultValue = "true", + category = "strategy", // but also adds edges to the action graph + help = "Use the --start-lib/--end-lib ld options if supported by the toolchain.") + public boolean useStartEndLib; + + @Option(name = "interface_shared_objects", + defaultValue = "true", + category = "strategy", // but also adds edges to the action graph + help = "Use interface shared objects if supported by the toolchain. " + + "All ELF toolchains currently support this setting.") + public boolean useInterfaceSharedObjects; + + @Option(name = "cc_include_scanning", + defaultValue = "true", + category = "strategy", + help = "Whether to perform include scanning. Without it, your build will most likely " + + "fail.") + public boolean scanIncludes; + + @Option(name = "extract_generated_inclusions", + defaultValue = "true", + category = "undocumented", + help = "Run grep-includes actions (used for include scanning) over " + + "generated headers and sources.") + public boolean extractInclusions; + + @Option(name = "fission", + defaultValue = "no", + converter = FissionOptionConverter.class, + category = "semantics", + help = "Specifies which compilation modes use fission for C++ compilations and links. " + + " May be any combination of {'fastbuild', 'dbg', 'opt'} or the special values 'yes' " + + " to enable all modes and 'no' to disable all modes.") + public List fissionModes; + + @Option(name = "dynamic_mode", + defaultValue = "default", + converter = DynamicModeConverter.class, + category = "semantics", + help = "Determines whether C++ binaries will be linked dynamically. 'default' means " + + "blaze will choose whether to link dynamically. 'fully' means all libraries " + + "will be linked dynamically. 'off' means that all libraries will be linked " + + "in mostly static mode.") + public DynamicModeFlag dynamicMode; + + @Option(name = "force_pic", + defaultValue = "false", + category = "semantics", + help = "If enabled, all C++ compilations produce position-independent code (\"-fPIC\")," + + " links prefer PIC pre-built libraries over non-PIC libraries, and links produce" + + " position-independent executables (\"-pie\").") + public boolean forcePic; + + @Option(name = "force_ignore_dash_static", + defaultValue = "false", + category = "semantics", + help = "If set, '-static' options in the linkopts of cc_* rules will be ignored.") + public boolean forceIgnoreDashStatic; + + @Option(name = "experimental_skip_static_outputs", + defaultValue = "false", + category = "semantics", + help = "This flag is experimental and may go away at any time. " + + "If true, linker output for mostly-static C++ executables is a tiny amount of " + + "dummy dependency information, and NOT a usable binary. Kludge, but can reduce " + + "network and disk I/O load (and thus, continuous build cycle times) by a lot. " + + "NOTE: use of this flag REQUIRES --distinct_host_configuration.") + public boolean skipStaticOutputs; + + @Option(name = "hdrs_check", + allowMultiple = false, + defaultValue = "loose", + converter = HdrsCheckConverter.class, + category = "semantics", + help = "Headers check mode for rules that don't specify it explicitly using a " + + "hdrs_check attribute. Allowed values: 'loose' allows undeclared headers, 'warn' " + + "warns about undeclared headers, and 'strict' disallows them.") + public HeadersCheckingMode headersCheckingMode; + + @Option(name = "copt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to gcc.") + public List coptList; + + @Option(name = "cwarn", + converter = GccWarnConverter.class, + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional warnings to enable when compiling C or C++ source files.") + public List cWarns; + + @Option(name = "cxxopt", + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional option to pass to gcc when compiling C++ source files.") + public List cxxoptList; + + @Option(name = "conlyopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional option to pass to gcc when compiling C source files.") + public List conlyoptList; + + @Option(name = "linkopt", + defaultValue = "", + category = "flags", + allowMultiple = true, + help = "Additional option to pass to gcc when linking.") + public List linkoptList; + + @Option(name = "stripopt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to strip when generating a '.stripped' binary.") + public List stripoptList; + + @Option(name = "custom_malloc", + defaultValue = "null", + category = "semantics", + help = "Specifies a custom malloc implementation. This setting overrides malloc " + + "attributes in build rules.", + converter = LabelConverter.class) + public Label customMalloc; + + @Option(name = "cpp_module_maps", + defaultValue = "true", + category = "flags", + help = "If true then C++ targets create a module map based on BUILD files, and " + + "pass them to the compiler.") + public boolean cppModuleMaps; + + @Option(name = "legacy_whole_archive", + defaultValue = "true", + category = "semantics", + help = "When on, use --whole-archive for cc_binary rules that have " + + "linkshared=1 and either linkstatic=1 or '-static' in linkopts. " + + "This is for backwards compatibility only. " + + "A better alternative is to use alwayslink=1 where required.") + public boolean legacyWholeArchive; + + @Option(name = "strip", + defaultValue = "sometimes", + category = "flags", + help = "Specifies whether to strip binaries and shared libraries " + + " (using \"-Wl,--strip-debug\"). The default value of 'sometimes'" + + " means strip iff --compilation_mode=fastbuild.", + converter = StripModeConverter.class) + public StripMode stripBinaries; + + @Option(name = "fdo_instrument", + defaultValue = "null", + converter = OptionsUtils.PathFragmentConverter.class, + category = "flags", + implicitRequirements = {"--copt=-Wno-error"}, + help = "Generate binaries with FDO instrumentation. Specify the relative " + + "directory name for the .gcda files at runtime.") + public PathFragment fdoInstrument; + + @Option(name = "fdo_optimize", + defaultValue = "null", + category = "flags", + help = "Use FDO profile information to optimize compilation. Specify the name " + + "of the zip file containing the .gcda file tree or an afdo file containing " + + "an auto profile. This flag also accepts files specified as labels, for " + + "example //foo/bar:file.afdo. Such labels must refer to input files; you may " + + "need to add an exports_files directive to the corresponding package to make " + + "the file visible to Blaze.") + public String fdoOptimize; + + @Option(name = "autofdo_lipo_data", + defaultValue = "false", + category = "flags", + help = "If true then the directory name for non-LIPO targets will have a " + + "'-lipodata' suffix in AutoFDO mode.") + public boolean autoFdoLipoData; + + @Option(name = "lipo", + defaultValue = "off", + converter = LipoModeConverter.class, + category = "flags", + help = "Enable LIPO optimization (lightweight inter-procedural optimization, The allowed " + + "values for this option are 'off' and 'binary', which enables LIPO. This option only " + + "has an effect when FDO is also enabled. Currently LIPO is only supported when " + + "building a single cc_binary rule.") + public LipoMode lipoMode; + + @Option(name = "lipo_context", + defaultValue = "null", + category = "flags", + converter = LabelConverter.class, + implicitRequirements = {"--linkopt=-Wl,--warn-unresolved-symbols"}, + help = "Specifies the binary from which the LIPO profile information comes.") + public Label lipoContext; + + @Option(name = "experimental_stl", + converter = LabelConverter.class, + defaultValue = "null", + category = "version", + help = "If set, use this label instead of the default STL implementation. " + + "This option is EXPERIMENTAL and may go away in a future release.") + public Label stl; + + @Option(name = "save_temps", + defaultValue = "false", + category = "what", + help = "If set, temporary outputs from gcc will be saved. " + + "These include .s files (assembler code), .i files (preprocessed C) and " + + ".ii files (preprocessed C++).") + public boolean saveTemps; + + @Option(name = "per_file_copt", + allowMultiple = true, + converter = PerLabelOptions.PerLabelOptionsConverter.class, + defaultValue = "", + category = "semantics", + help = "Additional options to selectively pass to gcc when compiling certain files. " + + "This option can be passed multiple times. " + + "Syntax: regex_filter@option_1,option_2,...,option_n. Where regex_filter stands " + + "for a list of include and exclude regular expression patterns (Also see " + + "--instrumentation_filter). option_1 to option_n stand for " + + "arbitrary command line options. If an option contains a comma it has to be " + + "quoted with a backslash. Options can contain @. Only the first @ is used to " + + "split the string. Example: " + + "--per_file_copt=//foo/.*\\.cc,-//foo/bar\\.cc@-O0 adds the -O0 " + + "command line option to the gcc command line of all cc files in //foo/ " + + "except bar.cc.") + public List perFileCopts; + + @Option(name = "host_crosstool_top", + defaultValue = "null", + converter = LabelConverter.class, + category = "semantics", + help = "By default, the --crosstool_top, --glibc, and --compiler options are also used " + + "for the host configuration. If this flag is provided, Blaze uses the default glibc " + + "and compiler for the given crosstool_top.") + public Label hostCrosstoolTop; + + @Option(name = "host_copt", + allowMultiple = true, + defaultValue = "", + category = "flags", + help = "Additional options to pass to gcc for host tools.") + public List hostCoptList; + + @Option(name = "define", + converter = Converters.AssignmentConverter.class, + defaultValue = "", + category = "semantics", + allowMultiple = true, + help = "Each --define option specifies an assignment for a build variable.") + public List> commandLineDefinedVariables; + + @Option(name = "grte_top", + defaultValue = "null", // The default value is chosen by the toolchain. + category = "version", + converter = LibcTopConverter.class, + help = "A label to a checked-in libc library. The default value is selected by the crosstool " + + "toolchain, and you almost never need to override it.") + public LibcTop libcTop; + + @Option(name = "host_grte_top", + defaultValue = "null", // The default value is chosen by the toolchain. + category = "version", + converter = LibcTopConverter.class, + help = "If specified, this setting overrides the libc top-level directory (--grte_top) " + + "for the host configuration.") + public LibcTop hostLibcTop; + + @Option(name = "output_symbol_counts", + defaultValue = "false", + category = "flags", + help = "If enabled, every C++ binary linked with gold will store the number of used " + + "symbols per object file in a .sc file.") + public boolean symbolCounts; + + @Option(name = "experimental_inmemory_dotd_files", + defaultValue = "false", + category = "experimental", + help = "If enabled, C++ .d files will be passed through in memory directly from the remote " + + "build nodes instead of being written to disk.") + public boolean inmemoryDotdFiles; + + @Option(name = "use_isystem_for_includes", + defaultValue = "true", + category = "undocumented", + help = "Instruct C and C++ compilations to treat 'includes' paths as system header " + + "paths, by translating it into -isystem instead of -I.") + public boolean useIsystemForIncludes; + + @Option(name = "experimental_omitfp", + defaultValue = "false", + category = "semantics", + help = "If true, use libunwind for stack unwinding, and compile with " + + "-fomit-frame-pointer and -fasynchronous-unwind-tables.") + public boolean experimentalOmitfp; + + @Option(name = "share_native_deps", + defaultValue = "true", + category = "strategy", + help = "If true, native libraries that contain identical functionality " + + "will be shared among different targets") + public boolean shareNativeDeps; + + @Override + public FragmentOptions getHost(boolean fallback) { + CppOptions host = (CppOptions) getDefault(); + + host.commandLineDefinedVariables = commandLineDefinedVariables; + + // The crosstool options are partially copied from the target configuration. + if (!fallback) { + if (hostCrosstoolTop == null) { + host.cppCompiler = cppCompiler; + host.crosstoolTop = crosstoolTop; + host.glibc = glibc; + } else { + host.crosstoolTop = hostCrosstoolTop; + } + } + + if (hostLibcTop != null) { + host.libcTop = hostLibcTop; + } else if (hostCrosstoolTop == null) { + // Track libc in the host configuration if no host crosstool is set. + host.libcTop = libcTop; + } + + // -g0 is the default, but allowMultiple options cannot have default values so we just pass + // -g0 first and let the user options override it. + host.coptList = ImmutableList.builder().add("-g0").addAll(hostCoptList).build(); + + host.useThinArchives = useThinArchives; + host.useStartEndLib = useStartEndLib; + host.extractInclusions = extractInclusions; + host.stripBinaries = StripMode.ALWAYS; + host.fdoOptimize = null; + host.lipoMode = LipoMode.OFF; + host.scanIncludes = scanIncludes; + host.inmemoryDotdFiles = inmemoryDotdFiles; + host.cppModuleMaps = cppModuleMaps; + + return host; + } + + @Override + public void addAllLabels(Multimap labelMap) { + labelMap.put("crosstool", crosstoolTop); + if (hostCrosstoolTop != null) { + labelMap.put("crosstool", hostCrosstoolTop); + } + + if (libcTop != null) { + Label libcLabel = libcTop.getLabel(); + if (libcLabel != null) { + labelMap.put("crosstool", libcLabel); + } + } + addOptionalLabel(labelMap, "fdo", fdoOptimize); + + if (stl != null) { + labelMap.put("STL", stl); + } + + if (customMalloc != null) { + labelMap.put("custom_malloc", customMalloc); + } + + if (getLipoContextLabel() != null) { + labelMap.put("lipo", getLipoContextLabel()); + } + } + + @Override + public Map> getDefaultsLabels(BuildConfiguration.Options commonOptions) { + Set

Contains two {@link Runfiles} objects: one for the eventual statically linked binary and + * one for the one that uses shared libraries. Data dependencies are present in both. + */ +@Immutable +public final class CppRunfilesProvider implements TransitiveInfoProvider { + private final Runfiles staticRunfiles; + private final Runfiles sharedRunfiles; + + public CppRunfilesProvider(Runfiles staticRunfiles, Runfiles sharedRunfiles) { + this.staticRunfiles = staticRunfiles; + this.sharedRunfiles = sharedRunfiles; + } + + public Runfiles getStaticRunfiles() { + return staticRunfiles; + } + + public Runfiles getSharedRunfiles() { + return sharedRunfiles; + } + + /** + * Returns a function that gets the static C++ runfiles from a {@link TransitiveInfoCollection} + * or the empty runfiles instance if it does not contain that provider. + */ + public static final Function STATIC_RUNFILES = + new Function() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getStaticRunfiles(); + } + }; + + /** + * Returns a function that gets the shared C++ runfiles from a {@link TransitiveInfoCollection} + * or the empty runfiles instance if it does not contain that provider. + */ + public static final Function SHARED_RUNFILES = + new Function() { + @Override + public Runfiles apply(TransitiveInfoCollection input) { + CppRunfilesProvider provider = input.getProvider(CppRunfilesProvider.class); + return provider == null + ? Runfiles.EMPTY + : provider.getSharedRunfiles(); + } + }; + + /** + * Returns a function that gets the C++ runfiles from a {@link TransitiveInfoCollection} or + * the empty runfiles instance if it does not contain that provider. + */ + public static final Function runfilesFunction( + boolean linkingStatically) { + return linkingStatically ? STATIC_RUNFILES : SHARED_RUNFILES; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java new file mode 100644 index 0000000000..600b2fa342 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppSemantics.java @@ -0,0 +1,49 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.vfs.PathFragment; + +/** + * Pluggable C++ compilation semantics. + */ +public interface CppSemantics { + /** + * Returns the "effective source path" of a source file. + * + *

It is used, among other things, for computing the output path. + */ + PathFragment getEffectiveSourcePath(Artifact source); + + /** + * Called before a C++ compile action is built. + * + *

Gives the semantics implementation the opportunity to change compile actions at the last + * minute. + */ + void finalizeCompileActionBuilder( + RuleContext ruleContext, CppCompileActionBuilder actionBuilder); + + /** + * Called before {@link CppCompilationContext}s are finalized. + * + *

Gives the semantics implementation the opportunity to change what the C++ rule propagates + * to dependent rules. + */ + void setupCompilationContext( + RuleContext ruleContext, CppCompilationContext.Builder contextBuilder); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java new file mode 100644 index 0000000000..1111189101 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationIdentifier.java @@ -0,0 +1,132 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.CToolchain; + +import java.util.Objects; + +/** + * Contains parameters which uniquely describe a crosstool configuration + * and methods for comparing two crosstools against each other. + * + *

Two crosstools which contain equivalent values of these parameters are + * considered equal. + */ +public final class CrosstoolConfigurationIdentifier implements CrosstoolConfigurationOptions { + + /** The CPU associated with this crosstool configuration. */ + private final String cpu; + + /** The compiler (e.g. gcc) associated with this crosstool configuration. */ + private final String compiler; + + /** The version of libc (e.g. glibc-2.11) associated with this crosstool configuration. */ + private final String libc; + + private CrosstoolConfigurationIdentifier(String cpu, String compiler, String libc) { + this.cpu = cpu; + this.compiler = compiler; + this.libc = libc; + } + + /** + * Creates a new crosstool configuration from the given crosstool release and + * configuration options. + */ + public static CrosstoolConfigurationIdentifier fromReleaseAndCrosstoolConfiguration( + CrosstoolConfig.CrosstoolRelease release, BuildOptions buildOptions) { + String cpu = buildOptions.get(BuildConfiguration.Options.class).getCpu(); + if (cpu == null) { + cpu = release.getDefaultTargetCpu(); + } + CppOptions cppOptions = buildOptions.get(CppOptions.class); + return new CrosstoolConfigurationIdentifier(cpu, cppOptions.cppCompiler, cppOptions.glibc); + } + + public static CrosstoolConfigurationIdentifier fromToolchain(CToolchain toolchain) { + return new CrosstoolConfigurationIdentifier( + toolchain.getTargetCpu(), toolchain.getCompiler(), toolchain.getTargetLibc()); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof CrosstoolConfigurationIdentifier)) { + return false; + } + CrosstoolConfigurationIdentifier otherCrosstool = (CrosstoolConfigurationIdentifier) other; + return Objects.equals(cpu, otherCrosstool.cpu) + && Objects.equals(compiler, otherCrosstool.compiler) + && Objects.equals(libc, otherCrosstool.libc); + } + + @Override + public int hashCode() { + return Objects.hash(cpu, compiler, libc); + } + + + /** + * Returns a series of command line flags which specify the configuration options. + * Any of these options may be null, in which case its flag is omitted. + * + *

The appended string will be along the lines of + * " --cpu='cpu' --compiler='compiler' --glibc='libc'". + */ + public String describeFlags() { + StringBuilder message = new StringBuilder(); + if (getCpu() != null) { + message.append(" --cpu='").append(getCpu()).append("'"); + } + if (getCompiler() != null) { + message.append(" --compiler='").append(getCompiler()).append("'"); + } + if (getLibc() != null) { + message.append(" --glibc='").append(getLibc()).append("'"); + } + return message.toString(); + } + + /** Returns true if the specified toolchain is a candidate for use with this crosstool. */ + public boolean isCandidateToolchain(CToolchain toolchain) { + return (toolchain.getTargetCpu().equals(getCpu()) + && (getLibc() == null || toolchain.getTargetLibc().equals(getLibc())) + && (getCompiler() == null || toolchain.getCompiler().equals( + getCompiler()))); + } + + @Override + public String toString() { + return describeFlags(); + } + + @Override + public String getCpu() { + return cpu; + } + + @Override + public String getCompiler() { + return compiler; + } + + @Override + public String getLibc() { + return libc; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java new file mode 100644 index 0000000000..a113f5f126 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java @@ -0,0 +1,327 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.io.BaseEncoding; +import com.google.devtools.build.lib.analysis.RedirectChaser; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.analysis.config.ConfigurationEnvironment; +import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.packages.Package; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.syntax.Label.SyntaxException; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig; +import com.google.protobuf.TextFormat; +import com.google.protobuf.TextFormat.ParseException; +import com.google.protobuf.UninitializedMessageException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.ExecutionException; + +import javax.annotation.Nullable; + +/** + * A loader that reads Crosstool configuration files and creates CToolchain + * instances from them. + */ +public class CrosstoolConfigurationLoader { + private static final String CROSSTOOL_CONFIGURATION_FILENAME = "CROSSTOOL"; + + /** + * Cache for storing result of toReleaseConfiguration function based on path and md5 sum of + * input file. We can use md5 because result of this function depends only on the file content. + */ + private static final LoadingCache, CrosstoolConfig.CrosstoolRelease> + crosstoolReleaseCache = CacheBuilder.newBuilder().concurrencyLevel(4).maximumSize(100).build( + new CacheLoader, CrosstoolConfig.CrosstoolRelease>() { + @Override + public CrosstoolConfig.CrosstoolRelease load(Pair key) throws IOException { + char[] data = FileSystemUtils.readContentAsLatin1(key.first); + return toReleaseConfiguration(key.first.getPathString(), new String(data)); + } + }); + + /** + * A class that holds the results of reading a CROSSTOOL file. + */ + public static class CrosstoolFile { + private final Label crosstoolTop; + private Path crosstoolPath; + private CrosstoolConfig.CrosstoolRelease crosstool; + private String md5; + + CrosstoolFile(Label crosstoolTop) { + this.crosstoolTop = crosstoolTop; + } + + void setCrosstoolPath(Path crosstoolPath) { + this.crosstoolPath = crosstoolPath; + } + + void setCrosstool(CrosstoolConfig.CrosstoolRelease crosstool) { + this.crosstool = crosstool; + } + + void setMd5(String md5) { + this.md5 = md5; + } + + /** + * Returns the crosstool top as resolved. + */ + public Label getCrosstoolTop() { + return crosstoolTop; + } + + /** + * Returns the absolute path from which the CROSSTOOL file was read. + */ + public Path getCrosstoolPath() { + return crosstoolPath; + } + + /** + * Returns the parsed contents of the CROSSTOOL file. + */ + public CrosstoolConfig.CrosstoolRelease getProto() { + return crosstool; + } + + /** + * Returns an MD5 hash of the CROSSTOOL file contents. + */ + public String getMd5() { + return md5; + } + } + + private CrosstoolConfigurationLoader() { + } + + /** + * Reads the given data String, which must be in ascii format, + * into a protocol buffer. It uses the name parameter for error + * messages. + * + * @throws IOException if the parsing failed + */ + @VisibleForTesting + static CrosstoolConfig.CrosstoolRelease toReleaseConfiguration(String name, String data) + throws IOException { + CrosstoolConfig.CrosstoolRelease.Builder builder = + CrosstoolConfig.CrosstoolRelease.newBuilder(); + try { + TextFormat.merge(data, builder); + return builder.build(); + } catch (ParseException e) { + throw new IOException("Could not read the crosstool configuration file '" + name + "', " + + "because of a parser error (" + e.getMessage() + ")"); + } catch (UninitializedMessageException e) { + throw new IOException("Could not read the crosstool configuration file '" + name + "', " + + "because of an incomplete protocol buffer (" + e.getMessage() + ")"); + } + } + + private static boolean findCrosstoolConfiguration( + ConfigurationEnvironment env, + CrosstoolConfigurationLoader.CrosstoolFile file) + throws IOException, InvalidConfigurationException { + Label crosstoolTop = file.getCrosstoolTop(); + Path path = null; + try { + Package containingPackage = env.getTarget(crosstoolTop.getLocalTargetLabel("BUILD")) + .getPackage(); + if (containingPackage == null) { + return false; + } + path = env.getPath(containingPackage, CROSSTOOL_CONFIGURATION_FILENAME); + } catch (SyntaxException e) { + throw new InvalidConfigurationException(e); + } catch (NoSuchThingException e) { + // Handled later + } + + // If we can't find a file, fall back to the provided alternative. + if (path == null || !path.exists()) { + throw new InvalidConfigurationException("The crosstool_top you specified was resolved to '" + + crosstoolTop + "', which does not contain a CROSSTOOL file. " + + "You can use a crosstool from the depot by specifying its label."); + } else { + // Do this before we read the data, so if it changes, we get a different MD5 the next time. + // Alternatively, we could calculate the MD5 of the contents, which we also read, but this + // is faster if the file comes from a file system with md5 support. + file.setCrosstoolPath(path); + String md5 = BaseEncoding.base16().lowerCase().encode(path.getMD5Digest()); + CrosstoolConfig.CrosstoolRelease release; + try { + release = crosstoolReleaseCache.get(new Pair(path, md5)); + file.setCrosstool(release); + file.setMd5(md5); + } catch (ExecutionException e) { + throw new InvalidConfigurationException(e); + } + } + return true; + } + + /** + * Reads a crosstool file. + */ + @Nullable + public static CrosstoolConfigurationLoader.CrosstoolFile readCrosstool( + ConfigurationEnvironment env, Label crosstoolTop) throws InvalidConfigurationException { + crosstoolTop = RedirectChaser.followRedirects(env, crosstoolTop, "crosstool_top"); + if (crosstoolTop == null) { + return null; + } + CrosstoolConfigurationLoader.CrosstoolFile file = + new CrosstoolConfigurationLoader.CrosstoolFile(crosstoolTop); + try { + boolean allDependenciesPresent = findCrosstoolConfiguration(env, file); + return allDependenciesPresent ? file : null; + } catch (IOException e) { + throw new InvalidConfigurationException(e); + } + } + + /** + * Selects a crosstool toolchain corresponding to the given crosstool + * configuration options. If all of these options are null, it returns the default + * toolchain specified in the crosstool release. If only cpu is non-null, it + * returns the default toolchain for that cpu, as specified in the crosstool + * release. Otherwise, all values must be non-null, and this method + * returns the toolchain which matches all of the values. + * + * @throws NullPointerException if {@code release} is null + * @throws InvalidConfigurationException if no matching toolchain can be found, or + * if the input parameters do not obey the constraints described above + */ + public static CrosstoolConfig.CToolchain selectToolchain( + CrosstoolConfig.CrosstoolRelease release, BuildOptions options, + Function cpuTransformer) + throws InvalidConfigurationException { + CrosstoolConfigurationIdentifier config = + CrosstoolConfigurationIdentifier.fromReleaseAndCrosstoolConfiguration(release, options); + if ((config.getCompiler() != null) || (config.getLibc() != null)) { + ArrayList candidateToolchains = new ArrayList<>(); + for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) { + if (config.isCandidateToolchain(toolchain)) { + candidateToolchains.add(toolchain); + } + } + switch (candidateToolchains.size()) { + case 0: { + StringBuilder message = new StringBuilder(); + message.append("No toolchain found for"); + message.append(config.describeFlags()); + message.append(". Valid toolchains are: "); + describeToolchainList(message, release.getToolchainList()); + throw new InvalidConfigurationException(message.toString()); + } + case 1: + return candidateToolchains.get(0); + default: { + StringBuilder message = new StringBuilder(); + message.append("Multiple toolchains found for"); + message.append(config.describeFlags()); + message.append(": "); + describeToolchainList(message, candidateToolchains); + throw new InvalidConfigurationException(message.toString()); + } + } + } + String selectedIdentifier = null; + // We use fake CPU values to allow cross-platform builds for other languages that use the + // C++ toolchain. Translate to the actual target architecture. + String desiredCpu = cpuTransformer.apply(config.getCpu()); + for (CrosstoolConfig.DefaultCpuToolchain selector : release.getDefaultToolchainList()) { + if (selector.getCpu().equals(desiredCpu)) { + selectedIdentifier = selector.getToolchainIdentifier(); + break; + } + } + checkToolChain(selectedIdentifier, desiredCpu); + for (CrosstoolConfig.CToolchain toolchain : release.getToolchainList()) { + if (toolchain.getToolchainIdentifier().equals(selectedIdentifier)) { + return toolchain; + } + } + throw new InvalidConfigurationException("Inconsistent crosstool configuration; no toolchain " + + "corresponding to '" + selectedIdentifier + "' found for cpu '" + config.getCpu() + "'"); + } + + private static String describeToolchainFlags(CrosstoolConfig.CToolchain toolchain) { + return CrosstoolConfigurationIdentifier.fromToolchain(toolchain).describeFlags(); + } + + /** + * Appends a series of toolchain descriptions (as the blaze command line flags + * that would specify that toolchain) to 'message'. + */ + private static void describeToolchainList(StringBuilder message, + Collection toolchains) { + message.append("["); + for (CrosstoolConfig.CToolchain toolchain : toolchains) { + message.append(describeToolchainFlags(toolchain)); + message.append(","); + } + message.append("]"); + } + + /** + * Makes sure that {@code selectedIdentifier} is a valid identifier for a toolchain, + * i.e. it starts with a letter or an underscore and continues with only dots, dashes, + * spaces, letters, digits or underscores (i.e. matches the following regular expression: + * "[a-zA-Z_][\.\- \w]*"). + * + * @throws InvalidConfigurationException if selectedIdentifier is null or does not match the + * aforementioned regular expression. + */ + private static void checkToolChain(String selectedIdentifier, String cpu) + throws InvalidConfigurationException { + if (selectedIdentifier == null) { + throw new InvalidConfigurationException("No toolchain found for cpu '" + cpu + "'"); + } + // If you update this regex, please do so in the javadoc comment too, and also in the + // crosstool_config.proto file. + String rx = "[a-zA-Z_][\\.\\- \\w]*"; + if (!selectedIdentifier.matches(rx)) { + throw new InvalidConfigurationException("Toolchain identifier for cpu '" + cpu + "' " + + "is illegal (does not match '" + rx + "')"); + } + } + + public static CrosstoolConfig.CrosstoolRelease getCrosstoolReleaseProto( + ConfigurationEnvironment env, BuildOptions options, + Label crosstoolTop, Function cpuTransformer) + throws InvalidConfigurationException { + CrosstoolConfigurationLoader.CrosstoolFile file = + readCrosstool(env, crosstoolTop); + // Make sure that we have the requested toolchain in the result. Throw an exception if not. + selectToolchain(file.getProto(), options, cpuTransformer); + return file.getProto(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java new file mode 100644 index 0000000000..e311ab6163 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationOptions.java @@ -0,0 +1,29 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +/** + * A container object which provides crosstool configuration options to the build. + */ +public interface CrosstoolConfigurationOptions { + /** Returns the CPU associated with this crosstool configuration. */ + public String getCpu(); + + /** Returns the compiler associated with this crosstool configuration. */ + public String getCompiler(); + + /** Returns the libc version associated with this crosstool configuration. */ + public String getLibc(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java new file mode 100644 index 0000000000..a446125e5c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DiscoveredSourceInputsHelper.java @@ -0,0 +1,139 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Helper for actions that do include scanning. Currently only deals with source files, so is only + * appropriate for actions that do not discover generated files. Currently does not do .d file + * parsing, so the set of artifacts returned may be an overapproximation to the ones actually used + * during execution. + */ +public class DiscoveredSourceInputsHelper { + + private DiscoveredSourceInputsHelper() { + } + + /** + * Converts PathFragments into source Artifacts using an ArtifactResolver, ignoring any that are + * already in mandatoryInputs. Silently drops any PathFragments that cannot be resolved into + * Artifacts. + */ + public static ImmutableList getDiscoveredInputsFromPaths( + Iterable mandatoryInputs, ArtifactResolver artifactResolver, + Collection inputPaths) { + Set knownPathFragments = new HashSet<>(); + for (Artifact input : mandatoryInputs) { + knownPathFragments.add(input.getExecPath()); + } + ImmutableList.Builder foundInputs = ImmutableList.builder(); + for (PathFragment execPath : inputPaths) { + if (!knownPathFragments.add(execPath)) { + // Don't add any inputs that we already added, or original inputs, which we probably + // couldn't convert into artifacts anyway. + continue; + } + Artifact artifact = artifactResolver.resolveSourceArtifact(execPath); + // It is unlikely that this artifact is null, but tolerate the situation just in case. + // It is safe to ignore such paths because dependency checker would identify change in inputs + // (ignored path was used before) and will force action execution. + if (artifact != null) { + foundInputs.add(artifact); + } + } + return foundInputs.build(); + } + + /** + * Converts ActionInputs discovered as inputs during execution into source Artifacts, ignoring any + * that are already in mandatoryInputs or that live in builtInIncludeDirectories. If any + * ActionInputs cannot be resolved, an ActionExecutionException will be thrown. + * + *

This method duplicates the functionality of CppCompileAction#populateActionInputs, though it + * is simpler because it need not deal with derived artifacts and doesn't parse the .d file. + */ + public static ImmutableList getDiscoveredInputsFromActionInputs( + Iterable mandatoryInputs, + ArtifactResolver artifactResolver, + Iterable discoveredInputs, + Iterable builtInIncludeDirectories, + Action action, + Artifact primaryInput) throws ActionExecutionException { + List systemIncludePrefixes = new ArrayList<>(); + for (PathFragment includePath : builtInIncludeDirectories) { + if (includePath.isAbsolute()) { + systemIncludePrefixes.add(includePath); + } + } + + // Avoid duplicates by keeping track of the ones we've seen so far, even though duplicates are + // unlikely, since they would have to be inputs to this (non-CppCompile) action and also + // #included by a C++ source file. + Set knownInputs = new HashSet<>(); + Iterables.addAll(knownInputs, mandatoryInputs); + ImmutableList.Builder foundInputs = ImmutableList.builder(); + // Check inclusions. + IncludeProblems problems = new IncludeProblems(); + for (ActionInput input : discoveredInputs) { + if (input instanceof Artifact) { + Artifact artifact = (Artifact) input; + if (knownInputs.add(artifact)) { + foundInputs.add(artifact); + } + continue; + } + PathFragment execPath = new PathFragment(input.getExecPathString()); + if (execPath.isAbsolute()) { + // Absolute includes from system paths are ignored. + if (FileSystemUtils.startsWithAny(execPath, systemIncludePrefixes)) { + continue; + } + // Theoretically, the more sophisticated logic of CppCompileAction#populateActioInputs could + // be used here, to allow absolute includes that started with the execRoot. However, since + // we don't hit this codepath for local execution, that should be unnecessary. If and when + // we examine the results of local execution for scanned includes, that case may need to be + // dealt with. + problems.add(execPath.getPathString()); + } + Artifact artifact = artifactResolver.resolveSourceArtifact(execPath); + if (artifact != null) { + if (knownInputs.add(artifact)) { + foundInputs.add(artifact); + } + } else { + // Abort if we see files that we can't resolve, likely caused by + // undeclared includes or illegal include constructs. + problems.add(execPath.getPathString()); + } + } + problems.assertProblemFree(action, primaryInput); + return foundInputs.build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java new file mode 100644 index 0000000000..142a67a98b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/DwoArtifactsCollector.java @@ -0,0 +1,120 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; + +/** + * Provides generic functionality for collecting the .dwo artifacts produced by any target + * that compiles C++ files. Supports both transitive and "only direct outputs" collection. + * Provides accessors for both PIC and non-PIC compilation modes. + */ +public class DwoArtifactsCollector { + + /** + * The .dwo files collected by this target in non-PIC compilation mode (i.e. myobject.dwo). + */ + private final NestedSet dwoArtifacts; + + /** + * The .dwo files collected by this target in PIC compilation mode (i.e. myobject.pic.dwo). + */ + private final NestedSet picDwoArtifacts; + + /** + * Instantiates a "real" collector on meaningful data. + */ + private DwoArtifactsCollector(CcCompilationOutputs compilationOutputs, + Iterable deps) { + + Preconditions.checkNotNull(compilationOutputs); + Preconditions.checkNotNull(deps); + + // Note: .dwo collection works fine with any order, but tests may assume a + // specific order for readability / simplicity purposes. See + // DebugInfoPackagingTest for details. + NestedSetBuilder dwoBuilder = NestedSetBuilder.compileOrder(); + NestedSetBuilder picDwoBuilder = NestedSetBuilder.compileOrder(); + + dwoBuilder.addAll(compilationOutputs.getDwoFiles()); + picDwoBuilder.addAll(compilationOutputs.getPicDwoFiles()); + + for (TransitiveInfoCollection info : deps) { + CppDebugFileProvider provider = info.getProvider(CppDebugFileProvider.class); + if (provider != null) { + dwoBuilder.addTransitive(provider.getTransitiveDwoFiles()); + picDwoBuilder.addTransitive(provider.getTransitivePicDwoFiles()); + } + } + + dwoArtifacts = dwoBuilder.build(); + picDwoArtifacts = picDwoBuilder.build(); + } + + /** + * Instantiates an empty collector. + */ + private DwoArtifactsCollector() { + dwoArtifacts = NestedSetBuilder.emptySet(Order.COMPILE_ORDER); + picDwoArtifacts = NestedSetBuilder.emptySet(Order.COMPILE_ORDER); + } + + /** + * Returns a new instance that collects direct outputs and transitive dependencies. + * + * @param compilationOutputs the output compilation context for the owning target + * @param deps which of the target's transitive info collections should be visited + */ + public static DwoArtifactsCollector transitiveCollector(CcCompilationOutputs compilationOutputs, + Iterable deps) { + return new DwoArtifactsCollector(compilationOutputs, deps); + } + + /** + * Returns a new instance that collects direct outputs only. + * + * @param compilationOutputs the output compilation context for the owning target + */ + public static DwoArtifactsCollector directCollector(CcCompilationOutputs compilationOutputs) { + return new DwoArtifactsCollector( + compilationOutputs, ImmutableList.of()); + } + + /** + * Returns a new instance that doesn't collect anything (its artifact sets are empty). + */ + public static DwoArtifactsCollector emptyCollector() { + return new DwoArtifactsCollector(); + } + + /** + * Returns the .dwo files applicable to non-PIC compilation mode (i.e. myobject.dwo). + */ + public NestedSet getDwoArtifacts() { + return dwoArtifacts; + } + + /** + * Returns the .dwo files applicable to PIC compilation mode (i.e. myobject.pic.dwo). + */ + public NestedSet getPicDwoArtifacts() { + return picDwoArtifacts; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java new file mode 100644 index 0000000000..15d7010f7c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ExtractInclusionAction.java @@ -0,0 +1,85 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.io.IOException; + +/** + * An action which greps for includes over a given .cc or .h file. + * This is a part of the work required for C++ include scanning. + * + *

Note that this may run grep-includes over-optimistically, where we previously + * had not. For example, consider a cc_library of generated headers. If another + * library depends on it, and only references one of the headers, the other + * grep-includes will have been wasted. + */ +final class ExtractInclusionAction extends AbstractAction { + + private static final String GUID = "45b43e5a-4734-43bb-a05e-012313808142"; + + /** + * Constructs a new action. + */ + public ExtractInclusionAction(ActionOwner owner, Artifact input, Artifact output) { + super(owner, ImmutableList.of(input), ImmutableList.of(output)); + } + + @Override + protected String computeKey() { + return GUID; + } + + @Override + public String describeStrategy(Executor executor) { + return executor.getContext(CppCompileActionContext.class).strategyLocality(); + } + + @Override + public String getMnemonic() { + return "GrepIncludes"; + } + + @Override + protected String getRawProgressMessage() { + return "Extracting include lines from " + getPrimaryInput().prettyPrint(); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + IncludeScanningContext context = executor.getContext(IncludeScanningContext.class); + try { + context.extractIncludes(actionExecutionContext, this, getPrimaryInput(), + getPrimaryOutput()); + } catch (IOException e) { + throw new ActionExecutionException(e, this, false); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java new file mode 100644 index 0000000000..bd15455491 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FakeCppCompileAction.java @@ -0,0 +1,212 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.rules.cpp.CcToolchainFeatures.FeatureConfiguration; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.UUID; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * Action that represents a fake C++ compilation step. + */ +@ThreadCompatible +public class FakeCppCompileAction extends CppCompileAction { + + private static final Logger LOG = Logger.getLogger(FakeCppCompileAction.class.getName()); + + public static final UUID GUID = UUID.fromString("b2d95c91-1434-47ae-a786-816017de8494"); + + private final PathFragment tempOutputFile; + + FakeCppCompileAction(ActionOwner owner, + ImmutableList features, + FeatureConfiguration featureConfiguration, + Artifact sourceFile, + Label sourceLabel, + NestedSet mandatoryInputs, + Artifact outputFile, + PathFragment tempOutputFile, + DotdFile dotdFile, + BuildConfiguration configuration, + CppConfiguration cppConfiguration, + CppCompilationContext context, + ImmutableList copts, + ImmutableList pluginOpts, + Predicate nocopts, + ImmutableList extraSystemIncludePrefixes, + boolean enableLayeringCheck, + @Nullable String fdoBuildStamp) { + super(owner, features, featureConfiguration, sourceFile, sourceLabel, mandatoryInputs, + outputFile, dotdFile, null, null, null, + configuration, cppConfiguration, + // We only allow inclusion of header files explicitly declared in + // "srcs", so we only use declaredIncludeSrcs, not declaredIncludeDirs. + // (Disallowing use of undeclared headers for cc_fake_binary is needed + // because the header files get included in the runfiles for the + // cc_fake_binary and for the negative compilation tests that depend on + // the cc_fake_binary, and the runfiles must be determined at analysis + // time, so they can't depend on the contents of the ".d" file.) + CppCompilationContext.disallowUndeclaredHeaders(context), null, copts, pluginOpts, nocopts, + extraSystemIncludePrefixes, enableLayeringCheck, fdoBuildStamp, VOID_INCLUDE_RESOLVER, + ImmutableList.of(), + GUID, /*compileHeaderModules=*/false); + this.tempOutputFile = Preconditions.checkNotNull(tempOutputFile); + } + + @Override + @ThreadCompatible + public void execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + + // First, do an normal compilation, to generate the ".d" file. The generated + // object file is built to a temporary location (tempOutputFile) and ignored + // afterwards. + LOG.info("Generating " + getDotdFile()); + CppCompileActionContext context = executor.getContext(CppCompileActionContext.class); + CppCompileActionContext.Reply reply = null; + try { + // We delegate stdout/stderr to nowhere, i.e. same as redirecting to /dev/null. + reply = context.execWithReply( + this, actionExecutionContext.withFileOutErr(new FileOutErr())); + } catch (ExecException e) { + // We ignore failures here (other than capturing the Distributor reply). + // The compilation may well fail (that's the whole point of negative compilation tests). + // We execute it here just for the side effect of generating the ".d" file. + reply = context.getReplyFromException(e, this); + if (reply == null) { + // This can only happen if the ExecException does not come from remote execution. + throw e.toActionExecutionException("", executor.getVerboseFailures(), this); + } + } + IncludeScanningContext scanningContext = executor.getContext(IncludeScanningContext.class); + updateActionInputs(executor.getExecRoot(), scanningContext.getArtifactResolver(), reply); + + // Even cc_fake_binary rules need to properly declare their dependencies... + // In fact, they need to declare their dependencies even more than cc_binary rules do. + // CcCommonConfiguredTarget passes in an empty set of declaredIncludeDirs, + // so this check below will only allow inclusion of header files that are explicitly + // listed in the "srcs" of the cc_fake_binary or in the "srcs" of a cc_library that it + // depends on. + try { + validateInclusions(actionExecutionContext.getMiddlemanExpander(), executor.getEventHandler()); + } catch (ActionExecutionException e) { + // TODO(bazel-team): (2009) make this into an error, once most of the current warnings + // are fixed. + executor.getEventHandler().handle(Event.warn( + getOwner().getLocation(), + e.getMessage() + ";\n this warning may eventually become an error")); + } + + // Generate a fake ".o" file containing the command line needed to generate + // the real object file. + LOG.info("Generating " + outputFile); + + // A cc_fake_binary rule generates fake .o files and a fake target file, + // which merely contain instructions on building the real target. We need to + // be careful to use a new set of output file names in the instructions, as + // to not overwrite the fake output files when someone tries to follow the + // instructions. As the real compilation is executed by the test from its + // runfiles directory (where writing is forbidden), we patch the command + // line to write to $TEST_TMPDIR instead. + final String outputPrefix = "$TEST_TMPDIR/"; + String argv = Joiner.on(' ').join( + Iterables.transform(getArgv(outputFile.getExecPath()), new Function() { + @Override + public String apply(String input) { + String result = ShellEscaper.escapeString(input); + if (input.equals(outputFile.getExecPathString()) + || input.equals(getDotdFile().getSafeExecPath().getPathString())) { + result = outputPrefix + result; + } + return result; + } + })); + + // Write the command needed to build the real .o file to the fake .o file. + // Generate a command to ensure that the output directory exists; otherwise + // the compilation would fail. + try { + // Ensure that the .d file and .o file are siblings, so that the "mkdir" below works for + // both. + Preconditions.checkState(outputFile.getExecPath().getParentDirectory().equals( + getDotdFile().getSafeExecPath().getParentDirectory())); + FileSystemUtils.writeContent(outputFile.getPath(), ISO_8859_1, + outputFile.getPath().getBaseName() + ": " + + "mkdir -p " + outputPrefix + "$(dirname " + outputFile.getExecPath() + ")" + + " && " + argv + "\n"); + } catch (IOException e) { + throw new ActionExecutionException("failed to create fake compile command for rule '" + + getOwner().getLabel() + ": " + e.getMessage(), + this, false); + } + } + + @Override + protected PathFragment getInternalOutputFile() { + return tempOutputFile; + } + + @Override + public String getMnemonic() { return "FakeCppCompile"; } + + @Override + public String describeStrategy(Executor executor) { + return "fake"; + } + + @Override + public ResourceSet estimateResourceConsumptionLocal() { + return new ResourceSet(/*memoryMb=*/1, /*cpuUsage=*/0.1, /*ioUsage=*/0.0); + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return executor.getContext(CppCompileActionContext.class).estimateResourceConsumption(this); + } + + @Override + protected boolean needsIncludeScanning(Executor executor) { + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java new file mode 100644 index 0000000000..f50a1ae5c2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoStubAction.java @@ -0,0 +1,70 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.vfs.Path; + +/** + * Stub action to be used as the generating action for FDO files that are extracted from the + * FDO zip. + * + *

This is needed because the extraction is currently not a bona fide action, therefore, Blaze + * would complain that these files have no generating action if we did not set it to an instance of + * this class. + */ +public class FdoStubAction extends AbstractAction { + public FdoStubAction(ActionOwner owner, Artifact output) { + // TODO(bazel-team): Make extracting the zip file a honest-to-God action so that we can do away + // with this ugliness. + super(owner, ImmutableList.of(), ImmutableList.of(output)); + } + + @Override + public String describeStrategy(Executor executor) { + return ""; + } + + @Override + public void execute(ActionExecutionContext actionExecutionContext) { + } + + @Override + public String getMnemonic() { + return "FdoStubAction"; + } + + @Override + protected String computeKey() { + return "fdoStubAction"; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return ResourceSet.ZERO; + } + + @Override + public void prepare(Path execRoot) { + // The superclass would delete the output files here. We can't let that happen, since this + // action does not in fact create those files; it is only a placeholder and the actual files + // are created *before* the execution phase in FdoSupport.extractFdoZip() + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java new file mode 100644 index 0000000000..911d888952 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java @@ -0,0 +1,679 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactFactory; +import com.google.devtools.build.lib.actions.PackageRootResolver; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.AnalysisEnvironment; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadHostile; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.skyframe.FileValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.lib.vfs.ZipFileSystem; +import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.LipoMode; +import com.google.devtools.build.skyframe.SkyFunction; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; +import java.util.zip.ZipException; + +/** + * Support class for FDO (feedback directed optimization) and LIPO (lightweight inter-procedural + * optimization). + * + *

There is a 1:1 relationship between {@link CppConfiguration} objects and {@code FdoSupport} + * objects. The FDO support of a build configuration can be retrieved using {@link + * CppConfiguration#getFdoSupport()}. + * + *

With respect to thread-safety, the {@link #prepareToBuild} method is not thread-safe, and must + * not be called concurrently with other methods on this class. + * + *

Here follows a quick run-down of how FDO/LIPO builds work (for non-FDO/LIPO builds, none + * of this applies): + * + *

{@link CppConfiguration#prepareHook} is called before the analysis phase, which calls + * {@link #prepareToBuild}, which extracts the FDO .zip (in case we work with an explicitly + * generated FDO profile file) or analyzes the .afdo.imports file next to the .afdo file (if + * AutoFDO is in effect). + * + *

.afdo.imports files contain one import a line. A line is two paths separated by a colon, + * with functions in the second path being referenced by functions in the first path. These are + * then put into the imports map. If we do AutoFDO, we don't handle individual .gcda files, so + * gcdaFiles will be empty. + * + *

Regular .fdo zip files contain .gcda files (which are added to gcdaFiles) and + * .gcda.imports files. There is one .gcda.imports file for every source file and it contains one + * path in every line, which can either be a path to a source file that contains a function + * referenced by the original source file or the .gcda file for such a referenced file. They + * both are added to the imports map. + * + *

If we do LIPO, we create an extra configuration that is called the "LIPO context collector", + * whose job it is to collect information that every configured target compiled with LIPO needs. + * The top-level target of this configuration is the LIPO context (always a cc_binary) and is an + * implicit dependency of every cc_* rule through their :lipo_context_collector attribute. The + * collected information is encapsulated in {@link LipoContextProvider}. + * + *

For each C++ compile action in the target configuration, {@link #configureCompilation} is + * called, which adds command line options and input files required for the build. There are + * three cases: + * + *

    + *
  • If we do AutoFDO, the .afdo file and the source files containing the functions imported + * by the original source file (as determined from the inputs map) are added. + *
  • If we do FDO, the .gcda file corresponding to the source file is added. + *
  • If we do LIPO, in addition to the .gcda file corresponding to the source file + * (like for FDO) the source files that contain the functions referenced by the source file and + * their .gcda files are added, too. + *
+ * + *

If we do LIPO, the actual C++ compilation context for LIPO compilation actions is pieced + * together from the CppCompileContext in LipoContextProvider and that of the rule being compiled. + * (see {@link CppCompilationContext#mergeForLipo}) This is so that the include files for the + * extra LIPO sources are found and is, strictly speaking, incorrect, since it also changes the + * declared include directories of the main source file, which in theory can result in the + * compilation passing even though it should fail with undeclared inclusion errors. + * + *

During the actual execution of the C++ compile action, the extra sources also need to be + * include scanned, which is the reason why they are {@link IncludeScannable} objects and not + * simple artifacts. We currently create these {@link IncludeScannable} objects by creating actual + * C++ compile actions in the LIPO context collector configuration which are then never executed. + * In fact, these C++ compile actions are never even registered with Skyframe. For this we + * propagate a bit from {@code BuildConfiguration.isActionsEnabled} to + * {@code CachingAnalysisEnvironment.allowRegisteringActions}, which causes actions to be silently + * discarded after configured targets are created. + */ +public class FdoSupport implements Serializable { + + /** + * Path within profile data .zip files that is considered the root of the + * profile information directory tree. + */ + private static final PathFragment ZIP_ROOT = new PathFragment("/"); + + /** + * Returns true if the give fdoFile represents an AutoFdo profile. + */ + public static final boolean isAutoFdo(String fdoFile) { + return CppFileTypes.GCC_AUTO_PROFILE.matches(fdoFile); + } + + /** + * Coverage information output directory passed to {@code --fdo_instrument}, + * or {@code null} if FDO instrumentation is disabled. + */ + private final PathFragment fdoInstrument; + + /** + * Path of the profile file passed to {@code --fdo_optimize}, or + * {@code null} if FDO optimization is disabled. The profile file + * can be a coverage ZIP or an AutoFDO feedback file. + */ + private final Path fdoProfile; + + /** + * Temporary directory to which the coverage ZIP file is extracted to + * (relative to the exec root), or {@code null} if FDO optimization is + * disabled. This is used to create artifacts for the extracted files. + * + *

Note that this root is intentionally not registered with the artifact + * factory. + */ + private final Root fdoRoot; + + /** + * The relative path of the FDO root to the exec root. + */ + private final PathFragment fdoRootExecPath; + + /** + * Path of FDO files under the FDO root. + */ + private final PathFragment fdoPath; + + /** + * LIPO mode passed to {@code --lipo}. This is only used if + * {@code fdoProfile != null}. + */ + private final LipoMode lipoMode; + + /** + * Flag indicating whether to use AutoFDO (as opposed to + * instrumentation-based FDO). + */ + private final boolean useAutoFdo; + + /** + * The {@code .gcda} files that have been extracted from the ZIP file, + * relative to the root of the ZIP file. + * + *

Set only in {@link #prepareToBuild}. + */ + private ImmutableSet gcdaFiles = ImmutableSet.of(); + + /** + * Multimap from .gcda file base names to auxiliary input files. + * + *

The keys of the multimap are the exec root relative paths of .gcda files + * with the extension removed. The values are the lines from the accompanying + * .gcda.imports file. + * + *

The contents of the multimap are copied verbatim from the .gcda.imports + * files and not yet checked for validity. + * + *

Set only in {@link #prepareToBuild}. + */ + private ImmutableMultimap imports; + + /** + * Creates an FDO support object. + * + * @param fdoInstrument value of the --fdo_instrument option + * @param fdoProfile path to the profile file passed to --fdo_optimize option + * @param lipoMode value of the --lipo_mode option + */ + public FdoSupport(PathFragment fdoInstrument, Path fdoProfile, LipoMode lipoMode, Path execRoot) { + this.fdoInstrument = fdoInstrument; + this.fdoProfile = fdoProfile; + this.fdoRoot = (fdoProfile == null) + ? null + : Root.asDerivedRoot(execRoot, execRoot.getRelative("blaze-fdo")); + this.fdoRootExecPath = fdoProfile == null + ? null + : fdoRoot.getExecPath().getRelative(new PathFragment("_fdo").getChild( + FileSystemUtils.removeExtension(fdoProfile.getBaseName()))); + this.fdoPath = fdoProfile == null + ? null + : new PathFragment("_fdo").getChild( + FileSystemUtils.removeExtension(fdoProfile.getBaseName())); + this.lipoMode = lipoMode; + this.useAutoFdo = fdoProfile != null && isAutoFdo(fdoProfile.getBaseName()); + } + + public Root getFdoRoot() { + return fdoRoot; + } + + public void declareSkyframeDependencies(SkyFunction.Environment env, Path execRoot) { + if (fdoProfile != null) { + if (isLipoEnabled()) { + // Incrementality is not supported for LIPO builds, see FdoSupport#scannables. + // Ensure that the Skyframe value containing the configuration will not be reused to avoid + // incrementality issues. + PrecomputedValue.dependOnBuildId(env); + return; + } + + // IMPORTANT: Keep the following in sync with #prepareToBuild. + Path path; + if (useAutoFdo) { + path = fdoProfile.getParentDirectory().getRelative( + fdoProfile.getBaseName() + ".imports"); + } else { + path = fdoProfile; + } + env.getValue(FileValue.key(RootedPath.toRootedPathMaybeUnderRoot(path, + ImmutableList.of(execRoot)))); + } + } + + /** + * Prepares the FDO support for building. + * + *

When an {@code --fdo_optimize} compile is requested, unpacks the given + * FDO gcda zip file into a clean working directory under execRoot. + * + * @throws FdoException if the FDO ZIP contains a file of unknown type + */ + @ThreadHostile // must be called before starting the build + public void prepareToBuild(Path execRoot, PathFragment genfilesPath, + ArtifactFactory artifactDeserializer, PackageRootResolver resolver) + throws IOException, FdoException { + // The execRoot != null case is only there for testing. We cannot provide a real ZIP file in + // tests because ZipFileSystem does not work with a ZIP on an in-memory file system. + // IMPORTANT: Keep in sync with #declareSkyframeDependencies to avoid incrementality issues. + if (fdoProfile != null && execRoot != null) { + Path fdoDirPath = execRoot.getRelative(fdoRootExecPath); + + FileSystemUtils.deleteTreesBelow(fdoDirPath); + FileSystemUtils.createDirectoryAndParents(fdoDirPath); + + if (useAutoFdo) { + Path fdoImports = fdoProfile.getParentDirectory().getRelative( + fdoProfile.getBaseName() + ".imports"); + if (isLipoEnabled()) { + imports = readAutoFdoImports(artifactDeserializer, fdoImports, genfilesPath, resolver); + } + FileSystemUtils.ensureSymbolicLink( + execRoot.getRelative(getAutoProfilePath()), fdoProfile); + } else { + Path zipFilePath = new ZipFileSystem(fdoProfile).getRootDirectory(); + if (!zipFilePath.getRelative("blaze-out").isDirectory()) { + throw new ZipException("FDO zip files must be zipped directly above 'blaze-out' " + + "for the compiler to find the profile"); + } + ImmutableSet.Builder gcdaFilesBuilder = ImmutableSet.builder(); + ImmutableMultimap.Builder importsBuilder = + ImmutableMultimap.builder(); + extractFdoZip(artifactDeserializer, zipFilePath, fdoDirPath, + gcdaFilesBuilder, importsBuilder, resolver); + gcdaFiles = gcdaFilesBuilder.build(); + imports = importsBuilder.build(); + } + } + } + + /** + * Recursively extracts a directory from the GCDA ZIP file into a target + * directory. + * + *

Imports files are not written to disk. Their content is directly added + * to an internal data structure. + * + *

The files are written at $EXECROOT/blaze-fdo/_fdo/(base name of profile zip), and the + * {@code _fdo} directory there is symlinked to from the exec root, so that the file are also + * available at $EXECROOT/_fdo/..., which is their exec path. We need to jump through these + * hoops because the FDO root 1. needs to be a source root, thus the exec path of its root is + * ".", 2. it must not be equal to the exec root so that the artifact factory does not get + * confused, 3. the files under it must be reachable by their exec path from the exec root. + * + * @throws IOException if any of the I/O operations failed + * @throws FdoException if the FDO ZIP contains a file of unknown type + */ + private void extractFdoZip(ArtifactFactory artifactFactory, Path sourceDir, + Path targetDir, ImmutableSet.Builder gcdaFilesBuilder, + ImmutableMultimap.Builder importsBuilder, + PackageRootResolver resolver) throws IOException, FdoException { + for (Path sourceFile : sourceDir.getDirectoryEntries()) { + Path targetFile = targetDir.getRelative(sourceFile.getBaseName()); + if (sourceFile.isDirectory()) { + targetFile.createDirectory(); + extractFdoZip(artifactFactory, sourceFile, targetFile, gcdaFilesBuilder, importsBuilder, + resolver); + } else { + if (CppFileTypes.COVERAGE_DATA.matches(sourceFile)) { + FileSystemUtils.copyFile(sourceFile, targetFile); + gcdaFilesBuilder.add( + sourceFile.relativeTo(sourceFile.getFileSystem().getRootDirectory())); + } else if (CppFileTypes.COVERAGE_DATA_IMPORTS.matches(sourceFile)) { + readCoverageImports(artifactFactory, sourceFile, importsBuilder, resolver); + } else { + throw new FdoException("FDO ZIP file contained a file of unknown type: " + + sourceFile); + } + } + } + } + + /** + * Reads a .gcda.imports file and stores the imports information. + * + * @throws FdoException if an auxiliary LIPO input was not found + */ + private void readCoverageImports(ArtifactFactory artifactFactory, Path importsFile, + ImmutableMultimap.Builder importsBuilder, + PackageRootResolver resolver) throws IOException, FdoException { + PathFragment key = importsFile.asFragment().relativeTo(ZIP_ROOT); + String baseName = key.getBaseName(); + String ext = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA_IMPORTS.getExtensions()); + key = key.replaceName(baseName.substring(0, baseName.length() - ext.length())); + + for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) { + if (!line.isEmpty()) { + // We can't yet fully check the validity of a line. this is done later + // when we actually parse the contained paths. + PathFragment execPath = new PathFragment(line); + if (execPath.isAbsolute()) { + throw new FdoException("Absolute paths not allowed in gcda imports file " + importsFile + + ": " + execPath); + } + Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(line), resolver); + if (artifact == null) { + throw new FdoException("Auxiliary LIPO input not found: " + line); + } + + importsBuilder.put(key, artifact); + } + } + } + + /** + * Reads a .afdo.imports file and stores the imports information. + */ + private ImmutableMultimap readAutoFdoImports( + ArtifactFactory artifactFactory, Path importsFile, PathFragment genFilePath, + PackageRootResolver resolver) + throws IOException, FdoException { + ImmutableMultimap.Builder importBuilder = ImmutableMultimap.builder(); + for (String line : FileSystemUtils.iterateLinesAsLatin1(importsFile)) { + if (!line.isEmpty()) { + PathFragment key = new PathFragment(line.substring(0, line.indexOf(':'))); + if (key.startsWith(genFilePath)) { + key = key.relativeTo(genFilePath); + } + if (key.isAbsolute()) { + throw new FdoException("Absolute paths not allowed in afdo imports file " + importsFile + + ": " + key); + } + key = FileSystemUtils.replaceSegments(key, "PROTECTED", "_protected", true); + for (String auxFile : line.substring(line.indexOf(':') + 1).split(" ")) { + if (auxFile.length() == 0) { + continue; + } + Artifact artifact = artifactFactory.deserializeArtifact(new PathFragment(auxFile), + resolver); + if (artifact == null) { + throw new FdoException("Auxiliary LIPO input not found: " + auxFile); + } + importBuilder.put(key, artifact); + } + } + } + return importBuilder.build(); + } + + /** + * Returns the imports from the .afdo.imports file of a source file. + * + * @param sourceName the source file + */ + private Collection getAutoFdoImports(PathFragment sourceName) { + Preconditions.checkState(isLipoEnabled()); + ImmutableCollection afdoImports = imports.get(sourceName); + Preconditions.checkState(afdoImports != null, + "AutoFDO import data missing for %s", sourceName); + return afdoImports; + } + + /** + * Returns the imports from the .gcda.imports file of an object file. + * + * @param objDirectory the object directory of the object file's target + * @param objectName the object file + */ + private Iterable getImports(PathFragment objDirectory, PathFragment objectName) { + Preconditions.checkState(isLipoEnabled()); + Preconditions.checkState(imports != null, + "Tried to look up imports of uninitialized FDOSupport"); + PathFragment key = objDirectory.getRelative(FileSystemUtils.removeExtension(objectName)); + ImmutableCollection importsForObject = imports.get(key); + Preconditions.checkState(importsForObject != null, "Import data missing for %s", key); + return importsForObject; + } + + /** + * Configures a compile action builder by adding command line options and + * auxiliary inputs according to the FDO configuration. This method does + * nothing If FDO is disabled. + */ + @ThreadSafe + public void configureCompilation(CppCompileActionBuilder builder, RuleContext ruleContext, + AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, final Pattern nocopts, + boolean usePic, LipoContextProvider lipoInputProvider) { + // It is a bug if this method is called with useLipo if lipo is disabled. However, it is legal + // if is is called with !useLipo, even though lipo is enabled. + Preconditions.checkArgument(lipoInputProvider == null || isLipoEnabled()); + + // FDO is disabled -> do nothing. + if ((fdoInstrument == null) && (fdoRoot == null)) { + return; + } + + List fdoCopts = new ArrayList<>(); + // Instrumentation phase + if (fdoInstrument != null) { + fdoCopts.add("-fprofile-generate=" + fdoInstrument.getPathString()); + if (lipoMode != LipoMode.OFF) { + fdoCopts.add("-fripa"); + } + } + + // Optimization phase + if (fdoRoot != null) { + // Declare dependency on contents of zip file. + if (env.getSkyframeEnv().valuesMissing()) { + return; + } + Iterable auxiliaryInputs = getAuxiliaryInputs( + ruleContext, env, lipoLabel, sourceName, usePic, lipoInputProvider); + builder.addMandatoryInputs(auxiliaryInputs); + if (!Iterables.isEmpty(auxiliaryInputs)) { + if (useAutoFdo) { + fdoCopts.add("-fauto-profile=" + getAutoProfilePath().getPathString()); + } else { + fdoCopts.add("-fprofile-use=" + fdoRootExecPath); + } + fdoCopts.add("-fprofile-correction"); + if (lipoInputProvider != null) { + fdoCopts.add("-fripa"); + } + } + } + Iterable filteredCopts = fdoCopts; + if (nocopts != null) { + // Filter fdoCopts with nocopts if they exist. + filteredCopts = Iterables.filter(fdoCopts, new Predicate() { + @Override + public boolean apply(String copt) { + return !nocopts.matcher(copt).matches(); + } + }); + } + builder.addCopts(0, filteredCopts); + } + + /** + * Returns the auxiliary files that need to be added to the {@link CppCompileAction}. + */ + private Iterable getAuxiliaryInputs( + RuleContext ruleContext, AnalysisEnvironment env, Label lipoLabel, PathFragment sourceName, + boolean usePic, LipoContextProvider lipoContextProvider) { + // If --fdo_optimize was not specified, we don't have any additional inputs. + if (fdoProfile == null) { + return ImmutableSet.of(); + } else if (useAutoFdo) { + ImmutableSet.Builder auxiliaryInputs = ImmutableSet.builder(); + + Artifact artifact = env.getDerivedArtifact( + fdoPath.getRelative(getAutoProfileRootRelativePath()), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + auxiliaryInputs.add(artifact); + if (lipoContextProvider != null) { + auxiliaryInputs.addAll(getAutoFdoImports(sourceName)); + } + return auxiliaryInputs.build(); + } else { + ImmutableSet.Builder auxiliaryInputs = ImmutableSet.builder(); + + PathFragment objectName = + FileSystemUtils.replaceExtension(sourceName, usePic ? ".pic.o" : ".o"); + + auxiliaryInputs.addAll( + getGcdaArtifactsForObjectFileName(ruleContext, env, objectName, lipoLabel)); + + if (lipoContextProvider != null) { + for (Artifact importedFile : getImports( + getNonLipoObjDir(ruleContext, lipoLabel), objectName)) { + if (CppFileTypes.COVERAGE_DATA.matches(importedFile.getFilename())) { + Artifact gcdaArtifact = getGcdaArtifactsForGcdaPath( + ruleContext, env, importedFile.getExecPath()); + if (gcdaArtifact == null) { + ruleContext.ruleError(String.format( + ".gcda file %s is not in the FDO zip (referenced by source file %s)", + importedFile.getExecPath(), sourceName)); + } else { + auxiliaryInputs.add(gcdaArtifact); + } + } else { + auxiliaryInputs.add(importedFile); + } + } + } + + return auxiliaryInputs.build(); + } + } + + /** + * Returns the .gcda file artifacts for a .gcda path from the .gcda.imports file or null if the + * referenced .gcda file is not in the FDO zip. + */ + private Artifact getGcdaArtifactsForGcdaPath(RuleContext ruleContext, + AnalysisEnvironment env, PathFragment gcdaPath) { + if (!gcdaFiles.contains(gcdaPath)) { + return null; + } + + Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaPath), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + return artifact; + } + + private PathFragment getNonLipoObjDir(RuleContext ruleContext, Label label) { + return ruleContext.getConfiguration().getBinFragment() + .getRelative(CppHelper.getObjDirectory(label)); + } + + /** + * Returns a list of .gcda file artifacts for an object file path. + * + *

The resulting set is either empty (because no .gcda file exists for the + * given object file) or contains one or two artifacts (the file itself and a + * symlink to it). + */ + private ImmutableList getGcdaArtifactsForObjectFileName(RuleContext ruleContext, + AnalysisEnvironment env, PathFragment objectFileName, Label lipoLabel) { + // We put the .gcda files relative to the location of the .o file in the instrumentation run. + String gcdaExt = Iterables.getOnlyElement(CppFileTypes.COVERAGE_DATA.getExtensions()); + PathFragment baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt); + PathFragment gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName); + + if (!gcdaFiles.contains(gcdaFile)) { + // If the object is a .pic.o file and .pic.gcda is not found, we should try finding .gcda too + String picoExt = Iterables.getOnlyElement(CppFileTypes.PIC_OBJECT_FILE.getExtensions()); + baseName = FileSystemUtils.replaceExtension(objectFileName, gcdaExt, picoExt); + if (baseName == null) { + // Object file is not .pic.o + return ImmutableList.of(); + } + gcdaFile = getNonLipoObjDir(ruleContext, lipoLabel).getRelative(baseName); + if (!gcdaFiles.contains(gcdaFile)) { + // .gcda file not found + return ImmutableList.of(); + } + } + + final Artifact artifact = env.getDerivedArtifact(fdoPath.getRelative(gcdaFile), fdoRoot); + env.registerAction(new FdoStubAction(ruleContext.getActionOwner(), artifact)); + + return ImmutableList.of(artifact); + } + + + private PathFragment getAutoProfilePath() { + return fdoRootExecPath.getRelative(getAutoProfileRootRelativePath()); + } + + private PathFragment getAutoProfileRootRelativePath() { + return new PathFragment(fdoProfile.getBaseName()); + } + + /** + * Returns whether LIPO is enabled. + */ + @ThreadSafe + public boolean isLipoEnabled() { + return fdoProfile != null && lipoMode != LipoMode.OFF; + } + + /** + * Returns whether AutoFDO is enabled. + */ + @ThreadSafe + public boolean isAutoFdoEnabled() { + return useAutoFdo; + } + + /** + * Returns an immutable list of command line arguments to add to the linker + * command line. If FDO is disabled, and empty list is returned. + */ + @ThreadSafe + public ImmutableList getLinkOptions() { + return fdoInstrument != null + ? ImmutableList.of("-fprofile-generate=" + fdoInstrument.getPathString()) + : ImmutableList.of(); + } + + /** + * Returns the path of the FDO output tree (relative to the execution root) + * containing the .gcda profile files, or null if FDO is not enabled. + */ + @VisibleForTesting + public PathFragment getFdoOptimizeDir() { + return fdoRootExecPath; + } + + /** + * Returns the path of the FDO zip containing the .gcda profile files, or null + * if FDO is not enabled. + */ + @VisibleForTesting + public Path getFdoOptimizeProfile() { + return fdoProfile; + } + + /** + * Returns the path fragment of the instrumentation output dir for gcc when + * FDO is enabled, or null if FDO is not enabled. + */ + @ThreadSafe + public PathFragment getFdoInstrument() { + return fdoInstrument; + } + + @VisibleForTesting + public void setGcdaFilesForTesting(ImmutableSet gcdaFiles) { + this.gcdaFiles = gcdaFiles; + } + + /** + * An exception indicating an issue with FDO coverage files. + */ + public static final class FdoException extends Exception { + FdoException(String message) { + super(message); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java new file mode 100644 index 0000000000..17e2e5ce11 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/HeaderTargetModuleMapProvider.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.List; + +/** + * A provider for cc_public_library rules to be able to convey the information about the + * header target's module map references to the public library target. + */ +@Immutable +public final class HeaderTargetModuleMapProvider implements TransitiveInfoProvider { + + private final ImmutableList cppModuleMaps; + + public HeaderTargetModuleMapProvider(Iterable cppModuleMaps) { + this.cppModuleMaps = ImmutableList.copyOf(cppModuleMaps); + } + + /** + * Returns the module maps referenced by cc_public_library's headers target. + */ + public List getCppModuleMaps() { + return cppModuleMaps; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java new file mode 100644 index 0000000000..4f2a5854d9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ImplementedCcPublicLibrariesProvider.java @@ -0,0 +1,42 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.syntax.Label; + +/** + * A provider for cc_library rules to be able to convey the information about which + * cc_public_library rules they implement to dependent targets. + */ +@Immutable +public final class ImplementedCcPublicLibrariesProvider implements TransitiveInfoProvider { + + private final ImmutableList

+ * The hints file is read, line by line, into a list of rules each of which + * encapsulates a line of four columns. Each non-blank, non-comment line has + * the format: + * + * 

+   *   "file"|"path"  match-pattern  find-root  find-filter
+   *
+ * + *

+ * The first column specifies whether the line is a rule based on matching + * source files (passed directly to gcc as inputs, or transitively + * #included by other inputs) or include paths (passed to gcc as + * -I, -iquote, or -isystem flags). + *

+ * The second column is a regexp for files or paths. Whenever a compiler + * argument of the specified type matches that regexp, the rule is taken. (All + * matching rules for every path and file on a compiler command line are + * followed, and the results are combined.) + *

+ * The third column is a point in the local filesystem from which to extract a + * recursive listing. (This follows symlinks) Backrefs may be used to refer to + * the regexp or its capturing groups. (This is mostly necessary because + * --package_path can cause input paths to carry arbitrary prefixes.) + *

+ * The fourth column is a regexp applied to each file found by the recursive + * listing. All matching files are treated as dependencies. + */ + public static class Hints implements SkyValue { + + private static final Pattern WS_PAT = Pattern.compile("\\s+"); + + private final Path workingDir; + private final List rules = new ArrayList<>(); + private final ArtifactFactory artifactFactory; + + private final LoadingCache> fileLevelHintsCache = + CacheBuilder.newBuilder().build( + new CacheLoader>() { + @Override + public Collection load(Artifact path) { + return getHintedInclusions(Rule.Type.FILE, path.getPath(), path.getRoot()); + } + }); + + private final LoadingCache> pathLevelHintsCache = + CacheBuilder.newBuilder().build( + new CacheLoader>() { + @Override + public Collection load(Path path) { + return getHintedInclusions(Rule.Type.PATH, path, null); + } + }); + + /** + * Constructs a hint set for a given working/exec directory and INCLUDE_HINTS file to read. + * + * @param workingDir the working/exec directory that processed paths are relative to + * @param hintsFile the hints file to read + * @throws IOException if the hints file can't be read or parsed + */ + public Hints(Path workingDir, Path hintsFile, ArtifactFactory artifactFactory) + throws IOException { + this.workingDir = workingDir; + this.artifactFactory = artifactFactory; + try (InputStream is = hintsFile.getInputStream()) { + for (String line : CharStreams.readLines(new InputStreamReader(is, "UTF-8"))) { + line = line.trim(); + if (line.length() == 0 || line.startsWith("#")) { + continue; + } + String[] tokens = WS_PAT.split(line); + try { + if (tokens.length == 3) { + rules.add(new Rule(tokens[0], tokens[1], tokens[2])); + } else if (tokens.length == 4) { + rules.add(new Rule(tokens[0], tokens[1], tokens[2], tokens[3])); + } else { + throw new IOException("Malformed hint line: " + line); + } + } catch (PatternSyntaxException e) { + throw new IOException("Malformed hint regex on: " + line + "\n " + e.getMessage()); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid type on: " + line + "\n " + e.getMessage()); + } + } + } + } + + /** + * Returns the "file" type hinted inclusions for a given path, caching results by path. + */ + public Collection getFileLevelHintedInclusions(Artifact path) { + return fileLevelHintsCache.getUnchecked(path); + } + + public Collection getPathLevelHintedInclusions(Path path) { + return pathLevelHintsCache.getUnchecked(path); + } + + /** + * Performs the work of matching a given file/path of a specified file/path type against the + * hints and returns the expanded paths. + */ + private Collection getHintedInclusions(Rule.Type type, Path path, + @Nullable Root sourceRoot) { + String pathString = path.getPathString(); + // Delay creation until we know we need one. Use a TreeSet to make sure that the results are + // sorted with a stable order and unique. + Set hints = null; + for (final Rule rule : rules) { + if (type != rule.type) { + continue; + } + Matcher m = rule.pattern.matcher(pathString); + if (!m.matches()) { + continue; + } + if (hints == null) { hints = Sets.newTreeSet(); } + Path root = workingDir.getRelative(m.replaceFirst(rule.findRoot)); + if (LOG_FINE) { + LOG.fine("hint for " + rule.type + " " + pathString + " root: " + root); + } + try { + // The assumption is made here that all files specified by this hint are under the same + // package path as the original file -- this filesystem tree traversal is completely + // ignorant of package paths. This could be violated if there were a hint that resolved to + // foo/**/*.h, there was a package foo/bar, and the packages foo and foo/bar were in + // different package paths. In that case, this traversal would fail to pick up + // foo/bar/**/*.h. No examples of this currently exist in the INCLUDE_HINTS + // file. + FileSystemUtils.traverseTree(hints, root, new Predicate() { + @Override + public boolean apply(Path p) { + boolean take = p.isFile() && rule.findFilter.matcher(p.getPathString()).matches(); + if (LOG_FINER && take) { + LOG.finer("hinted include: " + p); + } + return take; + } + }); + } catch (IOException e) { + LOG.warning("Error in hint expansion: " + e); + } + } + if (hints != null && !hints.isEmpty()) { + // Transform paths into source artifacts (all hints must be to source artifacts). + List result = new ArrayList<>(hints.size()); + for (Path hint : hints) { + if (hint.startsWith(workingDir)) { + // Paths that are under the execRoot can be resolved as source artifacts as usual. All + // include directories are specified relative to the execRoot, and so fall here. + result.add(Preconditions.checkNotNull( + artifactFactory.resolveSourceArtifact(hint.relativeTo(workingDir)), hint)); + } else { + // The file passed in might not have been under the execRoot, for instance + // /foo/foo.cc. + Preconditions.checkNotNull(sourceRoot, "%s %s", path, hint); + Path sourcePath = sourceRoot.getPath(); + Preconditions.checkState(hint.startsWith(sourcePath), + "%s %s %s", hint, path, sourceRoot); + result.add(Preconditions.checkNotNull( + artifactFactory.getSourceArtifact(hint.relativeTo(sourcePath), sourceRoot))); + } + } + return result; + } else { + return ImmutableList.of(); + } + } + + private Collection getHintedInclusions(Artifact path) { + String pathString = path.getPath().getPathString(); + // Delay creation until we know we need one. Use a LinkedHashSet to make sure that the results + // are sorted with a stable order and unique. + Set hints = null; + for (final Rule rule : rules) { + if ((rule.type != Rule.Type.INCLUDE_ANGLE) && (rule.type != Rule.Type.INCLUDE_QUOTE)) { + continue; + } + Matcher m = rule.pattern.matcher(pathString); + if (!m.matches()) { + continue; + } + if (hints == null) { hints = Sets.newLinkedHashSet(); } + Inclusion inclusion = new Inclusion(rule.findRoot, rule.type == Rule.Type.INCLUDE_QUOTE + ? Kind.QUOTE : Kind.ANGLE); + hints.add(inclusion); + if (LOG_FINE) { + LOG.fine("hint for " + rule.type + " " + pathString + " root: " + inclusion); + } + } + if (hints != null && !hints.isEmpty()) { + return ImmutableList.copyOf(hints); + } else { + return ImmutableList.of(); + } + } + } + + public Hints getHints() { + return hints; + } + + /** + * An immutable inclusion tuple. This models an {@code #include} or {@code + * #include_next} line in a file without the context how this file got + * included. + */ + public static class Inclusion { + /** The format of the #include in the source file -- quoted, angle bracket, etc. */ + public enum Kind { + /** Quote includes: {@code #include "name"}. */ + QUOTE, + + /** Angle bracket includes: {@code #include }. */ + ANGLE, + + /** Quote next includes: {@code #include_next "name"}. */ + NEXT_QUOTE, + + /** Angle next includes: {@code #include_next }. */ + NEXT_ANGLE, + + /** Computed or other unhandlable includes: {@code #include HEADER}. */ + OTHER; + + /** + * Returns true if this is an {@code #include_next} inclusion, + */ + public boolean isNext() { + return this == NEXT_ANGLE || this == NEXT_QUOTE; + } + } + + /** The kind of inclusion. */ + public final Kind kind; + /** The relative path of the inclusion. */ + public final PathFragment pathFragment; + + public Inclusion(String includeTarget, Kind kind) { + this.kind = kind; + this.pathFragment = new PathFragment(includeTarget); + } + + public Inclusion(PathFragment pathFragment, Kind kind) { + this.kind = kind; + this.pathFragment = Preconditions.checkNotNull(pathFragment); + } + + public String getPathString() { + return pathFragment.getPathString(); + } + + @Override + public String toString() { + return kind.toString() + ":" + pathFragment.getPathString(); + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (!(o instanceof Inclusion)) { + return false; + } + Inclusion that = (Inclusion) o; + return kind == that.kind && pathFragment.equals(that.pathFragment); + } + + @Override + public int hashCode() { + return pathFragment.hashCode() * 37 + kind.hashCode(); + } + } + + /** + * The externally-scoped immutable hints helper that is shared by all scanners. + */ + private final Hints hints; + + /** + * A scanner that extracts includes from an individual files remotely, used when scanning files + * generated remotely. + */ + private final Supplier remoteExtractor; + + /** + * Constructs a new FileParser. + * @param remoteExtractor a processor that extracts includes from an individual file remotely. + * @param hints regexps for converting computed includes into simple strings + */ + public IncludeParser(@Nullable RemoteIncludeExtractor remoteExtractor, Hints hints) { + this.hints = hints; + this.remoteExtractor = Suppliers.ofInstance(remoteExtractor); + } + + /** + * Constructs a new FileParser. + * @param remoteExtractorSupplier a supplier of a processor that extracts includes from an + * individual file remotely. + * @param hints regexps for converting computed includes into simple strings + */ + public IncludeParser(Supplier remoteExtractorSupplier, + Hints hints) { + this.hints = hints; + this.remoteExtractor = remoteExtractorSupplier; + } + + /** + * Skips whitespace, \+NL pairs, and block-style / * * / comments. Assumes + * line comments are handled outside. Does not handle digraphs, trigraphs or + * decahexagraphs. + * + * @param chars characters to scan + * @param pos the starting position + * @return the resulting position after skipping whitespace and comments. + */ + protected static int skipWhitespace(char[] chars, int pos, int end) { + while (pos < end) { + if (Character.isWhitespace(chars[pos])) { + pos++; + } else if (chars[pos] == '\\' && pos + 1 < end && chars[pos + 1] == '\n') { + pos++; + } else if (chars[pos] == '/' && pos + 1 < end && chars[pos + 1] == '*') { + pos += 2; + while (pos < end - 1) { + if (chars[pos++] == '*') { + if (chars[pos] == '/') { + pos++; + break; // proper comment end + } + } + } + } else { // not whitespace + return pos; + } + } + return pos; // pos == len, meaning we fell off the end. + } + + /** + * Checks for and skips a given token. + * + * @param chars characters to scan + * @param pos the starting position + * @param expected the expected token + * @return the resulting position if found, otherwise -1 + */ + protected static int expect(char[] chars, int pos, int end, String expected) { + int si = 0; + int expectedLen = expected.length(); + while (pos < end) { + if (si == expectedLen) { + return pos; + } + if (chars[pos++] != expected.charAt(si++)) { + return -1; + } + } + return -1; + } + + /** + * Finds the index of a given character token from a starting pos. + * + * @param chars characters to scan + * @param pos the starting position + * @param echar the character to find + * @return the resulting position of echar if found, otherwise -1 + */ + private static int indexOf(char[] chars, int pos, int end, char echar) { + while (pos < end) { + if (chars[pos] == echar) { + return pos; + } + pos++; + } + return -1; + } + + private static final Pattern BS_NL_PAT = Pattern.compile("\\\\" + "\n"); + + // Keep this in sync with the auxiliary binary's scanning output format. + private static final ImmutableMap KIND_MAP = ImmutableMap.of( + '"', Kind.QUOTE, + '<', Kind.ANGLE, + 'q', Kind.NEXT_QUOTE, + 'a', Kind.NEXT_ANGLE); + + /** + * Processes the output generated by an auxiliary include-scanning binary. Closes the stream upon + * completion. + * + *

If a source file has the following include statements: + * 

+   *   #include <string>
+   *   #include "directory/header.h"
+   *
+ * + *

Then the output file has the following contents: + * 

+   *   "directory/header.h
+   *   <string
+   *
+ *

Each line of the output is translated into an Inclusion object. + */ + public static List processIncludes(Object streamName, InputStream is) + throws IOException { + List inclusions = new ArrayList<>(); + InputStreamReader reader = new InputStreamReader(is, ISO_8859_1); + try { + for (String line : CharStreams.readLines(reader)) { + char qchar = line.charAt(0); + String name = line.substring(1); + Inclusion.Kind kind = KIND_MAP.get(qchar); + if (kind == null) { + throw new IOException("Illegal inclusion kind '" + qchar + "'"); + } + inclusions.add(new Inclusion(name, kind)); + } + } catch (IOException e) { + throw new IOException("Error reading include file " + streamName + ": " + e.getMessage()); + } finally { + reader.close(); + } + return inclusions; + } + + @VisibleForTesting + Inclusion extractInclusion(String line) { + return extractInclusion(line.toCharArray(), 0, line.length()); + } + + /** + * Extracts a new, unresolved an Inclusion from a line of source. + * + * @param chars the char array containing the line chars to parse + * @param lineBegin the position of the first character in the line + * @param lineEnd the position of the character after the last + * @return the inclusion object if possible, null if none + */ + private Inclusion extractInclusion(char[] chars, int lineBegin, int lineEnd) { + // expect WS#WS(include|include_next)WS("name"||junk) + int pos = expectIncludeKeyword(chars, lineBegin, lineEnd); + if (pos == -1 || pos == lineEnd) { + return null; + } + boolean isNext = false; + int npos = expect(chars, pos, lineEnd, "_next"); + if (npos >= 0) { + isNext = true; + pos = npos; + } + if ((pos = skipWhitespace(chars, pos, lineEnd)) == lineEnd) { + return null; + } + if (chars[pos] == '"' || chars[pos] == '<') { + char qchar = chars[pos++]; + int spos = pos; + pos = indexOf(chars, pos + 1, lineEnd, qchar == '<' ? '>' : '"'); + if (pos < 0) { + return null; + } + if (chars[spos] == '/') { + return null; // disallow absolute paths + } + String name = new String(chars, spos, pos - spos); + if (name.contains("\n")) { // strip any \+NL pairs within name + name = BS_NL_PAT.matcher(name).replaceAll(""); + } + if (isNext) { + return new Inclusion(name, qchar == '"' ? Kind.NEXT_QUOTE : Kind.NEXT_ANGLE); + } else { + return new Inclusion(name, qchar == '"' ? Kind.QUOTE : Kind.ANGLE); + } + } else { + return createOtherInclusion(new String(chars, pos, lineEnd - pos)); + } + } + + /** + * Extracts all inclusions from characters of a file. + * + * @param chars the file contents to parse & extract inclusions from + * @return a new set of inclusions, normalized to the cache + */ + @VisibleForTesting + List extractInclusions(char[] chars) { + List inclusions = new ArrayList<>(); + int lineBegin = 0; // the first char of each line + int end = chars.length; // the file end + while (lineBegin < end) { + int lineEnd = lineBegin; // the char after the last non-\n in each line + // skip to the next \n or after end of buffer, ignoring continuations + while (lineEnd < end) { + if (chars[lineEnd] == '\n') { + break; + } else if (chars[lineEnd] == '\\') { + lineEnd++; + if (chars[lineEnd] == '\n') { + lineEnd++; + } + } else { + lineEnd++; + } + } + + // TODO(bazel-team) handle multiline block comments /* */ for the cases: + // /* blah blah blah + // lalala */ #include "foo.h" + // and: + // /* blah + // #include "foo.h" + // */ + + // extract the inclusion, and save only the kind we care about. + Inclusion inclusion = extractInclusion(chars, lineBegin, lineEnd); + if (inclusion != null) { + if (isValidInclusionKind(inclusion.kind)) { + inclusions.add(inclusion); + } else { + //System.err.println("Funky include " + inclusion + " in " + file); + } + } + lineBegin = lineEnd + 1; // next line starts after the previous line + } + return inclusions; + } + + /** + * Extracts all inclusions from a given source file. + * + * @param file the file to parse & extract inclusions from + * @param greppedFile if non-null, this file has the already-grepped include lines of file. + * @param actionExecutionContext Services in the scope of the action, like the stream to which + * scanning messages are printed + * @return a new set of inclusions, normalized to the cache + */ + public Collection extractInclusions(Artifact file, @Nullable Path greppedFile, + ActionExecutionContext actionExecutionContext) + throws IOException, InterruptedException { + Collection inclusions; + if (greppedFile != null) { + inclusions = processIncludes(greppedFile, greppedFile.getInputStream()); + } else { + RemoteParseData remoteParseData = remoteExtractor.get() == null + ? null + : remoteExtractor.get().shouldParseRemotely(file.getPath()); + if (remoteParseData != null && remoteParseData.shouldParseRemotely()) { + inclusions = + remoteExtractor.get().extractInclusions(file, actionExecutionContext, + remoteParseData); + } else { + inclusions = extractInclusions(FileSystemUtils.readContentAsLatin1(file.getPath())); + } + } + if (hints != null) { + inclusions.addAll(hints.getHintedInclusions(file)); + } + return ImmutableList.copyOf(inclusions); + } + + /** + * Parses include keyword in the provided char array and returns position + * immediately after include keyword or -1 if keyword was not found. Can be + * overridden by subclasses. + */ + protected int expectIncludeKeyword(char[] chars, int position, int end) { + int pos = expect(chars, skipWhitespace(chars, position, end), end, "#"); + if (pos > 0) { + int npos = skipWhitespace(chars, pos, end); + if ((pos = expect(chars, npos, end, "include")) > 0) { + return pos; + } else if ((pos = expect(chars, npos, end, "import")) > 0) { + if (expect(chars, pos, end, "_") == -1) { // Needed to avoid #import_next. + return pos; + } + } + } + return -1; + } + + /** + * Returns true if we interested in the given inclusion kind. Can be + * overridden by the subclass. + */ + protected boolean isValidInclusionKind(Kind kind) { + return kind != Kind.OTHER; + } + + /** + * Returns inclusion object for non-standard inclusion cases or null if + * inclusion should be ignored. + */ + protected Inclusion createOtherInclusion(String inclusionContent) { + return new Inclusion(inclusionContent, Kind.OTHER); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java new file mode 100644 index 0000000000..f6be87747e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeProblems.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Accumulator for problems encountered while reading or validating inclusion + * results. + */ +class IncludeProblems { + + private StringBuilder message; // null when no problems + + void add(String included) { + if (message == null) { message = new StringBuilder(); } + message.append("\n '" + included + "'"); + } + + boolean hasProblems() { return message != null; } + + String getMessage(Action action, Artifact sourceFile) { + if (message != null) { + return "undeclared inclusion(s) in rule '" + action.getOwner().getLabel() + "':\n" + + "this rule is missing dependency declarations for the following files " + + "included by '" + sourceFile.prettyPrint() + "':" + + message; + } + return null; + } + + void assertProblemFree(Action action, Artifact sourceFile) throws ActionExecutionException { + if (hasProblems()) { + throw new ActionExecutionException(getMessage(action, sourceFile), action, false); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java new file mode 100644 index 0000000000..9c70090ac0 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScannable.java @@ -0,0 +1,90 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * To be implemented by actions (such as C++ compilation steps) whose inputs + * can be scanned to discover other implicit inputs (such as C++ header files). + * + *

This is useful for remote execution strategies to be able to compute the + * complete set of files that must be distributed in order to execute such an action. + */ +public interface IncludeScannable { + + /** + * Returns the built-in list of system include paths for the toolchain compiler. All paths in this + * list should be relative to the exec directory. They may be absolute if they are also installed + * on the remote build nodes or for local compilation. + */ + List getBuiltInIncludeDirectories(); + + /** + * Returns an immutable list of "-iquote" include paths that should be used by + * the IncludeScanner for this action. GCC searches these paths first, but + * only for {@code #include "foo"}, not for {@code #include <foo>}. + */ + List getQuoteIncludeDirs(); + + /** + * Returns an immutable list of "-I" include paths that should be used by the + * IncludeScanner for this action. GCC searches these paths ahead of the + * system include paths, but after "-iquote" include paths. + */ + List getIncludeDirs(); + + /** + * Returns an immutable list of "-isystem" include paths that should be used + * by the IncludeScanner for this action. GCC searches these paths ahead of + * the built-in system include paths, but after all other paths. "-isystem" + * paths are treated the same as normal system directories. + */ + List getSystemIncludeDirs(); + + /** + * Returns an immutable list of "-include" inclusions specified explicitly on + * the command line of this action. GCC will imagine that these files have + * been quote-included at the beginning of each source file. + */ + List getCmdlineIncludes(); + + /** + * Returns an immutable list of sources that the IncludeScanner should scan + * for this action. + */ + Collection getIncludeScannerSources(); + + /** + * Returns additional scannables that need also be scanned when scanning this + * scannable. May be empty but not null. This is not evaluated recursively. + */ + Iterable getAuxiliaryScannables(); + + /** + * Returns a map of generated files:files grepped for headers which may be reached during include + * scanning. Generated files which are reached, but not in the key set, must be ignored. + * + *

If grepping of output files is not enabled via --extract_generated_inclusions, keys + * should just map to null. + */ + Map getLegalGeneratedScannerFileMap(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java new file mode 100644 index 0000000000..9c00efd64a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanner.java @@ -0,0 +1,177 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.EnvironmentalExecException; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Scans source files to determine the bounding set of transitively referenced include files. + * + *

Note that include scanning is performance-critical code. + */ +public interface IncludeScanner { + /** + * Processes a source file and a list of includes extracted from command line + * flags. Adds all found files to the provided set {@code includes}. This + * method takes into account the path- and file-level hints that are part of + * this include scanner. + */ + public void process(Artifact source, Map legalOutputPaths, + List cmdlineIncludes, Set includes, + ActionExecutionContext actionExecutionContext) + throws IOException, ExecException, InterruptedException; + + /** Supplies IncludeScanners upon request. */ + interface IncludeScannerSupplier { + /** Returns the possibly shared scanner to be used for a given pair of include paths. */ + IncludeScanner scannerFor(List quoteIncludePaths, List includePaths); + } + + /** + * Helper class that exists just to provide a static method that prepares the arguments with which + * to call an IncludeScanner. + */ + class IncludeScanningPreparer { + private IncludeScanningPreparer() {} + + /** + * Returns the files transitively included by the source files of the given IncludeScannable. + * + * @param action IncludeScannable whose sources' transitive includes will be returned. + * @param includeScannerSupplier supplies IncludeScanners to actually do the transitive + * scanning (and caching results) for a given source file. + * @param actionExecutionContext the context for {@code action}. + * @param profilerTaskName what the {@link Profiler} should record this call for. + */ + public static Collection scanForIncludedInputs(IncludeScannable action, + IncludeScannerSupplier includeScannerSupplier, + ActionExecutionContext actionExecutionContext, + String profilerTaskName) + throws ExecException, InterruptedException, ActionExecutionException { + + Set includes = Sets.newConcurrentHashSet(); + + Executor executor = actionExecutionContext.getExecutor(); + Path execRoot = executor.getExecRoot(); + + final List absoluteBuiltInIncludeDirs = new ArrayList<>(); + + Profiler profiler = Profiler.instance(); + try { + profiler.startTask(ProfilerTask.SCANNER, profilerTaskName); + + // We need to scan the action itself, but also the auxiliary scannables + // (for LIPO). There is no need to call getAuxiliaryScannables + // recursively. + for (IncludeScannable scannable : + Iterables.concat(ImmutableList.of(action), action.getAuxiliaryScannables())) { + + Map legalOutputPaths = scannable.getLegalGeneratedScannerFileMap(); + List includeDirs = new ArrayList<>(scannable.getIncludeDirs()); + List quoteIncludeDirs = scannable.getQuoteIncludeDirs(); + List cmdlineIncludes = scannable.getCmdlineIncludes(); + + for (PathFragment pathFragment : scannable.getSystemIncludeDirs()) { + includeDirs.add(pathFragment); + } + + // Add the system include paths to the list of include paths. + for (PathFragment pathFragment : action.getBuiltInIncludeDirectories()) { + if (pathFragment.isAbsolute()) { + absoluteBuiltInIncludeDirs.add(execRoot.getRelative(pathFragment)); + } + includeDirs.add(pathFragment); + } + + IncludeScanner scanner = includeScannerSupplier.scannerFor( + relativeTo(execRoot, quoteIncludeDirs), + relativeTo(execRoot, includeDirs)); + + for (Artifact source : scannable.getIncludeScannerSources()) { + // Add all include scanning entry points to the inputs; this is necessary + // when we have more than one source to scan from, for example when building + // C++ modules. + // In that case we have one of two cases: + // 1. We compile a header module - there, the .cppmap file is the main source file + // (which we do not include-scan, as that would require an extra parser), and + // thus already in the input; all headers in the .cppmap file are our entry points + // for include scanning, but are not yet in the inputs - they get added here. + // 2. We compile an object file that uses a header module; currently using a header + // module requires all headers it can reference to be available for the compilation. + // The header module can reference headers that are not in the transitive include + // closure of the current translation unit. Therefore, {@code CppCompileAction} + // adds all headers specified transitively for compiled header modules as include + // scanning entry points, and we need to add the entry points to the inputs here. + includes.add(source); + scanner.process(source, legalOutputPaths, cmdlineIncludes, includes, + actionExecutionContext); + } + } + } catch (IOException e) { + throw new EnvironmentalExecException(e.getMessage()); + } finally { + profiler.completeTask(ProfilerTask.SCANNER); + } + + // Collect inputs and output + List inputs = new ArrayList<>(); + IncludeProblems includeProblems = new IncludeProblems(); + for (Artifact included : includes) { + if (FileSystemUtils.startsWithAny(included.getPath(), absoluteBuiltInIncludeDirs)) { + // Skip include files found in absolute include directories. This currently only applies + // to grte. + continue; + } + if (included.getRoot().getPath().getParentDirectory() == null) { + throw new UserExecException( + "illegal absolute path to include file: " + included.getPath()); + } + inputs.add(included); + } + return inputs; + } + + private static List relativeTo( + Path path, Collection fragments) { + List result = Lists.newArrayListWithCapacity(fragments.size()); + for (PathFragment fragment : fragments) { + result.add(path.getRelative(fragment)); + } + return result; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java new file mode 100644 index 0000000000..69cd26b6bd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/IncludeScanningContext.java @@ -0,0 +1,44 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.Executor.ActionContext; + +import java.io.IOException; + +/** + * Context for actions that do include scanning. + */ +public interface IncludeScanningContext extends ActionContext { + /** + * Extracts the set of include files from a source file. + * + * @param actionExecutionContext the execution context + * @param resourceOwner the resource owner + * @param primaryInput the source file to be include scanned + * @param primaryOutput the output file where the results should be put + */ + void extractIncludes(ActionExecutionContext actionExecutionContext, + ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput) + throws IOException, InterruptedException; + + /** + * Returns the artifact resolver. + */ + ArtifactResolver getArtifactResolver(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java new file mode 100644 index 0000000000..26175ebb95 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Link.java @@ -0,0 +1,274 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.FileTypeSet; + +import java.util.Iterator; + +/** + * Utility types and methods for generating command lines for the linker, given + * a CppLinkAction or LinkConfiguration. + * + *

The linker commands, e.g. "ar", may not be functional, i.e. + * they may mutate the output file rather than overwriting it. + * To avoid this, we need to delete the output file before invoking the + * command. But that is not done by this class; deleting the output + * file is the responsibility of the classes derived from LinkStrategy. + */ +public abstract class Link { + + private Link() {} // uninstantiable + + /** The set of valid linker input files. */ + public static final FileTypeSet VALID_LINKER_INPUTS = FileTypeSet.of( + CppFileTypes.ARCHIVE, CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, CppFileTypes.ALWAYS_LINK_PIC_LIBRARY, + CppFileTypes.OBJECT_FILE, CppFileTypes.PIC_OBJECT_FILE, + CppFileTypes.SHARED_LIBRARY, CppFileTypes.VERSIONED_SHARED_LIBRARY, + CppFileTypes.INTERFACE_SHARED_LIBRARY); + + /** + * These file are supposed to be added using {@code addLibrary()} calls to {@link CppLinkAction} + * but will never be expanded to their constituent {@code .o} files. {@link CppLinkAction} checks + * that these files are never added as non-libraries. + */ + public static final FileTypeSet SHARED_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.SHARED_LIBRARY, + CppFileTypes.VERSIONED_SHARED_LIBRARY, + CppFileTypes.INTERFACE_SHARED_LIBRARY); + + /** + * These need special handling when --thin_archive is true. {@link CppLinkAction} checks that + * these files are never added as non-libraries. + */ + public static final FileTypeSet ARCHIVE_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.ARCHIVE, + CppFileTypes.PIC_ARCHIVE, + CppFileTypes.ALWAYS_LINK_LIBRARY, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY); + + public static final FileTypeSet ARCHIVE_FILETYPES = FileTypeSet.of( + CppFileTypes.ARCHIVE, + CppFileTypes.PIC_ARCHIVE); + + public static final FileTypeSet LINK_LIBRARY_FILETYPES = FileTypeSet.of( + CppFileTypes.ALWAYS_LINK_LIBRARY, + CppFileTypes.ALWAYS_LINK_PIC_LIBRARY); + + + /** The set of object files */ + public static final FileTypeSet OBJECT_FILETYPES = FileTypeSet.of( + CppFileTypes.OBJECT_FILE, + CppFileTypes.PIC_OBJECT_FILE); + + /** + * Prefix that is prepended to command line entries that refer to the output + * of cc_fake_binary compile actions. This is a bad hack to signal to the code + * in {@code CppLinkAction#executeFake(Executor, FileOutErr)} that it needs + * special handling. + */ + public static final String FAKE_OBJECT_PREFIX = "fake:"; + + /** + * Types of ELF files that can be created by the linker (.a, .so, .lo, + * executable). + */ + public enum LinkTargetType { + /** A normal static archive. */ + STATIC_LIBRARY(".a", true), + + /** A static archive with .pic.o object files (compiled with -fPIC). */ + PIC_STATIC_LIBRARY(".pic.a", true), + + /** An interface dynamic library. */ + INTERFACE_DYNAMIC_LIBRARY(".ifso", false), + + /** A dynamic library. */ + DYNAMIC_LIBRARY(".so", false), + + /** A static archive without removal of unused object files. */ + ALWAYS_LINK_STATIC_LIBRARY(".lo", true), + + /** A PIC static archive without removal of unused object files. */ + ALWAYS_LINK_PIC_STATIC_LIBRARY(".pic.lo", true), + + /** An executable binary. */ + EXECUTABLE("", false); + + private final String extension; + private final boolean staticLibraryLink; + + private LinkTargetType(String extension, boolean staticLibraryLink) { + this.extension = extension; + this.staticLibraryLink = staticLibraryLink; + } + + public String getExtension() { + return extension; + } + + public boolean isStaticLibraryLink() { + return staticLibraryLink; + } + } + + /** + * The degree of "staticness" of symbol resolution during linking. + */ + public enum LinkStaticness { + FULLY_STATIC, // Static binding of all symbols. + MOSTLY_STATIC, // Use dynamic binding only for symbols from glibc. + DYNAMIC, // Use dynamic binding wherever possible. + } + + /** + * Types of archive. + */ + public enum ArchiveType { + FAT, // Regular archive that includes its members. + THIN, // Thin archive that just points to its members. + START_END_LIB // A --start-lib ... --end-lib group in the command line. + } + + static boolean useStartEndLib(LinkerInput linkerInput, ArchiveType archiveType) { + // TODO(bazel-team): Figure out if PicArchives are actually used. For it to be used, both + // linkingStatically and linkShared must me true, we must be in opt mode and cpu has to be k8. + return archiveType == ArchiveType.START_END_LIB + && ARCHIVE_FILETYPES.matches(linkerInput.getArtifact().getFilename()) + && linkerInput.containsObjectFiles(); + } + + /** + * Replace always used archives with its members. This is used to build the linker cmd line. + */ + public static Iterable mergeInputsCmdLine(NestedSet inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType) { + return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, false); + } + + /** + * Add in any object files which are implicitly named as inputs by the linker. + */ + public static Iterable mergeInputsDependencies(NestedSet inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType) { + return new FilterMembersForLinkIterable(inputs, globalNeedWholeArchive, archiveType, true); + } + + /** + * On the fly implementation to filter the members. + */ + private static final class FilterMembersForLinkIterable implements Iterable { + private final boolean globalNeedWholeArchive; + private final ArchiveType archiveType; + private final boolean deps; + + private final Iterable inputs; + + private FilterMembersForLinkIterable(Iterable inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) { + this.globalNeedWholeArchive = globalNeedWholeArchive; + this.archiveType = archiveType; + this.deps = deps; + this.inputs = CollectionUtils.makeImmutable(inputs); + } + + @Override + public Iterator iterator() { + return new FilterMembersForLinkIterator(inputs.iterator(), globalNeedWholeArchive, + archiveType, deps); + } + } + + /** + * On the fly implementation to filter the members. + */ + private static final class FilterMembersForLinkIterator extends AbstractIterator { + private final boolean globalNeedWholeArchive; + private final ArchiveType archiveType; + private final boolean deps; + + private final Iterator inputs; + private Iterator delayList = ImmutableList.of().iterator(); + + private FilterMembersForLinkIterator(Iterator inputs, + boolean globalNeedWholeArchive, ArchiveType archiveType, boolean deps) { + this.globalNeedWholeArchive = globalNeedWholeArchive; + this.archiveType = archiveType; + this.deps = deps; + this.inputs = inputs; + } + + @Override + protected LinkerInput computeNext() { + if (delayList.hasNext()) { + return delayList.next(); + } + + while (inputs.hasNext()) { + LibraryToLink inputLibrary = inputs.next(); + Artifact input = inputLibrary.getArtifact(); + String name = input.getFilename(); + + // True if the linker might use the members of this file, i.e., if the file is a thin or + // start_end_lib archive (aka static library). Also check if the library contains object + // files - otherwise getObjectFiles returns null, which would lead to an NPE in + // simpleLinkerInputs. + boolean needMembersForLink = archiveType != ArchiveType.FAT + && ARCHIVE_LIBRARY_FILETYPES.matches(name) && inputLibrary.containsObjectFiles(); + + // True if we will pass the members instead of the original archive. + boolean passMembersToLinkCmd = needMembersForLink + && (globalNeedWholeArchive || LINK_LIBRARY_FILETYPES.matches(name)); + + // If deps is false (when computing the inputs to be passed on the command line), then it's + // an if-then-else, i.e., the passMembersToLinkCmd flag decides whether to pass the object + // files or the archive itself. This flag in turn is based on whether the archives are fat + // or not (thin archives or start_end_lib) - we never expand fat archives, but we do expand + // non-fat archives if we need whole-archives for the entire link, or for the specific + // library (i.e., if alwayslink=1). + // + // If deps is true (when computing the inputs to be passed to the action as inputs), then it + // becomes more complicated. We always need to pass the members for thin and start_end_lib + // archives (needMembersForLink). And we _also_ need to pass the archive file itself unless + // it's a start_end_lib archive (unless it's an alwayslink library). + + // A note about ordering: the order in which the object files and the library are returned + // does not currently matter - this code results in the library returned first, and the + // object files returned after, but only if both are returned, which can only happen if + // deps is true, in which case this code only computes the list of inputs for the link + // action (so the order isn't critical). + if (passMembersToLinkCmd || (deps && needMembersForLink)) { + delayList = LinkerInputs.simpleLinkerInputs(inputLibrary.getObjectFiles()).iterator(); + } + + if (!(passMembersToLinkCmd || (deps && useStartEndLib(inputLibrary, archiveType)))) { + return inputLibrary; + } + + if (delayList.hasNext()) { + return delayList.next(); + } + } + return endOfData(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java new file mode 100644 index 0000000000..1dccafa8cc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLine.java @@ -0,0 +1,1121 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.actions.CommandLine; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.rules.cpp.Link.LinkStaticness; +import com.google.devtools.build.lib.rules.cpp.Link.LinkTargetType; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.util.ShellEscaper; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +/** + * Represents the command line of a linker invocation. It supports executables and dynamic + * libraries as well as static libraries. + */ +@Immutable +public final class LinkCommandLine extends CommandLine { + private final BuildConfiguration configuration; + private final CppConfiguration cppConfiguration; + private final ActionOwner owner; + private final Artifact output; + @Nullable private final Artifact interfaceOutput; + @Nullable private final Artifact symbolCountsOutput; + private final ImmutableList buildInfoHeaderArtifacts; + private final Iterable linkerInputs; + private final Iterable runtimeInputs; + private final LinkTargetType linkTargetType; + private final LinkStaticness linkStaticness; + private final ImmutableList linkopts; + private final ImmutableSet features; + private final ImmutableMap linkstamps; + private final ImmutableList linkstampCompileOptions; + @Nullable private final PathFragment runtimeSolibDir; + private final boolean nativeDeps; + private final boolean useTestOnlyFlags; + private final boolean needWholeArchive; + private final boolean supportsParamFiles; + @Nullable private final Artifact interfaceSoBuilder; + + private LinkCommandLine( + BuildConfiguration configuration, + ActionOwner owner, + Artifact output, + @Nullable Artifact interfaceOutput, + @Nullable Artifact symbolCountsOutput, + ImmutableList buildInfoHeaderArtifacts, + Iterable linkerInputs, + Iterable runtimeInputs, + LinkTargetType linkTargetType, + LinkStaticness linkStaticness, + ImmutableList linkopts, + ImmutableSet features, + ImmutableMap linkstamps, + ImmutableList linkstampCompileOptions, + @Nullable PathFragment runtimeSolibDir, + boolean nativeDeps, + boolean useTestOnlyFlags, + boolean needWholeArchive, + boolean supportsParamFiles, + Artifact interfaceSoBuilder) { + Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY, + "you can't link an interface dynamic library directly"); + if (linkTargetType != LinkTargetType.DYNAMIC_LIBRARY) { + Preconditions.checkArgument(interfaceOutput == null, + "interface output may only be non-null for dynamic library links"); + } + if (linkTargetType.isStaticLibraryLink()) { + Preconditions.checkArgument(linkstamps.isEmpty(), + "linkstamps may only be present on dynamic library or executable links"); + Preconditions.checkArgument(linkStaticness == LinkStaticness.FULLY_STATIC, + "static library link must be static"); + Preconditions.checkArgument(buildInfoHeaderArtifacts.isEmpty(), + "build info headers may only be present on dynamic library or executable links"); + Preconditions.checkArgument(symbolCountsOutput == null, + "the symbol counts output must be null for static links"); + Preconditions.checkArgument(runtimeSolibDir == null, + "the runtime solib directory must be null for static links"); + Preconditions.checkArgument(!nativeDeps, + "the native deps flag must be false for static links"); + Preconditions.checkArgument(!needWholeArchive, + "the need whole archive flag must be false for static links"); + } + + this.configuration = Preconditions.checkNotNull(configuration); + this.cppConfiguration = configuration.getFragment(CppConfiguration.class); + this.owner = Preconditions.checkNotNull(owner); + this.output = Preconditions.checkNotNull(output); + this.interfaceOutput = interfaceOutput; + this.symbolCountsOutput = symbolCountsOutput; + this.buildInfoHeaderArtifacts = Preconditions.checkNotNull(buildInfoHeaderArtifacts); + this.linkerInputs = Preconditions.checkNotNull(linkerInputs); + this.runtimeInputs = Preconditions.checkNotNull(runtimeInputs); + this.linkTargetType = Preconditions.checkNotNull(linkTargetType); + this.linkStaticness = Preconditions.checkNotNull(linkStaticness); + // For now, silently ignore linkopts if this is a static library link. + this.linkopts = linkTargetType.isStaticLibraryLink() + ? ImmutableList.of() + : Preconditions.checkNotNull(linkopts); + this.features = Preconditions.checkNotNull(features); + this.linkstamps = Preconditions.checkNotNull(linkstamps); + this.linkstampCompileOptions = linkstampCompileOptions; + this.runtimeSolibDir = runtimeSolibDir; + this.nativeDeps = nativeDeps; + this.useTestOnlyFlags = useTestOnlyFlags; + this.needWholeArchive = needWholeArchive; + this.supportsParamFiles = supportsParamFiles; + // For now, silently ignore interfaceSoBuilder if we don't build an interface dynamic library. + this.interfaceSoBuilder = + ((linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) && (interfaceOutput != null)) + ? Preconditions.checkNotNull(interfaceSoBuilder, + "cannot build interface dynamic library without builder") + : null; + } + + /** + * Returns an interface shared object output artifact produced during linking. This only returns + * non-null if {@link #getLinkTargetType} is {@code DYNAMIC_LIBRARY} and an interface shared + * object was requested. + */ + @Nullable public Artifact getInterfaceOutput() { + return interfaceOutput; + } + + /** + * Returns an artifact containing the number of symbols used per object file passed to the linker. + * This is currently a gold only feature, and is only produced for executables. If another target + * is being linked, or if symbol counts output is disabled, this will be null. + */ + @Nullable public Artifact getSymbolCountsOutput() { + return symbolCountsOutput; + } + + /** + * Returns the (ordered, immutable) list of header files that contain build info. + */ + public ImmutableList getBuildInfoHeaderArtifacts() { + return buildInfoHeaderArtifacts; + } + + /** + * Returns the (ordered, immutable) list of paths to the linker's input files. + */ + public Iterable getLinkerInputs() { + return linkerInputs; + } + + /** + * Returns the runtime inputs to the linker. + */ + public Iterable getRuntimeInputs() { + return runtimeInputs; + } + + /** + * Returns the current type of link target set. + */ + public LinkTargetType getLinkTargetType() { + return linkTargetType; + } + + /** + * Returns the "staticness" of the link. + */ + public LinkStaticness getLinkStaticness() { + return linkStaticness; + } + + /** + * Returns the additional linker options for this link. + */ + public ImmutableList getLinkopts() { + return linkopts; + } + + /** + * Returns a (possibly empty) mapping of (C++ source file, .o output file) pairs for source files + * that need to be compiled at link time. + * + *

This is used to embed various values from the build system into binaries to identify their + * provenance. + */ + public ImmutableMap getLinkstamps() { + return linkstamps; + } + + /** + * Returns the location of the C++ runtime solib symlinks. If null, the C++ dynamic runtime + * libraries either do not exist (because they do not come from the depot) or they are in the + * regular solib directory. + */ + @Nullable public PathFragment getRuntimeSolibDir() { + return runtimeSolibDir; + } + + /** + * Returns true for libraries linked as native dependencies for other languages. + */ + public boolean isNativeDeps() { + return nativeDeps; + } + + /** + * Returns true if this link should use test-specific flags (e.g. $EXEC_ORIGIN as the root for + * finding shared libraries or lazy binding); false by default. See bug "Please use + * $EXEC_ORIGIN instead of $ORIGIN when linking cc_tests" for further context. + */ + public boolean useTestOnlyFlags() { + return useTestOnlyFlags; + } + + /** + * Splits the link command-line into a part to be written to a parameter file, and the remaining + * actual command line to be executed (which references the parameter file). Call {@link + * #canBeSplit} first to check if the command-line can be split. + * + * @throws IllegalStateException if the command-line cannot be split + */ + @VisibleForTesting + final Pair, List> splitCommandline(PathFragment paramExecPath) { + Preconditions.checkState(canBeSplit()); + List args = getRawLinkArgv(); + if (linkTargetType.isStaticLibraryLink()) { + // Ar link commands can also generate huge command lines. + List paramFileArgs = args.subList(1, args.size()); + List commandlineArgs = new ArrayList<>(); + commandlineArgs.add(args.get(0)); + + commandlineArgs.add("@" + paramExecPath.getPathString()); + return Pair.of(commandlineArgs, paramFileArgs); + } else { + // Gcc link commands tend to generate humongous commandlines for some targets, which may + // not fit on some remote execution machines. To work around this we will employ the help of + // a parameter file and pass any linker options through it. + List paramFileArgs = new ArrayList<>(); + List commandlineArgs = new ArrayList<>(); + extractArgumentsForParamFile(args, commandlineArgs, paramFileArgs); + + commandlineArgs.add("-Wl,@" + paramExecPath.getPathString()); + return Pair.of(commandlineArgs, paramFileArgs); + } + } + + boolean canBeSplit() { + if (!supportsParamFiles) { + return false; + } + switch (linkTargetType) { + // We currently can't split dynamic library links if they have interface outputs. That was + // probably an unintended side effect of the change that introduced interface outputs. + case DYNAMIC_LIBRARY: + return interfaceOutput == null; + case EXECUTABLE: + case STATIC_LIBRARY: + case PIC_STATIC_LIBRARY: + case ALWAYS_LINK_STATIC_LIBRARY: + case ALWAYS_LINK_PIC_STATIC_LIBRARY: + return true; + default: + return false; + } + } + + private static void extractArgumentsForParamFile(List args, List commandlineArgs, + List paramFileArgs) { + // Note, that it is not important that all linker arguments are extracted so that + // they can be moved into a parameter file, but the vast majority should. + commandlineArgs.add(args.get(0)); // gcc command, must not be moved! + int argsSize = args.size(); + for (int i = 1; i < argsSize; i++) { + String arg = args.get(i); + if (arg.equals("-Wl,-no-whole-archive")) { + paramFileArgs.add("-no-whole-archive"); + } else if (arg.equals("-Wl,-whole-archive")) { + paramFileArgs.add("-whole-archive"); + } else if (arg.equals("-Wl,--start-group")) { + paramFileArgs.add("--start-group"); + } else if (arg.equals("-Wl,--end-group")) { + paramFileArgs.add("--end-group"); + } else if (arg.equals("-Wl,--start-lib")) { + paramFileArgs.add("--start-lib"); + } else if (arg.equals("-Wl,--end-lib")) { + paramFileArgs.add("--end-lib"); + } else if (arg.equals("--incremental-unchanged")) { + paramFileArgs.add(arg); + } else if (arg.equals("--incremental-changed")) { + paramFileArgs.add(arg); + } else if (arg.charAt(0) == '-') { + if (arg.startsWith("-l")) { + paramFileArgs.add(arg); + } else { + // Anything else starting with a '-' can stay on the commandline. + commandlineArgs.add(arg); + if (arg.equals("-o")) { + // Special case for '-o': add the following argument as well - it is the output file! + commandlineArgs.add(args.get(++i)); + } + } + } else if (arg.endsWith(".a") || arg.endsWith(".lo") || arg.endsWith(".so") + || arg.endsWith(".ifso") || arg.endsWith(".o") + || CppFileTypes.VERSIONED_SHARED_LIBRARY.matches(arg)) { + // All objects of any kind go into the linker parameters. + paramFileArgs.add(arg); + } else { + // Everything that's left stays conservatively on the commandline. + commandlineArgs.add(arg); + } + } + } + + /** + * Returns a raw link command for the given link invocation, including both command and + * arguments (argv). After any further usage-specific processing, this can be passed to + * {@link #finalizeWithLinkstampCommands} to give the final command line. + * + * @return raw link command line. + */ + public List getRawLinkArgv() { + List argv = new ArrayList<>(); + switch (linkTargetType) { + case EXECUTABLE: + addCppArgv(argv); + break; + + case DYNAMIC_LIBRARY: + if (interfaceOutput != null) { + argv.add(configuration.getShExecutable().getPathString()); + argv.add("-c"); + argv.add("build_iface_so=\"$0\"; impl=\"$1\"; iface=\"$2\"; cmd=\"$3\"; shift 3; " + + "\"$cmd\" \"$@\" && \"$build_iface_so\" \"$impl\" \"$iface\""); + argv.add(interfaceSoBuilder.getExecPathString()); + argv.add(output.getExecPathString()); + argv.add(interfaceOutput.getExecPathString()); + } + addCppArgv(argv); + // -pie is not compatible with -shared and should be + // removed when the latter is part of the link command. Should we need to further + // distinguish between shared libraries and executables, we could add additional + // command line / CROSSTOOL flags that distinguish them. But as long as this is + // the only relevant use case we're just special-casing it here. + Iterables.removeIf(argv, Predicates.equalTo("-pie")); + break; + + case STATIC_LIBRARY: + case PIC_STATIC_LIBRARY: + case ALWAYS_LINK_STATIC_LIBRARY: + case ALWAYS_LINK_PIC_STATIC_LIBRARY: + // The static library link command follows this template: + // ar + argv.add(cppConfiguration.getArExecutable().getPathString()); + argv.addAll( + cppConfiguration.getArFlags(cppConfiguration.archiveType() == Link.ArchiveType.THIN)); + argv.add(output.getExecPathString()); + addInputFileLinkOptions(argv, /*needWholeArchive=*/false, + /*includeLinkopts=*/false); + break; + + default: + throw new IllegalArgumentException(); + } + + // Fission mode: debug info is in .dwo files instead of .o files. Inform the linker of this. + if (!linkTargetType.isStaticLibraryLink() && cppConfiguration.useFission()) { + argv.add("-Wl,--gdb-index"); + } + + return argv; + } + + @Override + public List arguments() { + return finalizeWithLinkstampCommands(getRawLinkArgv()); + } + + /** + * Takes a raw link command line and gives the final link command that will + * also first compile any linkstamps necessary. Elements of rawLinkArgv are + * shell-escaped. + * + * @param rawLinkArgv raw link command line + * + * @return final link command line suitable for execution + */ + public List finalizeWithLinkstampCommands(List rawLinkArgv) { + return addLinkstampingToCommand(getLinkstampCompileCommands(""), rawLinkArgv, true); + } + + /** + * Takes a raw link command line and gives the final link command that will also first compile any + * linkstamps necessary. Elements of rawLinkArgv are not shell-escaped. + * + * @param rawLinkArgv raw link command line + * @param outputPrefix prefix to add before the linkstamp outputs' exec paths + * + * @return final link command line suitable for execution + */ + public List finalizeAlreadyEscapedWithLinkstampCommands( + List rawLinkArgv, String outputPrefix) { + return addLinkstampingToCommand(getLinkstampCompileCommands(outputPrefix), rawLinkArgv, false); + } + + /** + * Adds linkstamp compilation to the (otherwise) fully specified link + * command if {@link #getLinkstamps} is non-empty. + * + *

Linkstamps were historically compiled implicitly as part of the link + * command, but implicit compilation doesn't guarantee consistent outputs. + * For example, the command "gcc input.o input.o foo/linkstamp.cc -o myapp" + * causes gcc to implicitly run "gcc foo/linkstamp.cc -o /tmp/ccEtJHDB.o", + * for some internally decided output path /tmp/ccEtJHDB.o, then add that path + * to the linker's command line options. The name of this path can change + * even between equivalently specified gcc invocations. + * + *

So now we explicitly compile these files in their own command + * invocations before running the link command, thus giving us direct + * control over the naming of their outputs. This method adds those extra + * steps as necessary. + * @param linkstampCommands individual linkstamp compilation commands + * @param linkCommand the complete list of link command arguments (after + * .params file compacting) for an invocation + * @param escapeArgs if true, linkCommand arguments are shell escaped. if + * false, arguments are returned as-is + * + * @return The original argument list if no linkstamps compilation commands + * are given, otherwise an expanded list that adds the linkstamp + * compilation commands and funnels their outputs into the link step. + * Note that these outputs only need to persist for the duration of + * the link step. + */ + private static List addLinkstampingToCommand( + List linkstampCommands, + List linkCommand, + boolean escapeArgs) { + if (linkstampCommands.isEmpty()) { + return linkCommand; + } else { + List batchCommand = Lists.newArrayListWithCapacity(3); + batchCommand.add("/bin/bash"); + batchCommand.add("-c"); + batchCommand.add( + Joiner.on(" && ").join(linkstampCommands) + " && " + + (escapeArgs + ? ShellEscaper.escapeJoinAll(linkCommand) + : Joiner.on(" ").join(linkCommand))); + return ImmutableList.copyOf(batchCommand); + } + } + + /** + * Computes, for each C++ source file in + * {@link #getLinkstamps}, the command necessary to compile + * that file such that the output is correctly fed into the link command. + * + *

As these options (as well as all others) are taken into account when + * computing the action key, they do not directly contain volatile build + * information to avoid unnecessary relinking. Instead this information is + * passed as an additional header generated by + * {@link com.google.devtools.build.lib.rules.cpp.WriteBuildInfoHeaderAction}. + * + * @param outputPrefix prefix to add before the linkstamp outputs' exec paths + * @return a list of shell-escaped compiler commmands, one for each entry + * in {@link #getLinkstamps} + */ + public List getLinkstampCompileCommands(String outputPrefix) { + if (linkstamps.isEmpty()) { + return ImmutableList.of(); + } + + String compilerCommand = cppConfiguration.getCppExecutable().getPathString(); + List commands = Lists.newArrayListWithCapacity(linkstamps.size()); + + for (Map.Entry linkstamp : linkstamps.entrySet()) { + List optionList = new ArrayList<>(); + + // Defines related to the build info are read from generated headers. + for (Artifact header : buildInfoHeaderArtifacts) { + optionList.add("-include"); + optionList.add(header.getExecPathString()); + } + + String labelReplacement = Matcher.quoteReplacement( + isSharedNativeLibrary() ? output.getExecPathString() : Label.print(owner.getLabel())); + String outputPathReplacement = Matcher.quoteReplacement( + output.getExecPathString()); + for (String option : linkstampCompileOptions) { + optionList.add(option + .replaceAll(Pattern.quote("${LABEL}"), labelReplacement) + .replaceAll(Pattern.quote("${OUTPUT_PATH}"), outputPathReplacement)); + } + + optionList.add("-DGPLATFORM=\"" + cppConfiguration + "\""); + + // Needed to find headers included from linkstamps. + optionList.add("-I."); + + // Add sysroot. + PathFragment sysroot = cppConfiguration.getSysroot(); + if (sysroot != null) { + optionList.add("--sysroot=" + sysroot.getPathString()); + } + + // Add toolchain compiler options. + optionList.addAll(cppConfiguration.getCompilerOptions(features)); + optionList.addAll(cppConfiguration.getCOptions()); + optionList.addAll(cppConfiguration.getUnfilteredCompilerOptions(features)); + + // For dynamic libraries, produce position independent code. + if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + && cppConfiguration.toolchainNeedsPic()) { + optionList.add("-fPIC"); + } + + // Stamp FDO builds with FDO subtype string + String fdoBuildStamp = CppHelper.getFdoBuildStamp(cppConfiguration); + if (fdoBuildStamp != null) { + optionList.add("-D" + CppConfiguration.FDO_STAMP_MACRO + "=\"" + fdoBuildStamp + "\""); + } + + // Add the compilation target. + optionList.add("-c"); + optionList.add(linkstamp.getKey().getExecPathString()); + + // Assemble the final command, exempting outputPrefix from shell escaping. + commands.add(compilerCommand + " " + + ShellEscaper.escapeJoinAll(optionList) + + " -o " + + outputPrefix + + ShellEscaper.escapeString(linkstamp.getValue().getExecPathString())); + } + + return commands; + } + + /** + * Determine the arguments to pass to the C++ compiler when linking. + * Add them to the {@code argv} parameter. + */ + private void addCppArgv(List argv) { + argv.add(cppConfiguration.getCppExecutable().getPathString()); + + // When using gold to link an executable, output the number of used and unused symbols. + if (symbolCountsOutput != null) { + argv.add("-Wl,--print-symbol-counts=" + symbolCountsOutput.getExecPathString()); + } + + if (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY) { + argv.add("-shared"); + } + + // Add the outputs of any associated linkstamp compilations. + for (Artifact linkstampOutput : linkstamps.values()) { + argv.add(linkstampOutput.getExecPathString()); + } + + boolean fullyStatic = (linkStaticness == LinkStaticness.FULLY_STATIC); + boolean mostlyStatic = (linkStaticness == LinkStaticness.MOSTLY_STATIC); + boolean sharedLinkopts = + linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + || linkopts.contains("-shared") + || cppConfiguration.getLinkOptions().contains("-shared"); + + if (output != null) { + argv.add("-o"); + String execpath = output.getExecPathString(); + if (mostlyStatic + && linkTargetType == LinkTargetType.EXECUTABLE + && cppConfiguration.skipStaticOutputs()) { + // Linked binary goes to /dev/null; bogus dependency info in its place. + Collections.addAll(argv, "/dev/null", "-MMD", "-MF", execpath); // thanks Ambrose + } else { + argv.add(execpath); + } + } + + addInputFileLinkOptions(argv, needWholeArchive, /*includeLinkopts=*/true); + + // Extra toolchain link options based on the output's link staticness. + if (fullyStatic) { + argv.addAll(cppConfiguration.getFullyStaticLinkOptions(features, sharedLinkopts)); + } else if (mostlyStatic) { + argv.addAll(cppConfiguration.getMostlyStaticLinkOptions(features, sharedLinkopts)); + } else { + argv.addAll(cppConfiguration.getDynamicLinkOptions(features, sharedLinkopts)); + } + + // Extra test-specific link options. + if (useTestOnlyFlags) { + argv.addAll(cppConfiguration.getTestOnlyLinkOptions()); + } + + if (configuration.isCodeCoverageEnabled()) { + argv.add("-lgcov"); + } + + if (linkTargetType == LinkTargetType.EXECUTABLE && cppConfiguration.forcePic()) { + argv.add("-pie"); + } + + argv.addAll(cppConfiguration.getLinkOptions()); + argv.addAll(cppConfiguration.getFdoSupport().getLinkOptions()); + } + + private static boolean isDynamicLibrary(LinkerInput linkInput) { + Artifact libraryArtifact = linkInput.getArtifact(); + String name = libraryArtifact.getFilename(); + return Link.SHARED_LIBRARY_FILETYPES.matches(name) && name.startsWith("lib"); + } + + private boolean isSharedNativeLibrary() { + return nativeDeps && cppConfiguration.shareNativeDeps(); + } + + /** + * When linking a shared library fully or mostly static then we need to link in + * *all* dependent files, not just what the shared library needs for its own + * code. This is done by wrapping all objects/libraries with + * -Wl,-whole-archive and -Wl,-no-whole-archive. For this case the + * globalNeedWholeArchive parameter must be set to true. Otherwise only + * library objects (.lo) need to be wrapped with -Wl,-whole-archive and + * -Wl,-no-whole-archive. + */ + private void addInputFileLinkOptions(List argv, boolean globalNeedWholeArchive, + boolean includeLinkopts) { + // The Apple ld doesn't support -whole-archive/-no-whole-archive. It + // does have -all_load/-noall_load, but -all_load is a global setting + // that affects all subsequent files, and -noall_load is simply ignored. + // TODO(bazel-team): Not sure what the implications of this are, other than + // bloated binaries. + boolean macosx = cppConfiguration.getTargetLibc().equals("macosx"); + if (globalNeedWholeArchive) { + argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive"); + } + + // Used to collect -L and -Wl,-rpath options, ensuring that each used only once. + Set libOpts = new LinkedHashSet<>(); + + // List of command line parameters to link input files (either directly or using -l). + List linkerInputs = new ArrayList<>(); + + // List of command line parameters that need to be placed *outside* of + // --whole-archive ... --no-whole-archive. + List noWholeArchiveInputs = new ArrayList<>(); + + PathFragment solibDir = configuration.getBinDirectory().getExecPath() + .getRelative(cppConfiguration.getSolibDirectory()); + String runtimeSolibName = runtimeSolibDir != null ? runtimeSolibDir.getBaseName() : null; + boolean runtimeRpath = runtimeSolibDir != null + && (linkTargetType == LinkTargetType.DYNAMIC_LIBRARY + || (linkTargetType == LinkTargetType.EXECUTABLE + && linkStaticness == LinkStaticness.DYNAMIC)); + + String rpathRoot = null; + List runtimeRpathEntries = new ArrayList<>(); + + if (output != null) { + String origin = + useTestOnlyFlags && cppConfiguration.supportsExecOrigin() ? "$EXEC_ORIGIN/" : "$ORIGIN/"; + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + runtimeSolibName + "/"); + } + + // Calculate the correct relative value for the "-rpath" link option (which sets + // the search path for finding shared libraries). + if (isSharedNativeLibrary()) { + // For shared native libraries, special symlinking is applied to ensure C++ + // runtimes are available under $ORIGIN/_solib_[arch]. So we set the RPATH to find + // them. + // + // Note that we have to do this because $ORIGIN points to different paths for + // different targets. In other words, blaze-bin/d1/d2/d3/a_shareddeps.so and + // blaze-bin/d4/b_shareddeps.so have different path depths. The first could + // reference a standard blaze-bin/_solib_[arch] via $ORIGIN/../../../_solib[arch], + // and the second could use $ORIGIN/../_solib_[arch]. But since this is a shared + // artifact, both are symlinks to the same place, so + // there's no *one* RPATH setting that fits all targets involved in the sharing. + rpathRoot = "-Wl,-rpath," + origin + ":" + + origin + cppConfiguration.getSolibDirectory() + "/"; + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/"); + } + } else { + // For all other links, calculate the relative path from the output file to _solib_[arch] + // (the directory where all shared libraries are stored, which resides under the blaze-bin + // directory. In other words, given blaze-bin/my/package/binary, rpathRoot would be + // "../../_solib_[arch]". + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1) + + runtimeSolibName + "/"); + } + + rpathRoot = "-Wl,-rpath," + + origin + Strings.repeat("../", output.getRootRelativePath().segmentCount() - 1) + + cppConfiguration.getSolibDirectory() + "/"; + + if (nativeDeps) { + // We also retain the $ORIGIN/ path to solibs that are in _solib_, as opposed to + // the package directory) + if (runtimeRpath) { + runtimeRpathEntries.add("-Wl,-rpath," + origin + "../" + runtimeSolibName + "/"); + } + rpathRoot += ":" + origin; + } + } + } + + boolean includeSolibDir = false; + + for (LinkerInput input : getLinkerInputs()) { + if (isDynamicLibrary(input)) { + PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory(); + Preconditions.checkState( + libDir.startsWith(solibDir), + "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir); + if (libDir.equals(solibDir)) { + includeSolibDir = true; + } + addDynamicInputLinkOptions(input, linkerInputs, libOpts, solibDir, rpathRoot); + } else { + addStaticInputLinkOptions(input, linkerInputs); + } + } + + boolean includeRuntimeSolibDir = false; + + for (LinkerInput input : runtimeInputs) { + List optionsList = globalNeedWholeArchive + ? noWholeArchiveInputs + : linkerInputs; + + if (isDynamicLibrary(input)) { + PathFragment libDir = input.getArtifact().getExecPath().getParentDirectory(); + Preconditions.checkState(runtimeSolibDir != null && libDir.equals(runtimeSolibDir), + "Artifact '%s' is not under directory '%s'.", input.getArtifact(), solibDir); + includeRuntimeSolibDir = true; + addDynamicInputLinkOptions(input, optionsList, libOpts, solibDir, rpathRoot); + } else { + addStaticInputLinkOptions(input, optionsList); + } + } + + // rpath ordering matters for performance; first add the one where most libraries are found. + if (includeSolibDir && rpathRoot != null) { + argv.add(rpathRoot); + } + if (includeRuntimeSolibDir) { + argv.addAll(runtimeRpathEntries); + } + argv.addAll(libOpts); + + // Need to wrap static libraries with whole-archive option + for (String option : linkerInputs) { + if (!globalNeedWholeArchive && Link.LINK_LIBRARY_FILETYPES.matches(option)) { + argv.add(macosx ? "-Wl,-all_load" : "-Wl,-whole-archive"); + argv.add(option); + argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive"); + } else { + argv.add(option); + } + } + + if (globalNeedWholeArchive) { + argv.add(macosx ? "-Wl,-noall_load" : "-Wl,-no-whole-archive"); + argv.addAll(noWholeArchiveInputs); + } + + if (includeLinkopts) { + /* + * For backwards compatibility, linkopts come _after_ inputFiles. + * This is needed to allow linkopts to contain libraries and + * positional library-related options such as + * -Wl,--begin-group -lfoo -lbar -Wl,--end-group + * or + * -Wl,--as-needed -lfoo -Wl,--no-as-needed + * + * As for the relative order of the three different flavours of linkopts + * (global defaults, per-target linkopts, and command-line linkopts), + * we have no idea what the right order should be, or if anyone cares. + */ + argv.addAll(linkopts); + } + } + + /** + * Adds command-line options for a dynamic library input file into + * options and libOpts. + */ + private void addDynamicInputLinkOptions(LinkerInput input, List options, + Set libOpts, PathFragment solibDir, String rpathRoot) { + Preconditions.checkState(isDynamicLibrary(input)); + Preconditions.checkState( + !Link.useStartEndLib(input, cppConfiguration.archiveType())); + + Artifact inputArtifact = input.getArtifact(); + PathFragment libDir = inputArtifact.getExecPath().getParentDirectory(); + if (rpathRoot != null + && !libDir.equals(solibDir) + && (runtimeSolibDir == null || !runtimeSolibDir.equals(libDir))) { + String dotdots = ""; + PathFragment commonParent = solibDir; + while (!libDir.startsWith(commonParent)) { + dotdots += "../"; + commonParent = commonParent.getParentDirectory(); + } + + libOpts.add(rpathRoot + dotdots + libDir.relativeTo(commonParent).getPathString()); + } + + libOpts.add("-L" + inputArtifact.getExecPath().getParentDirectory().getPathString()); + + String name = inputArtifact.getFilename(); + if (CppFileTypes.SHARED_LIBRARY.matches(name)) { + String libName = name.replaceAll("(^lib|\\.so$)", ""); + options.add("-l" + libName); + } else { + // Interface shared objects have a non-standard extension + // that the linker won't be able to find. So use the + // filename directly rather than a -l option. Since the + // library has an SONAME attribute, this will work fine. + options.add(inputArtifact.getExecPathString()); + } + } + + /** + * Adds command-line options for a static library or non-library input + * into options. + */ + private void addStaticInputLinkOptions(LinkerInput input, List options) { + Preconditions.checkState(!isDynamicLibrary(input)); + + // start-lib/end-lib library: adds its input object files. + if (Link.useStartEndLib(input, cppConfiguration.archiveType())) { + Iterable archiveMembers = input.getObjectFiles(); + if (!Iterables.isEmpty(archiveMembers)) { + options.add("-Wl,--start-lib"); + for (Artifact member : archiveMembers) { + options.add(member.getExecPathString()); + } + options.add("-Wl,--end-lib"); + } + // For anything else, add the input directly. + } else { + Artifact inputArtifact = input.getArtifact(); + if (input.isFake()) { + options.add(Link.FAKE_OBJECT_PREFIX + inputArtifact.getExecPathString()); + } else { + options.add(inputArtifact.getExecPathString()); + } + } + } + + /** + * A builder for a {@link LinkCommandLine}. + */ + public static final class Builder { + // TODO(bazel-team): Pass this in instead of having it here. Maybe move to cc_toolchain. + private static final ImmutableList DEFAULT_LINKSTAMP_OPTIONS = ImmutableList.of( + // G3_VERSION_INFO and G3_TARGET_NAME are C string literals that normally + // contain the label of the target being linked. However, they are set + // differently when using shared native deps. In that case, a single .so file + // is shared by multiple targets, and its contents cannot depend on which + // target(s) were specified on the command line. So in that case we have + // to use the (obscure) name of the .so file instead, or more precisely + // the path of the .so file relative to the workspace root. + "-DG3_VERSION_INFO=\"${LABEL}\"", + "-DG3_TARGET_NAME=\"${LABEL}\"", + + // G3_BUILD_TARGET is a C string literal containing the output of this + // link. (An undocumented and untested invariant is that G3_BUILD_TARGET is the location of + // the executable, either absolutely, or relative to the directory part of BUILD_INFO.) + "-DG3_BUILD_TARGET=\"${OUTPUT_PATH}\""); + + private final BuildConfiguration configuration; + private final ActionOwner owner; + + @Nullable private Artifact output; + @Nullable private Artifact interfaceOutput; + @Nullable private Artifact symbolCountsOutput; + private ImmutableList buildInfoHeaderArtifacts = ImmutableList.of(); + private Iterable linkerInputs = ImmutableList.of(); + private Iterable runtimeInputs = ImmutableList.of(); + @Nullable private LinkTargetType linkTargetType; + private LinkStaticness linkStaticness = LinkStaticness.FULLY_STATIC; + private ImmutableList linkopts = ImmutableList.of(); + private ImmutableSet features = ImmutableSet.of(); + private ImmutableMap linkstamps = ImmutableMap.of(); + private List linkstampCompileOptions = new ArrayList<>(); + @Nullable private PathFragment runtimeSolibDir; + private boolean nativeDeps; + private boolean useTestOnlyFlags; + private boolean needWholeArchive; + private boolean supportsParamFiles; + @Nullable private Artifact interfaceSoBuilder; + + public Builder(BuildConfiguration configuration, ActionOwner owner) { + this.configuration = configuration; + this.owner = owner; + } + + public Builder(RuleContext ruleContext) { + this(ruleContext.getConfiguration(), ruleContext.getActionOwner()); + } + + public LinkCommandLine build() { + ImmutableList actualLinkstampCompileOptions; + if (linkstampCompileOptions.isEmpty()) { + actualLinkstampCompileOptions = DEFAULT_LINKSTAMP_OPTIONS; + } else { + actualLinkstampCompileOptions = ImmutableList.copyOf( + Iterables.concat(DEFAULT_LINKSTAMP_OPTIONS, linkstampCompileOptions)); + } + return new LinkCommandLine(configuration, owner, output, interfaceOutput, + symbolCountsOutput, buildInfoHeaderArtifacts, linkerInputs, runtimeInputs, linkTargetType, + linkStaticness, linkopts, features, linkstamps, actualLinkstampCompileOptions, + runtimeSolibDir, nativeDeps, useTestOnlyFlags, needWholeArchive, supportsParamFiles, + interfaceSoBuilder); + } + + /** + * Sets the type of the link. It is an error to try to set this to {@link + * LinkTargetType#INTERFACE_DYNAMIC_LIBRARY}. Note that all the static target types (see {@link + * LinkTargetType#isStaticLibraryLink}) are equivalent, and there is no check that the output + * artifact matches the target type extension. + */ + public Builder setLinkTargetType(LinkTargetType linkTargetType) { + Preconditions.checkArgument(linkTargetType != LinkTargetType.INTERFACE_DYNAMIC_LIBRARY); + this.linkTargetType = linkTargetType; + return this; + } + + /** + * Sets the primary output artifact. This must be called before calling {@link #build}. + */ + public Builder setOutput(Artifact output) { + this.output = output; + return this; + } + + /** + * Sets a list of linker inputs. These get turned into linker options depending on the + * staticness and the target type. This call makes an immutable copy of the inputs, if the + * provided Iterable isn't already immutable (see {@link CollectionUtils#makeImmutable}). + */ + public Builder setLinkerInputs(Iterable linkerInputs) { + this.linkerInputs = CollectionUtils.makeImmutable(linkerInputs); + return this; + } + + public Builder setRuntimeInputs(ImmutableList runtimeInputs) { + this.runtimeInputs = runtimeInputs; + return this; + } + + /** + * Sets the additional interface output artifact, which is only used for dynamic libraries. The + * {@link #build} method throws an exception if the target type is not {@link + * LinkTargetType#DYNAMIC_LIBRARY}. + */ + public Builder setInterfaceOutput(Artifact interfaceOutput) { + this.interfaceOutput = interfaceOutput; + return this; + } + + /** + * Sets an additional output artifact that contains symbol counts. The {@link #build} method + * throws an exception if this is non-null for a static link (see + * {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setSymbolCountsOutput(Artifact symbolCountsOutput) { + this.symbolCountsOutput = symbolCountsOutput; + return this; + } + + /** + * Sets the linker options. These are passed to the linker in addition to the other linker + * options like linker inputs, symbol count options, etc. The {@link #build} method + * throws an exception if the linker options are non-empty for a static link (see {@link + * LinkTargetType#isStaticLibraryLink}). + */ + public Builder setLinkopts(ImmutableList linkopts) { + this.linkopts = linkopts; + return this; + } + + /** + * Sets how static the link is supposed to be. For static target types (see {@link + * LinkTargetType#isStaticLibraryLink}), the {@link #build} method throws an exception if this + * is not {@link LinkStaticness#FULLY_STATIC}. The default setting is {@link + * LinkStaticness#FULLY_STATIC}. + */ + public Builder setLinkStaticness(LinkStaticness linkStaticness) { + this.linkStaticness = linkStaticness; + return this; + } + + /** + * Sets the binary that should be used to create the interface output for a dynamic library. + * This is ignored unless the target type is {@link LinkTargetType#DYNAMIC_LIBRARY} and an + * interface output artifact is specified. + */ + public Builder setInterfaceSoBuilder(Artifact interfaceSoBuilder) { + this.interfaceSoBuilder = interfaceSoBuilder; + return this; + } + + /** + * Sets the linkstamps. Linkstamps are additional C++ source files that are compiled as part of + * the link command. The {@link #build} method throws an exception if the linkstamps are + * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setLinkstamps(ImmutableMap linkstamps) { + this.linkstamps = linkstamps; + return this; + } + + /** + * Adds the given C++ compiler options to the list of options passed to the linkstamp + * compilation. + */ + public Builder addLinkstampCompileOptions(List linkstampCompileOptions) { + this.linkstampCompileOptions.addAll(linkstampCompileOptions); + return this; + } + + /** + * The build info header artifacts are generated header files that are used for link stamping. + * The {@link #build} method throws an exception if the build info header artifacts are + * non-empty for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setBuildInfoHeaderArtifacts(ImmutableList buildInfoHeaderArtifacts) { + this.buildInfoHeaderArtifacts = buildInfoHeaderArtifacts; + return this; + } + + /** + * Sets the features enabled for the rule. + */ + public Builder setFeatures(ImmutableSet features) { + this.features = features; + return this; + } + + /** + * Sets the directory of the dynamic runtime libraries, which is added to the rpath. The {@link + * #build} method throws an exception if the runtime dir is non-null for a static link (see + * {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setRuntimeSolibDir(PathFragment runtimeSolibDir) { + this.runtimeSolibDir = runtimeSolibDir; + return this; + } + + /** + * Whether the resulting library is intended to be used as a native library from another + * programming language. This influences the rpath. The {@link #build} method throws an + * exception if this is true for a static link (see {@link LinkTargetType#isStaticLibraryLink}). + */ + public Builder setNativeDeps(boolean nativeDeps) { + this.nativeDeps = nativeDeps; + return this; + } + + /** + * Sets whether to use test-specific linker flags, e.g. {@code $EXEC_ORIGIN} instead of + * {@code $ORIGIN} in the rpath or lazy binding. + */ + public Builder setUseTestOnlyFlags(boolean useTestOnlyFlags) { + this.useTestOnlyFlags = useTestOnlyFlags; + return this; + } + + public Builder setNeedWholeArchive(boolean needWholeArchive) { + this.needWholeArchive = needWholeArchive; + return this; + } + + public Builder setSupportsParamFiles(boolean supportsParamFiles) { + this.supportsParamFiles = supportsParamFiles; + return this; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java new file mode 100644 index 0000000000..4f7673ecb2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkStrategy.java @@ -0,0 +1,35 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +/** + * A strategy for executing {@link CppLinkAction}s. + * + *

The linker commands, e.g. "ar", are not necessary functional, i.e. + * they may mutate the output file rather than overwriting it. + * To avoid this, we need to delete the output file before invoking the + * command. That must be done by the classes that extend this class. + */ +public abstract class LinkStrategy implements CppLinkActionContext { + public LinkStrategy() { + } + + /** The strategy name, preferably suitable for passing to --link_strategy. */ + public abstract String linkStrategyName(); + + @Override + public String strategyLocality(CppLinkAction execOwner) { + return linkStrategyName(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java new file mode 100644 index 0000000000..15a8b90c7f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInput.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.Artifact; + +/** + * Something that appears on the command line of the linker. Since we sometimes expand archive + * files to their constituent object files, we need to keep information whether a certain file + * contains embedded objects and if so, the list of the object files themselves. + */ +public interface LinkerInput { + /** + * Returns the artifact that is the input of the linker. + */ + Artifact getArtifact(); + + /** + * Returns the original library to link. If this library is a solib symlink, returns the + * artifact the symlink points to, otherwise, the library itself. + */ + Artifact getOriginalLibraryArtifact(); + + /** + * Whether the input artifact contains object files or is opaque. + */ + boolean containsObjectFiles(); + + /** + * Returns whether the input artifact is a fake object file or not. + */ + boolean isFake(); + + /** + * Return the list of object files included in the input artifact, if there are any. It is + * legal to call this only when {@link #containsObjectFiles()} returns true. + */ + Iterable getObjectFiles(); +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java new file mode 100644 index 0000000000..24120ce44f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkerInputs.java @@ -0,0 +1,353 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.collect.CollectionUtils; +import com.google.devtools.build.lib.concurrent.ThreadSafety; + +/** + * Factory for creating new {@link LinkerInput} objects. + */ +public abstract class LinkerInputs { + /** + * An opaque linker input that is not a library, for example a linker script or an individual + * object file. + */ + @ThreadSafety.Immutable + public static class SimpleLinkerInput implements LinkerInput { + private final Artifact artifact; + + public SimpleLinkerInput(Artifact artifact) { + this.artifact = Preconditions.checkNotNull(artifact); + } + + @Override + public Artifact getArtifact() { + return artifact; + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return artifact; + } + + @Override + public boolean containsObjectFiles() { + return false; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable getObjectFiles() { + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof SimpleLinkerInput)) { + return false; + } + + SimpleLinkerInput other = (SimpleLinkerInput) that; + return artifact.equals(other.artifact) && isFake() == other.isFake(); + } + + @Override + public int hashCode() { + return artifact.hashCode(); + } + + @Override + public String toString() { + return "SimpleLinkerInput(" + artifact.toString() + ")"; + } + } + + /** + * A linker input that is a fake object file generated by cc_fake_binary. The contained + * artifact must be an object file. + */ + @ThreadSafety.Immutable + private static class FakeLinkerInput extends SimpleLinkerInput { + private FakeLinkerInput(Artifact artifact) { + super(artifact); + Preconditions.checkState(Link.OBJECT_FILETYPES.matches(artifact.getFilename())); + } + + @Override + public boolean isFake() { + return true; + } + } + + /** + * A library the user can link to. This is different from a simple linker input in that it also + * has a library identifier. + */ + public interface LibraryToLink extends LinkerInput { + /** + * Returns whether the library is a solib symlink. + */ + boolean isSolibSymlink(); + } + + /** + * This class represents a solib library symlink. Its library identifier is inherited from + * the library that it links to. + */ + @ThreadSafety.Immutable + public static class SolibLibraryToLink implements LibraryToLink { + private final Artifact solibSymlinkArtifact; + private final Artifact libraryArtifact; + + private SolibLibraryToLink(Artifact solibSymlinkArtifact, Artifact libraryArtifact) { + this.solibSymlinkArtifact = Preconditions.checkNotNull(solibSymlinkArtifact); + this.libraryArtifact = libraryArtifact; + } + + @Override + public String toString() { + return String.format("SolibLibraryToLink(%s -> %s", + solibSymlinkArtifact.toString(), libraryArtifact.toString()); + } + + @Override + public Artifact getArtifact() { + return solibSymlinkArtifact; + } + + @Override + public boolean containsObjectFiles() { + return false; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable getObjectFiles() { + throw new IllegalStateException(); + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return libraryArtifact; + } + + @Override + public boolean isSolibSymlink() { + return true; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof SolibLibraryToLink)) { + return false; + } + + SolibLibraryToLink thatSolib = (SolibLibraryToLink) that; + return + solibSymlinkArtifact.equals(thatSolib.solibSymlinkArtifact) && + libraryArtifact.equals(thatSolib.libraryArtifact); + } + + @Override + public int hashCode() { + return solibSymlinkArtifact.hashCode(); + } + } + + /** + * This class represents a library that may contain object files. + */ + @ThreadSafety.Immutable + private static class CompoundLibraryToLink implements LibraryToLink { + private final Artifact libraryArtifact; + private final Iterable objectFiles; + + private CompoundLibraryToLink(Artifact libraryArtifact, Iterable objectFiles) { + this.libraryArtifact = Preconditions.checkNotNull(libraryArtifact); + this.objectFiles = objectFiles == null ? null : CollectionUtils.makeImmutable(objectFiles); + } + + @Override + public String toString() { + return String.format("CompoundLibraryToLink(%s)", libraryArtifact.toString()); + } + + @Override + public Artifact getArtifact() { + return libraryArtifact; + } + + @Override + public Artifact getOriginalLibraryArtifact() { + return libraryArtifact; + } + + @Override + public boolean containsObjectFiles() { + return objectFiles != null; + } + + @Override + public boolean isFake() { + return false; + } + + @Override + public Iterable getObjectFiles() { + Preconditions.checkNotNull(objectFiles); + return objectFiles; + } + + @Override + public boolean equals(Object that) { + if (this == that) { + return true; + } + + if (!(that instanceof CompoundLibraryToLink)) { + return false; + } + + return libraryArtifact.equals(((CompoundLibraryToLink) that).libraryArtifact); + } + + @Override + public int hashCode() { + return libraryArtifact.hashCode(); + } + + @Override + public boolean isSolibSymlink() { + return false; + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // Public factory constructors: + ////////////////////////////////////////////////////////////////////////////////////// + + /** + * Creates linker input objects for non-library files. + */ + public static Iterable simpleLinkerInputs(Iterable input) { + return Iterables.transform(input, new Function() { + @Override + public LinkerInput apply(Artifact artifact) { + return simpleLinkerInput(artifact); + } + }); + } + + /** + * Creates a linker input for which we do not know what objects files it consists of. + */ + public static LinkerInput simpleLinkerInput(Artifact artifact) { + // This precondition check was in place and *most* of the tests passed with them; the only + // exception is when you mention a generated .a file in the srcs of a cc_* rule. + // Preconditions.checkArgument(!ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType())); + return new SimpleLinkerInput(artifact); + } + + /** + * Creates a fake linker input. The artifact must be an object file. + */ + public static LinkerInput fakeLinkerInput(Artifact artifact) { + return new FakeLinkerInput(artifact); + } + + /** + * Creates input libraries for which we do not know what objects files it consists of. + */ + public static Iterable opaqueLibrariesToLink(Iterable input) { + return Iterables.transform(input, new Function() { + @Override + public LibraryToLink apply(Artifact artifact) { + return opaqueLibraryToLink(artifact); + } + }); + } + + /** + * Creates a solib library symlink from the given artifact. + */ + public static LibraryToLink solibLibraryToLink(Artifact solibSymlink, Artifact original) { + return new SolibLibraryToLink(solibSymlink, original); + } + + /** + * Creates an input library for which we do not know what objects files it consists of. + */ + public static LibraryToLink opaqueLibraryToLink(Artifact artifact) { + // This precondition check was in place and *most* of the tests passed with them; the only + // exception is when you mention a generated .a file in the srcs of a cc_* rule. + // It was very useful for proving that this actually works, though. + // Preconditions.checkArgument( + // !(artifact.getGeneratingAction() instanceof CppLinkAction) || + // !Link.ARCHIVE_LIBRARY_FILETYPES.contains(artifact.getFileType())); + return new CompoundLibraryToLink(artifact, null); + } + + /** + * Creates a library to link with the specified object files. + */ + public static LibraryToLink newInputLibrary(Artifact library, Iterable objectFiles) { + return new CompoundLibraryToLink(library, objectFiles); + } + + private static final Function LIBRARY_TO_NON_SOLIB = + new Function() { + @Override + public Artifact apply(LibraryToLink input) { + return input.getOriginalLibraryArtifact(); + } + }; + + public static Iterable toNonSolibArtifacts(Iterable libraries) { + return Iterables.transform(libraries, LIBRARY_TO_NON_SOLIB); + } + + /** + * Returns the linker input artifacts from a collection of {@link LinkerInput} objects. + */ + public static Iterable toLibraryArtifacts(Iterable artifacts) { + return Iterables.transform(artifacts, new Function() { + @Override + public Artifact apply(LinkerInput input) { + return input.getArtifact(); + } + }); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java new file mode 100644 index 0000000000..8018108bea --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LinkingMode.java @@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +/** + * This class represents the different linking modes. + */ +public enum LinkingMode { + + /** + * Everything is linked statically; e.g. {@code gcc -static x.o libfoo.a + * libbar.a -lm}. Specified by {@code -static} in linkopts. + */ + FULLY_STATIC, + + /** + * Link binaries statically except for system libraries + * e.g. {@code gcc x.o libfoo.a libbar.a -lm}. Specified by {@code linkstatic=1}. + * + *

This mode applies to executables. + */ + MOSTLY_STATIC, + + /** + * Same as MOSTLY_STATIC, but for shared libraries. + */ + MOSTLY_STATIC_LIBRARIES, + + /** + * All libraries are linked dynamically (if a dynamic version is available), + * e.g. {@code gcc x.o libfoo.so libbar.so -lm}. Specified by {@code + * linkstatic=0}. + */ + DYNAMIC; +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java new file mode 100644 index 0000000000..a9ffea8f6b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LipoContextProvider.java @@ -0,0 +1,58 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +import java.util.Map; + +/** + * Provides LIPO context information to the LIPO-enabled target configuration. + * + *

This is a rollup of the data collected in the LIPO context collector configuration. + * Each target in the LIPO context collector configuration has a {@link TransitiveLipoInfoProvider} + * which is used to transitively collect the data, then the {@code cc_binary} that is referred to + * in {@code --lipo_context} puts the collected data into {@link LipoContextProvider}, of which + * there is only one in any given build. + */ +@Immutable +public final class LipoContextProvider implements TransitiveInfoProvider { + + private final CppCompilationContext cppCompilationContext; + + private final ImmutableMap includeScannables; + public LipoContextProvider(CppCompilationContext cppCompilationContext, + Map scannables) { + this.cppCompilationContext = cppCompilationContext; + this.includeScannables = ImmutableMap.copyOf(scannables); + } + + /** + * Returns merged compilation context for the whole LIPO subtree. + */ + public CppCompilationContext getLipoContext() { + return cppCompilationContext; + } + + /** + * Returns the map from source artifact to the include scannable object representing + * the corresponding FDO source input file. + */ + public ImmutableMap getIncludeScannables() { + return includeScannables; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java new file mode 100644 index 0000000000..80ee23d70b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalGccStrategy.java @@ -0,0 +1,96 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.common.options.OptionsClassProvider; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Run gcc locally by delegating to spawn. + */ +@ExecutionStrategy(name = { "local" }, + contextType = CppCompileActionContext.class) +public class LocalGccStrategy implements CppCompileActionContext { + private static final Reply CANNED_REPLY = new Reply() { + @Override + public byte[] getContents() { + throw new IllegalStateException("Remotely computed data requested for local action"); + } + }; + + public LocalGccStrategy(OptionsClassProvider options) { + } + + @Override + public String strategyLocality() { + return "local"; + } + + public static void updateEnv(CppCompileAction action, Map env) { + // We cannot locally execute an action that does not expect to output a .d file, since we would + // have no way to tell what files that it included were used during compilation. + env.put("INTERCEPT_LOCALLY_EXECUTABLE", action.getDotdFile().artifact() == null ? "0" : "1"); + } + + @Override + public boolean needsIncludeScanning() { + return false; + } + + @Override + public Collection findAdditionalInputs(CppCompileAction action, + ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { + return ImmutableList.of(); + } + + @Override + public CppCompileActionContext.Reply execWithReply( + CppCompileAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, InterruptedException { + Map env = new HashMap<>(); + env.putAll(action.getEnvironment()); + updateEnv(action, env); + actionExecutionContext.getExecutor().getSpawnActionContext(action.getMnemonic()) + .exec(new BaseSpawn.Local(action.getArgv(), env, action), + actionExecutionContext); + return null; + } + + @Override + public ResourceSet estimateResourceConsumption(CppCompileAction action) { + return action.estimateResourceConsumptionLocal(); + } + + @Override + public Collection getScannedIncludeFiles( + CppCompileAction action, ActionExecutionContext actionExecutionContext) { + return ImmutableList.of(); + } + + @Override + public Reply getReplyFromException(ExecException e, CppCompileAction action) { + return CANNED_REPLY; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java new file mode 100644 index 0000000000..3e7c863f42 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/LocalLinkStrategy.java @@ -0,0 +1,62 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.BaseSpawn; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; + +import java.util.List; + +/** + * A link strategy that runs the linking step on the local host. + * + *

The set of input files necessary to successfully complete the link is the middleman-expanded + * set of the action's dependency inputs (which includes crosstool and libc dependencies, as + * defined by {@link com.google.devtools.build.lib.rules.cpp.CppHelper#getCrosstoolInputsForLink + * CppHelper.getCrosstoolInputsForLink}). + */ +@ExecutionStrategy(contextType = CppLinkActionContext.class, name = { "local" }) +public final class LocalLinkStrategy extends LinkStrategy { + + public LocalLinkStrategy() { + } + + @Override + public void exec(CppLinkAction action, ActionExecutionContext actionExecutionContext) + throws ExecException, ActionExecutionException, InterruptedException { + Executor executor = actionExecutionContext.getExecutor(); + List argv = + action.prepareCommandLine(executor.getExecRoot(), null); + executor.getSpawnActionContext(action.getMnemonic()).exec( + new BaseSpawn.Local(argv, ImmutableMap.of(), action), + actionExecutionContext); + } + + @Override + public String linkStrategyName() { + return "local"; + } + + @Override + public ResourceSet estimateResourceConsumption(CppLinkAction action) { + return action.estimateResourceConsumptionLocal(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java new file mode 100644 index 0000000000..87a07123c6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/RemoteIncludeExtractor.java @@ -0,0 +1,52 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.rules.cpp.IncludeParser.Inclusion; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.IOException; +import java.util.Collection; + +/** Parses a single file for its (direct) includes, possibly using a remote service. */ +public interface RemoteIncludeExtractor extends ActionContext { + /** Result of checking if this object should be used to parse a given file. */ + interface RemoteParseData { + boolean shouldParseRemotely(); + } + + /** + * Returns whether to use this object to parse the given file for includes. The returned data + * should be passed to {@link #extractInclusions} to direct its behavior. + */ + RemoteParseData shouldParseRemotely(Path file); + + /** + * Extracts all inclusions from a given source file, possibly using a remote service. + * + * @param file the file from which to parse and extract inclusions. + * @param actionExecutionContext services in the scope of the action. Like the Err/Out stream + * outputs. + * @param remoteParseData the returned value of {@link #shouldParseRemotely}. + * @return a collection of inclusions, normalized to the cache + */ + public Collection extractInclusions(Artifact file, + ActionExecutionContext actionExecutionContext, RemoteParseData remoteParseData) + throws IOException, InterruptedException; + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java new file mode 100644 index 0000000000..120ba86a34 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/SolibSymlinkAction.java @@ -0,0 +1,234 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.AbstractAction; +import com.google.devtools.build.lib.actions.Action; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionExecutionException; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.ResourceSet; +import com.google.devtools.build.lib.actions.Root; +import com.google.devtools.build.lib.analysis.RuleContext; +import com.google.devtools.build.lib.analysis.config.BuildConfiguration; +import com.google.devtools.build.lib.rules.cpp.LinkerInputs.LibraryToLink; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.IOException; + +/** + * Creates mangled symlinks in the solib directory for all shared libraries. + * Libraries that have a potential to contain SONAME field rely on the mangled + * symlink to the parent directory instead. + * + * Such symlinks are used by the linker to ensure that all rpath entries can be + * specified relative to the $ORIGIN. + */ +public final class SolibSymlinkAction extends AbstractAction { + + private final Artifact library; + private final Path target; + private final Artifact symlink; + + private SolibSymlinkAction(ActionOwner owner, Artifact library, Artifact symlink) { + super(owner, ImmutableList.of(library), ImmutableList.of(symlink)); + + Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename())); + this.library = Preconditions.checkNotNull(library); + this.symlink = Preconditions.checkNotNull(symlink); + this.target = library.getPath(); + } + + @Override + protected void deleteOutputs(Path execRoot) throws IOException { + // Do not delete outputs if action does not intend to do anything. + if (target != null) { + super.deleteOutputs(execRoot); + } + } + + @Override + public void execute( + ActionExecutionContext actionExecutionContext) throws ActionExecutionException { + Path mangledPath = symlink.getPath(); + try { + FileSystemUtils.createDirectoryAndParents(mangledPath.getParentDirectory()); + mangledPath.createSymbolicLink(target); + } catch (IOException e) { + throw new ActionExecutionException("failed to create _solib symbolic link '" + + symlink.prettyPrint() + "' to target '" + target + "'", e, this, false); + } + } + + @Override + public Artifact getPrimaryInput() { + return library; + } + + @Override + public Artifact getPrimaryOutput() { + return symlink; + } + + @Override + public ResourceSet estimateResourceConsumption(Executor executor) { + return new ResourceSet(/*memoryMb=*/0, /*cpuUsage=*/0, /*ioUsage=*/0.0); + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addPath(symlink.getPath()); + if (target != null) { + f.addPath(target); + } + return f.hexDigestAndReset(); + } + + @Override + public String getMnemonic() { return "SolibSymlink"; } + + @Override + public String describeStrategy(Executor executor) { + return "local"; + } + + @Override + protected String getRawProgressMessage() { return null; } + + /** + * Replaces shared library artifact with mangled symlink and creates related + * symlink action. For artifacts that should retain filename (e.g. libraries + * with SONAME tag), link is created to the parent directory instead. + * + * This action is performed to minimize number of -rpath entries used during + * linking process (by essentially "collecting" as many shared libraries as + * possible in the single directory), since we will be paying quadratic price + * for each additional entry on the -rpath. + * + * @param ruleContext rule context, that requested symlink. + * @param library Shared library artifact that needs to be mangled. + * @param preserveName whether to preserve the name of the library + * @param prefixConsumer whether to prefix the output artifact name with the label of the + * consumer + * @return mangled symlink artifact. + */ + public static LibraryToLink getDynamicLibrarySymlink(final RuleContext ruleContext, + final Artifact library, + boolean preserveName, + boolean prefixConsumer, + BuildConfiguration configuration) { + PathFragment mangledName = getMangledName( + ruleContext, library.getRootRelativePath(), preserveName, prefixConsumer, + configuration.getFragment(CppConfiguration.class)); + return getDynamicLibrarySymlinkInternal(ruleContext, library, mangledName, configuration); + } + + /** + * Version of {@link #getDynamicLibrarySymlink} for the special case of C++ runtime libraries. + * These are handled differently than other libraries: neither their names nor directories are + * mangled, i.e. libstdc++.so.6 is symlinked from _solib_[arch]/libstdc++.so.6 + */ + public static LibraryToLink getCppRuntimeSymlink(RuleContext ruleContext, Artifact library, + String solibDirOverride, BuildConfiguration configuration) { + PathFragment solibDir = new PathFragment(solibDirOverride != null + ? solibDirOverride + : configuration.getFragment(CppConfiguration.class).getSolibDirectory()); + PathFragment symlinkName = solibDir.getRelative(library.getRootRelativePath().getBaseName()); + return getDynamicLibrarySymlinkInternal(ruleContext, library, symlinkName, configuration); + } + + /** + * Internal implementation that takes a pre-determined symlink name; supports both the + * generic {@link #getDynamicLibrarySymlink} and the specialized {@link #getCppRuntimeSymlink}. + */ + private static LibraryToLink getDynamicLibrarySymlinkInternal(RuleContext ruleContext, + Artifact library, PathFragment symlinkName, BuildConfiguration configuration) { + Preconditions.checkArgument(Link.SHARED_LIBRARY_FILETYPES.matches(library.getFilename())); + Preconditions.checkArgument(!library.getRootRelativePath().getSegment(0).startsWith("_solib_")); + + // Ignore libraries that are already represented by the symlinks. + Root root = configuration.getBinDirectory(); + Artifact symlink = ruleContext.getAnalysisEnvironment().getDerivedArtifact(symlinkName, root); + ruleContext.registerAction( + new SolibSymlinkAction(ruleContext.getActionOwner(), library, symlink)); + return LinkerInputs.solibLibraryToLink(symlink, library); + } + + /** + * Returns the name of the symlink that will be created for a library, given + * its name. + * + * @param ruleContext rule context that requests symlink + * @param libraryPath the root-relative path of the library + * @param preserveName true if filename should be preserved + * @param prefixConsumer true if the result should be prefixed with the label of the consumer + * @returns root relative path name + */ + public static PathFragment getMangledName(RuleContext ruleContext, + PathFragment libraryPath, + boolean preserveName, + boolean prefixConsumer, + CppConfiguration cppConfiguration) { + String escapedRulePath = Actions.escapedPath( + "_" + ruleContext.getLabel()); + String soname = getDynamicLibrarySoname(libraryPath, preserveName); + PathFragment solibDir = new PathFragment(cppConfiguration.getSolibDirectory()); + if (preserveName) { + String escapedLibraryPath = + Actions.escapedPath("_" + libraryPath.getParentDirectory().getPathString()); + PathFragment mangledDir = solibDir.getRelative(prefixConsumer + ? escapedRulePath + "__" + escapedLibraryPath + : escapedLibraryPath); + return mangledDir.getRelative(soname); + } else { + return solibDir.getRelative(prefixConsumer + ? escapedRulePath + "__" + soname + : soname); + } + } + + /** + * Compute the SONAME to use for a dynamic library. This name is basically the + * name of the shared library in its final symlinked location. + * + * @param libraryPath name of the shared library that needs to be mangled + * @param preserveName true if filename should be preserved, false - mangled + * @return soname to embed in the dynamic library + */ + public static String getDynamicLibrarySoname(PathFragment libraryPath, + boolean preserveName) { + String mangledName; + if (preserveName) { + mangledName = libraryPath.getBaseName(); + } else { + mangledName = "lib" + Actions.escapedPath(libraryPath.getPathString()); + } + return mangledName; + } + + @Override + public boolean shouldReportPathPrefixConflict(Action action) { + return false; // Always ignore path prefix conflict for the SolibSymlinkAction. + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java new file mode 100644 index 0000000000..40941249a2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/TransitiveLipoInfoProvider.java @@ -0,0 +1,51 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; + +/** + * A target that can contribute profiling information to LIPO C++ compilations. + * + *

This is used in the LIPO context collector tree to collect data from the transitive + * closure of the :lipo_context_collector target. It is eventually passed to the configured + * targets in the target configuration through {@link LipoContextProvider}. + */ +@Immutable +public final class TransitiveLipoInfoProvider implements TransitiveInfoProvider { + public static final TransitiveLipoInfoProvider EMPTY = + new TransitiveLipoInfoProvider( + NestedSetBuilder.emptySet(Order.STABLE_ORDER)); + + private final NestedSet includeScannables; + + public TransitiveLipoInfoProvider(NestedSet includeScannables) { + this.includeScannables = includeScannables; + } + + /** + * Returns the include scannables in the transitive closure. + * + *

This is used for constructing the path fragment -> include scannable map in the + * LIPO-enabled target configuration. + */ + public NestedSet getTransitiveIncludeScannables() { + return includeScannables; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java new file mode 100644 index 0000000000..58b33309b6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/WriteBuildInfoHeaderAction.java @@ -0,0 +1,194 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.cpp; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.analysis.BuildInfoHelper; +import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Fingerprint; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * An action that creates a C++ header containing the build information in the + * form of #define directives. + */ +public final class WriteBuildInfoHeaderAction extends AbstractFileWriteAction { + private static final String GUID = "b0798174-1352-4a54-854a-9785aaea491b"; + + private final ImmutableList valueArtifacts; + + private final boolean writeVolatileInfo; + private final boolean writeStableInfo; + + /** + * Creates an action that writes a C++ header with the build information. + * + *

It reads the set of build info keys from an action context that is usually contributed + * to Bazel by the workspace status module, and the value associated with said keys from the + * workspace status files (stable and volatile) written by the workspace status action. + * + *

Without input artifacts this action uses redacted build information. + * @param inputs Artifacts that contain build information, or an empty + * collection to use redacted build information + * @param output the C++ header Artifact created by this action + * @param writeVolatileInfo whether to write the volatile part of the build + * information to the generated header + * @param writeStableInfo whether to write the non-volatile part of the + * build information to the generated header + */ + public WriteBuildInfoHeaderAction(Collection inputs, + Artifact output, boolean writeVolatileInfo, boolean writeStableInfo) { + super(BuildInfoHelper.BUILD_INFO_ACTION_OWNER, + inputs, output, /*makeExecutable=*/false); + valueArtifacts = ImmutableList.copyOf(inputs); + if (!inputs.isEmpty()) { + // With non-empty inputs we should not generate both volatile and non-volatile data + // in the same header file. + Preconditions.checkState(writeVolatileInfo ^ writeStableInfo); + } + Preconditions.checkState( + output.isConstantMetadata() == (writeVolatileInfo && !inputs.isEmpty())); + + this.writeVolatileInfo = writeVolatileInfo; + this.writeStableInfo = writeStableInfo; + } + + @Override + public DeterministicWriter newDeterministicWriter(EventHandler eventHandler, Executor executor) + throws IOException { + WorkspaceStatusAction.Context context = + executor.getContext(WorkspaceStatusAction.Context.class); + + final Map keys = new LinkedHashMap<>(); + if (writeVolatileInfo) { + keys.putAll(context.getVolatileKeys()); + } + + if (writeStableInfo) { + keys.putAll(context.getStableKeys()); + } + + final Map values = new LinkedHashMap<>(); + for (Artifact valueFile : valueArtifacts) { + values.putAll(WorkspaceStatusAction.parseValues(valueFile.getPath())); + } + + final boolean redacted = valueArtifacts.isEmpty(); + + return new DeterministicWriter() { + @Override + public void writeOutputFile(OutputStream out) throws IOException { + Writer writer = new OutputStreamWriter(out, UTF_8); + + for (Map.Entry key : keys.entrySet()) { + if (!key.getValue().isInLanguage("C++")) { + continue; + } + + String value = redacted ? key.getValue().getRedactedValue() + : values.containsKey(key.getKey()) ? values.get(key.getKey()) + : key.getValue().getDefaultValue(); + + switch (key.getValue().getType()) { + case VERBATIM: + case INTEGER: + break; + + case STRING: + value = quote(value); + break; + + default: + throw new IllegalStateException(); + } + define(writer, key.getKey(), value); + + } + writer.flush(); + } + }; + } + + @Override + protected String computeKey() { + Fingerprint f = new Fingerprint(); + f.addString(GUID); + f.addBoolean(writeStableInfo); + f.addBoolean(writeVolatileInfo); + return f.hexDigestAndReset(); + } + + @Override + public boolean executeUnconditionally() { + // Note: isVolatile must return true if executeUnconditionally can ever return true + // for this instance. + return isUnconditional(); + } + + @Override + public boolean isVolatile() { + return isUnconditional(); + } + + private boolean isUnconditional() { + // Because of special handling in the MetadataHandler, changed volatile build + // information does not trigger relinking of all libraries that have + // linkstamps. But we do want to regenerate the header in case libraries are + // relinked because of other reasons. + // Without inputs the contents of the header do not change, so there is no + // point in executing the action again in that case. + return writeVolatileInfo && !Iterables.isEmpty(getInputs()); + } + + /** + * Quote a string with double quotes. + */ + private String quote(String string) { + // TODO(bazel-team): This is doesn't really work if the string contains quotes. Or a newline. + // Or a backslash. Or anything unusual, really. + return "\"" + string + "\""; + } + + /** + * Write a preprocessor define directive to a Writer. + */ + private void define(Writer writer, String name, String value) throws IOException { + writer.write("#define "); + writer.write(name); + writer.write(' '); + writer.write(value); + writer.write('\n'); + } + + @Override + protected String getRawProgressMessage() { + return null; + } +} -- cgit v1.2.3