From 6186fa5fe9869c6102597b18244fc9546f931632 Mon Sep 17 00:00:00 2001 From: Googler Date: Tue, 13 Jun 2017 17:39:50 +0200 Subject: Add support for zipped LLVM profile files. This change is a follow-up to a recent change which allowed LLVM raw profile files to be directly used with blaze. This change allows zipped LLVM raw profile files. This uses //tools/zip:zipper to extract the zipped file contents. This also adds a new option to //tools/zip:zipper, 'j', to junk directories while unzipping. Tested: blaze test //devtools/blaze/integration:fdo_test blaze test //third_party/ijar/test:zip_test RELNOTES[NEW]: Zipped LLVM profiles are now supported. PiperOrigin-RevId: 158849516 --- .../devtools/build/lib/rules/cpp/CcToolchain.java | 77 +++++++++++++++++----- .../build/lib/rules/cpp/CcToolchainRule.java | 19 ++++++ .../build/lib/rules/cpp/CppConfiguration.java | 7 +- .../devtools/build/lib/rules/cpp/FdoSupport.java | 11 +--- .../build/lib/rules/cpp/FdoSupportFunction.java | 12 ++-- .../build/lib/rules/cpp/FdoSupportValue.java | 14 +++- 6 files changed, 108 insertions(+), 32 deletions(-) (limited to 'src/main/java/com/google/devtools') diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java index ea3deb880e..af4c3e5acb 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchain.java @@ -45,6 +45,7 @@ import com.google.devtools.build.lib.packages.License; import com.google.devtools.build.lib.rules.MakeVariableProvider; import com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory; import com.google.devtools.build.lib.rules.cpp.FdoSupport.FdoException; +import com.google.devtools.build.lib.util.FileType; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -72,12 +73,16 @@ public class CcToolchain implements RuleConfiguredTargetFactory { private static final PathFragment BUILTIN_INCLUDE_FILE_SUFFIX = PathFragment.create("include/stdc-predef.h"); - private static String getLLVMProfileFileName(Path fdoProfile) { - if (CppFileTypes.LLVM_PROFILE.matches(fdoProfile)) { + /* + * Returns the profile name with the same file name as fdoProfile and an + * extension that matches {@link FileType}. + */ + private static String getLLVMProfileFileName(Path fdoProfile, FileType type) { + if (type.matches(fdoProfile)) { return fdoProfile.getBaseName(); } else { return FileSystemUtils.removeExtension(fdoProfile.getBaseName()) - + CppFileTypes.LLVM_PROFILE.getExtensions().get(0); + + type.getExtensions().get(0); } } @@ -91,7 +96,9 @@ public class CcToolchain implements RuleConfiguredTargetFactory { Artifact profileArtifact = ruleContext.getUniqueDirectoryArtifact( - "fdo", getLLVMProfileFileName(fdoProfile), ruleContext.getBinOrGenfilesDirectory()); + "fdo", + getLLVMProfileFileName(fdoProfile, CppFileTypes.LLVM_PROFILE), + ruleContext.getBinOrGenfilesDirectory()); // If the profile file is already in the desired format, symlink to it and return. if (CppFileTypes.LLVM_PROFILE.matches(fdoProfile)) { @@ -106,14 +113,52 @@ public class CcToolchain implements RuleConfiguredTargetFactory { Artifact rawProfileArtifact = ruleContext.getUniqueDirectoryArtifact( - "fdo", fdoProfile.getBaseName(), ruleContext.getBinOrGenfilesDirectory()); + "fdo", + getLLVMProfileFileName(fdoProfile, CppFileTypes.LLVM_PROFILE_RAW), + ruleContext.getBinOrGenfilesDirectory()); + + if (fdoProfile.getBaseName().endsWith(".zip")) { + // Get the zipper binary for unzipping the profile. + Artifact zipperBinaryArtifact = ruleContext.getPrerequisiteArtifact(":zipper", Mode.HOST); + if (zipperBinaryArtifact == null) { + ruleContext.ruleError("Cannot find zipper binary to unzip the profile"); + return null; + } - ruleContext.registerAction( - new SymlinkAction( - ruleContext.getActionOwner(), - PathFragment.create(fdoProfile.getPathString()), - rawProfileArtifact, - "Symlinking LLVM Profile " + fdoProfile.getPathString())); + // Symlink to the zipped profile file to extract the contents. + Artifact zipProfileArtifact = + ruleContext.getUniqueDirectoryArtifact( + "fdo", fdoProfile.getBaseName(), ruleContext.getBinOrGenfilesDirectory()); + ruleContext.registerAction( + new SymlinkAction( + ruleContext.getActionOwner(), + PathFragment.create(fdoProfile.getPathString()), + zipProfileArtifact, + "Symlinking LLVM ZIP Profile " + fdoProfile.getPathString())); + + // Unzip the profile. + ruleContext.registerAction( + new SpawnAction.Builder() + .addInput(zipProfileArtifact) + .addInput(zipperBinaryArtifact) + .addOutput(rawProfileArtifact) + .useDefaultShellEnvironment() + .setExecutable(zipperBinaryArtifact) + .addArguments("xf", zipProfileArtifact.getExecPathString()) + .addArguments( + "-d", rawProfileArtifact.getExecPath().getParentDirectory().getSafePathString()) + .setProgressMessage( + "LLVMUnzipProfileAction: Generating " + rawProfileArtifact.prettyPrint()) + .setMnemonic("LLVMUnzipProfileAction") + .build(ruleContext)); + } else { + ruleContext.registerAction( + new SymlinkAction( + ruleContext.getActionOwner(), + PathFragment.create(fdoProfile.getPathString()), + rawProfileArtifact, + "Symlinking LLVM Raw Profile " + fdoProfile.getPathString())); + } if (cppConfiguration.getLLVMProfDataExecutable() == null) { ruleContext.ruleError( @@ -154,10 +199,12 @@ public class CcToolchain implements RuleConfiguredTargetFactory { Path fdoZip = ruleContext.getConfiguration().getCompilationMode() == CompilationMode.OPT ? cppConfiguration.getFdoZip() : null; - SkyKey fdoKey = FdoSupportValue.key( - cppConfiguration.getLipoMode(), - fdoZip, - cppConfiguration.getFdoInstrument()); + SkyKey fdoKey = + FdoSupportValue.key( + cppConfiguration.getLipoMode(), + fdoZip, + cppConfiguration.getFdoInstrument(), + cppConfiguration.isLLVMOptimizedFdo()); SkyFunction.Environment skyframeEnv = ruleContext.getAnalysisEnvironment().getSkyframeEnv(); FdoSupportValue fdoSupport; diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java index f7414408f8..1fad6472b6 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcToolchainRule.java @@ -58,6 +58,7 @@ public final class CcToolchainRule implements RuleDefinition { @Override public RuleClass build(Builder builder, RuleDefinitionEnvironment env) { + final Label zipper = env.getToolsLabel("//tools/zip:zipper"); return builder .setUndocumented() .requiresConfigurationFragments(CppConfiguration.class) @@ -85,6 +86,24 @@ public final class CcToolchainRule implements RuleDefinition { .cfg(HOST) .singleArtifact() .value(env.getToolsLabel("//tools/cpp:link_dynamic_library"))) + .add( + attr(":zipper", LABEL) + .cfg(HOST) + .singleArtifact() + .value( + new LateBoundLabel() { + @Override + public Label resolve( + Rule rule, AttributeMap attributes, BuildConfiguration configuration) { + CppConfiguration cppConfiguration = + configuration.getFragment(CppConfiguration.class); + if (cppConfiguration.isLLVMOptimizedFdo()) { + return zipper; + } else { + return null; + } + } + })) .add(attr(":libc_top", LABEL).value(LIBC_TOP)) .add( attr(":lipo_context_collector", LABEL) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java index 1d50ed608c..617fce8bf2 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java @@ -1601,7 +1601,12 @@ public class CppConfiguration extends BuildConfiguration.Fragment { return cppOptions.isFdo() && cppOptions.getFdoOptimize() != null && (CppFileTypes.LLVM_PROFILE.matches(cppOptions.getFdoOptimize()) - || CppFileTypes.LLVM_PROFILE_RAW.matches(cppOptions.getFdoOptimize())); + || CppFileTypes.LLVM_PROFILE_RAW.matches(cppOptions.getFdoOptimize()) + // TODO(tmsriram): Checking for "llvm" does not handle all the cases. This + // is temporary until the crosstool configuration is modified to add fields that + // indicate which flavor of fdo is being used. + || (getToolchainIdentifier().contains("llvm") + && cppOptions.getFdoOptimize().endsWith(".zip"))); } /** diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java index 9bf74e93c5..781061b79b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupport.java @@ -154,14 +154,6 @@ public class FdoSupport { return CppFileTypes.GCC_AUTO_PROFILE.matches(fdoFile); } - /** - * Returns true if the given fdoFile represents an LLVM profile. - */ - public static final boolean isLLVMFdo(String fdoFile) { - return (CppFileTypes.LLVM_PROFILE.matches(fdoFile) - || CppFileTypes.LLVM_PROFILE_RAW.matches(fdoFile)); - } - /** * Coverage information output directory passed to {@code --fdo_instrument}, * or {@code null} if FDO instrumentation is disabled. @@ -262,12 +254,13 @@ public class FdoSupport { PathFragment fdoInstrument, Path fdoProfile, LipoMode lipoMode, + boolean llvmFdo, Path execRoot) throws IOException, FdoException, InterruptedException { FdoMode fdoMode; if (fdoProfile != null && isAutoFdo(fdoProfile.getBaseName())) { fdoMode = FdoMode.AUTO_FDO; - } else if (fdoProfile != null && isLLVMFdo(fdoProfile.getBaseName())) { + } else if (fdoProfile != null && llvmFdo) { fdoMode = FdoMode.LLVM_FDO; } else if (fdoProfile != null) { fdoMode = FdoMode.VANILLA; diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportFunction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportFunction.java index d4c838969e..0a2f326186 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportFunction.java @@ -23,9 +23,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; - import java.io.IOException; - import javax.annotation.Nullable; /** @@ -57,8 +55,14 @@ public class FdoSupportFunction implements SkyFunction { FdoSupportValue.Key key = (FdoSupportValue.Key) skyKey.argument(); FdoSupport fdoSupport; try { - fdoSupport = FdoSupport.create(env, - key.getFdoInstrument(), key.getFdoZip(), key.getLipoMode(), execRoot); + fdoSupport = + FdoSupport.create( + env, + key.getFdoInstrument(), + key.getFdoZip(), + key.getLipoMode(), + key.getLLVMFdo(), + execRoot); if (env.valuesMissing()) { return null; } diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportValue.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportValue.java index 90ee9326b7..0f94dcd486 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportValue.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/FdoSupportValue.java @@ -44,11 +44,13 @@ public class FdoSupportValue implements SkyValue { private final LipoMode lipoMode; private final Path fdoZip; private final PathFragment fdoInstrument; + private final boolean llvmFdo; - private Key(LipoMode lipoMode, Path fdoZip, PathFragment fdoInstrument) { + private Key(LipoMode lipoMode, Path fdoZip, PathFragment fdoInstrument, boolean llvmFdo) { this.lipoMode = lipoMode; this.fdoZip = fdoZip; this.fdoInstrument = fdoInstrument; + this.llvmFdo = llvmFdo; } public LipoMode getLipoMode() { @@ -63,6 +65,10 @@ public class FdoSupportValue implements SkyValue { return fdoInstrument; } + public boolean getLLVMFdo() { + return llvmFdo; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -76,6 +82,7 @@ public class FdoSupportValue implements SkyValue { Key that = (Key) o; return Objects.equals(this.lipoMode, that.lipoMode) && Objects.equals(this.fdoZip, that.fdoZip) + && Objects.equals(this.llvmFdo, that.llvmFdo) && Objects.equals(this.fdoInstrument, that.fdoInstrument); } @@ -95,7 +102,8 @@ public class FdoSupportValue implements SkyValue { return fdoSupport; } - public static SkyKey key(LipoMode lipoMode, Path fdoZip, PathFragment fdoInstrument) { - return LegacySkyKey.create(SKYFUNCTION, new Key(lipoMode, fdoZip, fdoInstrument)); + public static SkyKey key( + LipoMode lipoMode, Path fdoZip, PathFragment fdoInstrument, boolean llvmFdo) { + return LegacySkyKey.create(SKYFUNCTION, new Key(lipoMode, fdoZip, fdoInstrument, llvmFdo)); } } -- cgit v1.2.3