diff options
author | Ulf Adams <ulfjack@google.com> | 2016-03-30 12:23:50 +0000 |
---|---|---|
committer | Klaus Aehlig <aehlig@google.com> | 2016-03-30 15:23:50 +0000 |
commit | ab43b97040a04d21cb951e1c59b760fc67dfaa3d (patch) | |
tree | 0acf85e9804cad3ef40f9f9f79249f2329cee82f /src/main | |
parent | 94b72db92b54af7a6b1e7a5a48b218b26ac761e5 (diff) |
Move most of the workspace-handling code from BlazeRuntime to a new class.
--
MOS_MIGRATED_REVID=118563271
Diffstat (limited to 'src/main')
9 files changed, 323 insertions, 248 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java index 7da7e2872e..7523cc217b 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java @@ -176,7 +176,7 @@ public final class BuildTool { + "'test' right now!"); } } - configurations = runtime.getSkyframeExecutor().createConfigurations( + configurations = env.getSkyframeExecutor().createConfigurations( env.getReporter(), runtime.getConfigurationFactory(), buildOptions, env.getDirectories(), request.getMultiCpus(), request.getViewOptions().keepGoing); @@ -204,7 +204,7 @@ public final class BuildTool { // Execution phase. if (needsExecutionPhase(request.getBuildOptions())) { - runtime.getSkyframeExecutor().injectTopLevelContext(request.getTopLevelArtifactContext()); + env.getSkyframeExecutor().injectTopLevelContext(request.getTopLevelArtifactContext()); executionTool.executeBuild(request.getId(), analysisResult, result, configurations, transformPackageRoots(analysisResult.getPackageRoots())); } @@ -228,7 +228,7 @@ public final class BuildTool { // Delete dirty nodes to ensure that they do not accumulate indefinitely. long versionWindow = request.getViewOptions().versionWindowForDirtyNodeGc; if (versionWindow != -1) { - runtime.getSkyframeExecutor().deleteOldNodes(versionWindow); + env.getSkyframeExecutor().deleteOldNodes(versionWindow); } if (executionTool != null) { @@ -413,11 +413,11 @@ public final class BuildTool { @Override public void notifyVisitedPackages(Set<PackageIdentifier> visitedPackages) { - runtime.getSkyframeExecutor().updateLoadedPackageSet(visitedPackages); + env.getSkyframeExecutor().updateLoadedPackageSet(visitedPackages); } }; - LoadingPhaseRunner loadingPhaseRunner = runtime.getSkyframeExecutor().getLoadingPhaseRunner( + LoadingPhaseRunner loadingPhaseRunner = env.getSkyframeExecutor().getLoadingPhaseRunner( runtime.getPackageFactory().getRuleClassNames(), request.getLoadingOptions().useSkyframeTargetPatternEvaluator); LoadingResult result = loadingPhaseRunner.execute(getReporter(), @@ -464,7 +464,7 @@ public final class BuildTool { Profiler.instance().markPhase(ProfilePhase.ANALYZE); BuildView view = new BuildView(env.getDirectories(), runtime.getRuleClassProvider(), - runtime.getSkyframeExecutor(), runtime.getCoverageReportActionFactory()); + env.getSkyframeExecutor(), runtime.getCoverageReportActionFactory()); AnalysisResult analysisResult = view.update( loadingResult, diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 0b1e95ba43..f4da845e8f 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -437,7 +437,7 @@ public class ExecutionTool { executor, builtTargets, request.getBuildOptions().explanationPath != null, - runtime.getLastExecutionTimeRange()); + env.getBlazeWorkspace().getLastExecutionTimeRange()); buildCompleted = true; } catch (BuildFailedException | TestExecException e) { buildCompleted = true; diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java index ad45de4b62..97cdbdd0b5 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java @@ -143,7 +143,7 @@ public class BlazeCommandDispatcher { } Path doNotBuild = workspace.getParentDirectory().getRelative( - BlazeRuntime.DO_NOT_BUILD_FILE_NAME); + BlazeWorkspace.DO_NOT_BUILD_FILE_NAME); if (doNotBuild.exists()) { if (!commandAnnotation.canRunInOutputDirectory()) { @@ -319,7 +319,7 @@ public class BlazeCommandDispatcher { // Do this before an actual crash so we don't have to worry about // allocating memory post-crash. - String[] crashData = runtime.getCrashData(env); + String[] crashData = env.getCrashData(); int numericExitCode = ExitCode.BLAZE_INTERNAL_ERROR.getNumericExitCode(); PrintStream savedOut = System.out; PrintStream savedErr = System.err; diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java index 638e5f964f..fc736b5a8d 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java @@ -14,9 +14,6 @@ package com.google.devtools.build.lib.runtime; -import static com.google.devtools.build.lib.profiler.AutoProfiler.profiledAndLogged; -import static java.nio.charset.StandardCharsets.ISO_8859_1; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Predicate; @@ -25,16 +22,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.SubscriberExceptionContext; import com.google.common.eventbus.SubscriberExceptionHandler; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Uninterruptibles; -import com.google.devtools.build.lib.actions.cache.ActionCache; -import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache; -import com.google.devtools.build.lib.actions.cache.NullActionCache; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; @@ -44,11 +37,9 @@ import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.OutputFilter; -import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.packages.Preprocessor; import com.google.devtools.build.lib.packages.RuleClassProvider; -import com.google.devtools.build.lib.pkgcache.PackageManager; import com.google.devtools.build.lib.profiler.AutoProfiler; import com.google.devtools.build.lib.profiler.MemoryProfiler; import com.google.devtools.build.lib.profiler.ProfilePhase; @@ -93,7 +84,6 @@ import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.ThreadUtils; import com.google.devtools.build.lib.util.io.OutErr; import com.google.devtools.build.lib.vfs.FileSystem; -import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.JavaIoFileSystem; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -144,8 +134,6 @@ import javax.annotation.Nullable; * <p>The parts specific to the current command are stored in {@link CommandEnvironment}. */ public final class BlazeRuntime { - public static final String DO_NOT_BUILD_FILE_NAME = "DO_NOT_BUILD_HERE"; - private static final Pattern suppressFromLog = Pattern.compile(".*(auth|pass|cookie).*", Pattern.CASE_INSENSITIVE); @@ -173,13 +161,7 @@ public final class BlazeRuntime { private final QueryEnvironmentFactory queryEnvironmentFactory; // Workspace state (currently exactly one workspace per server) - private final BlazeDirectories directories; - private final SkyframeExecutor skyframeExecutor; - /** The action cache is loaded lazily on the first build command. */ - private ActionCache actionCache; - /** The execution time range of the previous build command in this server, if any. */ - @Nullable - private Range<Long> lastExecutionRange = null; + private final BlazeWorkspace workspace; private BlazeRuntime(BlazeDirectories directories, WorkspaceStatusAction.Factory workspaceStatusActionFactory, @@ -209,14 +191,7 @@ public final class BlazeRuntime { this.queryEnvironmentFactory = queryEnvironmentFactory; // Workspace state - this.directories = directories; - this.skyframeExecutor = skyframeExecutor; - - if (directories.inWorkspace()) { - writeOutputBaseReadmeFile(); - writeDoNotBuildHereFile(); - } - setupExecRoot(); + this.workspace = new BlazeWorkspace(this, directories, skyframeExecutor); } private static InvocationPolicy createInvocationPolicyFromModules( @@ -274,17 +249,11 @@ public final class BlazeRuntime { public CommandEnvironment initCommand() { EventBus eventBus = new EventBus(eventBusExceptionHandler); - skyframeExecutor.setEventBus(eventBus); + workspace.getSkyframeExecutor().setEventBus(eventBus); UUID commandId = UUID.randomUUID(); return new CommandEnvironment(this, commandId, eventBus); } - private void clearEventBus() { - // EventBus does not have an unregister() method, so this is how we release memory associated - // with handlers. - skyframeExecutor.setEventBus(null); - } - @Nullable public InvocationPolicy getInvocationPolicy() { return invocationPolicy; @@ -301,7 +270,7 @@ public final class BlazeRuntime { try { if (options.profilePath != null) { - Path profilePath = getWorkspace().getRelative(options.profilePath); + Path profilePath = env.getWorkspace().getRelative(options.profilePath); recordFullProfilerData = options.recordFullProfilerData; out = new BufferedOutputStream(profilePath.getOutputStream(), 1024 * 1024); @@ -314,7 +283,7 @@ public final class BlazeRuntime { } if (profiledTasks != ProfiledTaskKinds.NONE) { Profiler.instance().start(profiledTasks, out, - "Blaze profile for " + getOutputBase() + " at " + new Date() + "Blaze profile for " + env.getOutputBase() + " at " + new Date() + ", build ID: " + buildID, recordFullProfilerData, clock, execStartTimeNanos); return true; @@ -325,101 +294,8 @@ public final class BlazeRuntime { return false; } - /** - * Generates a README file in the output base directory. This README file - * contains the name of the workspace directory, so that users can figure out - * which output base directory corresponds to which workspace. - */ - private void writeOutputBaseReadmeFile() { - Preconditions.checkNotNull(getWorkspace()); - Path outputBaseReadmeFile = getOutputBase().getRelative("README"); - try { - FileSystemUtils.writeIsoLatin1(outputBaseReadmeFile, "WORKSPACE: " + getWorkspace(), "", - "The first line of this file is intentionally easy to parse for various", - "interactive scripting and debugging purposes. But please DO NOT write programs", - "that exploit it, as they will be broken by design: it is not possible to", - "reverse engineer the set of source trees or the --package_path from the output", - "tree, and if you attempt it, you will fail, creating subtle and", - "hard-to-diagnose bugs, that will no doubt get blamed on changes made by the", - "Blaze team.", "", "This directory was generated by Blaze.", - "Do not attempt to modify or delete any files in this directory.", - "Among other issues, Blaze's file system caching assumes that", - "only Blaze will modify this directory and the files in it,", - "so if you change anything here you may mess up Blaze's cache."); - } catch (IOException e) { - LOG.warning("Couldn't write to '" + outputBaseReadmeFile + "': " + e.getMessage()); - } - } - - private void writeDoNotBuildHereFile(Path filePath) { - try { - FileSystemUtils.createDirectoryAndParents(filePath.getParentDirectory()); - FileSystemUtils.writeContent(filePath, ISO_8859_1, getWorkspace().toString()); - } catch (IOException e) { - LOG.warning("Couldn't write to '" + filePath + "': " + e.getMessage()); - } - } - - private void writeDoNotBuildHereFile() { - Preconditions.checkNotNull(getWorkspace()); - writeDoNotBuildHereFile(getOutputBase().getRelative(DO_NOT_BUILD_FILE_NAME)); - if (startupOptionsProvider.getOptions(BlazeServerStartupOptions.class).deepExecRoot) { - writeDoNotBuildHereFile(getOutputBase().getRelative("execroot").getRelative( - DO_NOT_BUILD_FILE_NAME)); - } - } - - /** - * Creates the execRoot dir under outputBase. - */ - private void setupExecRoot() { - try { - FileSystemUtils.createDirectoryAndParents(directories.getExecRoot()); - } catch (IOException e) { - LOG.warning("failed to create execution root '" + directories.getExecRoot() + "': " - + e.getMessage()); - } - } - - void recordLastExecutionTime(long commandStartTime) { - long currentTimeMillis = clock.currentTimeMillis(); - lastExecutionRange = currentTimeMillis >= commandStartTime - ? Range.closed(commandStartTime, currentTimeMillis) - : null; - } - - /** - * Range that represents the last execution time of a build in millis since epoch. - */ - @Nullable - public Range<Long> getLastExecutionTimeRange() { - return lastExecutionRange; - } - - /** - * Returns the Blaze directories object for this runtime. - */ - public BlazeDirectories getDirectories() { - return directories; - } - - /** - * Returns the working directory of the server. - * - * <p>This is often the first entry on the {@code --package_path}, but not always. - * Callers should certainly not make this assumption. The Path returned may be null. - */ - private Path getWorkspace() { - return directories.getWorkspace(); - } - - /** - * Returns the output base directory associated with this Blaze server - * process. This is the base directory for shared Blaze state as well as tool - * and strategy specific subdirectories. - */ - private Path getOutputBase() { - return directories.getOutputBase(); + public BlazeWorkspace getWorkspace() { + return workspace; } /** @@ -427,7 +303,7 @@ public final class BlazeRuntime { * file and a log. */ private Path getServerDirectory() { - return getDirectories().getOutputBase().getChild("server"); + return getWorkspace().getDirectories().getOutputBase().getChild("server"); } public BinTools getBinTools() { @@ -435,13 +311,6 @@ public final class BlazeRuntime { } /** - * Returns the skyframe executor. - */ - public SkyframeExecutor getSkyframeExecutor() { - return skyframeExecutor; - } - - /** * Returns the {@link QueryEnvironmentFactory} that should be used to create a * {@link AbstractBlazeQueryEnvironment}, whenever one is needed. */ @@ -466,13 +335,6 @@ public final class BlazeRuntime { return result.build(); } - /** - * Returns the package manager. - */ - public PackageManager getPackageManager() { - return skyframeExecutor.getPackageManager(); - } - public WorkspaceStatusAction.Factory getworkspaceStatusActionFactory() { return workspaceStatusActionFactory; } @@ -504,55 +366,6 @@ public final class BlazeRuntime { } /** - * Returns reference to the lazily instantiated persistent action cache - * instance. Note, that method may recreate instance between different build - * requests, so return value should not be cached. - */ - public ActionCache getPersistentActionCache(Reporter reporter) throws IOException { - if (actionCache == null) { - if (OS.getCurrent() == OS.WINDOWS) { - // TODO(bazel-team): Add support for a persistent action cache on Windows. - actionCache = new NullActionCache(); - return actionCache; - } - try (AutoProfiler p = profiledAndLogged("Loading action cache", ProfilerTask.INFO, LOG)) { - try { - actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock); - } catch (IOException e) { - LOG.log(Level.WARNING, "Failed to load action cache: " + e.getMessage(), e); - LoggingUtil.logToRemote(Level.WARNING, "Failed to load action cache: " - + e.getMessage(), e); - reporter.handle( - Event.error("Error during action cache initialization: " + e.getMessage() - + ". Corrupted files were renamed to '" + getCacheDirectory() + "/*.bad'. " - + "Blaze will now reset action cache data, causing a full rebuild")); - actionCache = new CompactPersistentActionCache(getCacheDirectory(), clock); - } - } - } - return actionCache; - } - - /** - * Removes in-memory caches. - */ - public void clearCaches() throws IOException { - skyframeExecutor.resetEvaluator(); - actionCache = null; - FileSystemUtils.deleteTree(getCacheDirectory()); - } - - /** - * Returns path to the cache directory. Path must be inside output base to - * ensure that users can run concurrent instances of blaze in different - * clients without attempting to concurrently write to the same action cache - * on disk, which might not be safe. - */ - private Path getCacheDirectory() { - return getOutputBase().getChild("action_cache"); - } - - /** * Returns a provider for project file objects. Can be null if no such provider was set by any of * the modules. */ @@ -610,7 +423,7 @@ public final class BlazeRuntime { // thread won the race (unlikely, but possible), this may be incorrectly logged as a success. return; } - skyframeExecutor.getEventBus().post(new CommandCompleteEvent(exitCode)); + workspace.getSkyframeExecutor().getEventBus().post(new CommandCompleteEvent(exitCode)); } /** @@ -628,7 +441,7 @@ public final class BlazeRuntime { module.afterCommand(); } - clearEventBus(); + env.getBlazeWorkspace().clearEventBus(); try { Profiler.instance().stop(); @@ -671,26 +484,6 @@ public final class BlazeRuntime { return startupOptionsProvider; } - /** - * An array of String values useful if Blaze crashes. - * For now, just returns the size of the action cache and the build id. - */ - public String[] getCrashData(CommandEnvironment env) { - return new String[]{ - getFileSizeString(CompactPersistentActionCache.cacheFile(getCacheDirectory()), - "action cache"), - env.getCommandId() + " (build id)", - }; - } - - private String getFileSizeString(Path path, String type) { - try { - return String.format("%d bytes (%s)", path.getFileSize(), type); - } catch (IOException e) { - return String.format("unknown file size (%s)", type); - } - } - public Map<String, BlazeCommand> getCommandMap() { return commandMap; } @@ -1034,7 +827,8 @@ public final class BlazeRuntime { }; RPCServer server = RPCServer.newServerWith(runtime.getClock(), blazeCommand, - runtime.getServerDirectory(), runtime.getWorkspace(), startupOptions.maxIdleSeconds); + runtime.getServerDirectory(), runtime.workspace.getWorkspace(), + startupOptions.maxIdleSeconds); return server; } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java new file mode 100644 index 0000000000..65ba1acd6a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeWorkspace.java @@ -0,0 +1,254 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import static com.google.devtools.build.lib.profiler.AutoProfiler.profiledAndLogged; +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.collect.Range; +import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache; +import com.google.devtools.build.lib.actions.cache.NullActionCache; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.profiler.AutoProfiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.util.LoggingUtil; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.Preconditions; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.common.options.OptionsProvider; + +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +/** + * This class represents a workspace, and contains operations and data related to it. In contrast, + * the BlazeRuntime class represents the Blaze server, and contains operations and data that are + * (supposed to be) independent of the workspace or the current command. + * + * <p>At this time, there is still a 1:1 relationship between the BlazeRuntime and the + * BlazeWorkspace, but the introduction of this class is a step towards allowing 1:N relationships. + */ +public final class BlazeWorkspace { + public static final String DO_NOT_BUILD_FILE_NAME = "DO_NOT_BUILD_HERE"; + + private static final Logger LOG = Logger.getLogger(BlazeRuntime.class.getName()); + + private final BlazeRuntime runtime; + + private final BlazeDirectories directories; + private final SkyframeExecutor skyframeExecutor; + /** The action cache is loaded lazily on the first build command. */ + private ActionCache actionCache; + /** The execution time range of the previous build command in this server, if any. */ + @Nullable + private Range<Long> lastExecutionRange = null; + + public BlazeWorkspace( + BlazeRuntime runtime, BlazeDirectories directories, SkyframeExecutor skyframeExecutor) { + this.runtime = runtime; + this.directories = directories; + this.skyframeExecutor = skyframeExecutor; + + if (directories.inWorkspace()) { + writeOutputBaseReadmeFile(); + writeDoNotBuildHereFile(runtime.getStartupOptionsProvider()); + } + setupExecRoot(); + } + + /** + * Returns the Blaze directories object for this runtime. + */ + public BlazeDirectories getDirectories() { + return directories; + } + + public SkyframeExecutor getSkyframeExecutor() { + return skyframeExecutor; + } + + /** + * Returns the working directory of the server. + * + * <p>This is often the first entry on the {@code --package_path}, but not always. + * Callers should certainly not make this assumption. The Path returned may be null. + */ + public Path getWorkspace() { + return directories.getWorkspace(); + } + + /** + * Returns the output base directory associated with this Blaze server + * process. This is the base directory for shared Blaze state as well as tool + * and strategy specific subdirectories. + */ + public Path getOutputBase() { + return directories.getOutputBase(); + } + + /** + * Returns the output path associated with this Blaze server process.. + */ + public Path getOutputPath() { + return directories.getOutputPath(); + } + + public Path getInstallBase() { + return directories.getInstallBase(); + } + + /** + * Returns the execution root directory associated with this Blaze server + * process. This is where all input and output files visible to the actual + * build reside. + */ + public Path getExecRoot() { + return directories.getExecRoot(); + } + + /** + * Returns path to the cache directory. Path must be inside output base to + * ensure that users can run concurrent instances of blaze in different + * clients without attempting to concurrently write to the same action cache + * on disk, which might not be safe. + */ + Path getCacheDirectory() { + return getOutputBase().getChild("action_cache"); + } + + void recordLastExecutionTime(long commandStartTime) { + long currentTimeMillis = runtime.getClock().currentTimeMillis(); + lastExecutionRange = currentTimeMillis >= commandStartTime + ? Range.closed(commandStartTime, currentTimeMillis) + : null; + } + + /** + * Range that represents the last execution time of a build in millis since epoch. + */ + @Nullable + public Range<Long> getLastExecutionTimeRange() { + return lastExecutionRange; + } + + void clearEventBus() { + // EventBus does not have an unregister() method, so this is how we release memory associated + // with handlers. + skyframeExecutor.setEventBus(null); + } + + /** + * Removes in-memory caches. + */ + public void clearCaches() throws IOException { + skyframeExecutor.resetEvaluator(); + actionCache = null; + FileSystemUtils.deleteTree(getCacheDirectory()); + } + + /** + * Returns reference to the lazily instantiated persistent action cache + * instance. Note, that method may recreate instance between different build + * requests, so return value should not be cached. + */ + public ActionCache getPersistentActionCache(Reporter reporter) throws IOException { + if (actionCache == null) { + if (OS.getCurrent() == OS.WINDOWS) { + // TODO(bazel-team): Add support for a persistent action cache on Windows. + actionCache = new NullActionCache(); + return actionCache; + } + try (AutoProfiler p = profiledAndLogged("Loading action cache", ProfilerTask.INFO, LOG)) { + try { + actionCache = new CompactPersistentActionCache(getCacheDirectory(), runtime.getClock()); + } catch (IOException e) { + LOG.log(Level.WARNING, "Failed to load action cache: " + e.getMessage(), e); + LoggingUtil.logToRemote(Level.WARNING, "Failed to load action cache: " + + e.getMessage(), e); + reporter.handle( + Event.error("Error during action cache initialization: " + e.getMessage() + + ". Corrupted files were renamed to '" + getCacheDirectory() + "/*.bad'. " + + "Blaze will now reset action cache data, causing a full rebuild")); + actionCache = new CompactPersistentActionCache(getCacheDirectory(), runtime.getClock()); + } + } + } + return actionCache; + } + + /** + * Generates a README file in the output base directory. This README file + * contains the name of the workspace directory, so that users can figure out + * which output base directory corresponds to which workspace. + */ + private void writeOutputBaseReadmeFile() { + Preconditions.checkNotNull(getWorkspace()); + Path outputBaseReadmeFile = getOutputBase().getRelative("README"); + try { + FileSystemUtils.writeIsoLatin1(outputBaseReadmeFile, "WORKSPACE: " + getWorkspace(), "", + "The first line of this file is intentionally easy to parse for various", + "interactive scripting and debugging purposes. But please DO NOT write programs", + "that exploit it, as they will be broken by design: it is not possible to", + "reverse engineer the set of source trees or the --package_path from the output", + "tree, and if you attempt it, you will fail, creating subtle and", + "hard-to-diagnose bugs, that will no doubt get blamed on changes made by the", + "Blaze team.", "", "This directory was generated by Blaze.", + "Do not attempt to modify or delete any files in this directory.", + "Among other issues, Blaze's file system caching assumes that", + "only Blaze will modify this directory and the files in it,", + "so if you change anything here you may mess up Blaze's cache."); + } catch (IOException e) { + LOG.warning("Couldn't write to '" + outputBaseReadmeFile + "': " + e.getMessage()); + } + } + + private void writeDoNotBuildHereFile(Path filePath) { + try { + FileSystemUtils.createDirectoryAndParents(filePath.getParentDirectory()); + FileSystemUtils.writeContent(filePath, ISO_8859_1, getWorkspace().toString()); + } catch (IOException e) { + LOG.warning("Couldn't write to '" + filePath + "': " + e.getMessage()); + } + } + + private void writeDoNotBuildHereFile(OptionsProvider startupOptions) { + Preconditions.checkNotNull(getWorkspace()); + writeDoNotBuildHereFile(getOutputBase().getRelative(DO_NOT_BUILD_FILE_NAME)); + if (startupOptions.getOptions(BlazeServerStartupOptions.class).deepExecRoot) { + writeDoNotBuildHereFile(getOutputBase().getRelative("execroot").getRelative( + DO_NOT_BUILD_FILE_NAME)); + } + } + + /** + * Creates the execRoot dir under outputBase. + */ + private void setupExecRoot() { + try { + FileSystemUtils.createDirectoryAndParents(directories.getExecRoot()); + } catch (IOException e) { + LOG.warning("failed to create execution root '" + directories.getExecRoot() + "': " + + e.getMessage()); + } + } +} + diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java index ee80c85deb..7c05e8ca75 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.PackageRootResolver; import com.google.devtools.build.lib.actions.cache.ActionCache; +import com.google.devtools.build.lib.actions.cache.CompactPersistentActionCache; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.BuildView; import com.google.devtools.build.lib.analysis.SkyframePackageRootResolver; @@ -73,6 +74,7 @@ import java.util.concurrent.atomic.AtomicReference; */ public final class CommandEnvironment { private final BlazeRuntime runtime; + private final BlazeWorkspace workspace; private final BlazeDirectories directories; private final UUID commandId; // Unique identifier for the command being run @@ -108,7 +110,8 @@ public final class CommandEnvironment { public CommandEnvironment(BlazeRuntime runtime, UUID commandId, EventBus eventBus) { this.runtime = runtime; - this.directories = runtime.getDirectories(); + this.workspace = runtime.getWorkspace(); + this.directories = workspace.getDirectories(); this.commandId = commandId; this.reporter = new Reporter(); this.eventBus = eventBus; @@ -129,6 +132,10 @@ public final class CommandEnvironment { return runtime; } + public BlazeWorkspace getBlazeWorkspace() { + return workspace; + } + public BlazeDirectories getDirectories() { return directories; } @@ -172,7 +179,7 @@ public final class CommandEnvironment { } public PackageManager getPackageManager() { - return runtime.getPackageManager(); + return getSkyframeExecutor().getPackageManager(); } public PathFragment getRelativeWorkingDirectory() { @@ -202,11 +209,11 @@ public final class CommandEnvironment { } public SkyframeExecutor getSkyframeExecutor() { - return runtime.getSkyframeExecutor(); + return workspace.getSkyframeExecutor(); } public SkyframeBuildView getSkyframeBuildView() { - return runtime.getSkyframeExecutor().getSkyframeBuildView(); + return getSkyframeExecutor().getSkyframeBuildView(); } /** @@ -286,7 +293,27 @@ public final class CommandEnvironment { } public ActionCache getPersistentActionCache() throws IOException { - return runtime.getPersistentActionCache(reporter); + return workspace.getPersistentActionCache(reporter); + } + + /** + * An array of String values useful if Blaze crashes. + * For now, just returns the size of the action cache and the build id. + */ + public String[] getCrashData() { + return new String[]{ + getFileSizeString(CompactPersistentActionCache.cacheFile(workspace.getCacheDirectory()), + "action cache"), + getCommandId() + " (build id)", + }; + } + + private static String getFileSizeString(Path path, String type) { + try { + return String.format("%d bytes (%s)", path.getFileSize(), type); + } catch (IOException e) { + return String.format("unknown file size (%s)", type); + } } /** @@ -314,8 +341,7 @@ public final class CommandEnvironment { private boolean loadForConfigurations(EventHandler eventHandler, Set<Label> labelsToLoad, boolean keepGoing) throws InterruptedException { // Use a new Label Visitor here to avoid erasing the cache on the existing one. - TransitivePackageLoader transitivePackageLoader = - runtime.getSkyframeExecutor().getPackageManager().newTransitiveLoader(); + TransitivePackageLoader transitivePackageLoader = getPackageManager().newTransitiveLoader(); boolean loadingSuccessful = transitivePackageLoader.sync( eventHandler, ImmutableSet.<Target>of(), labelsToLoad, keepGoing, /*parallelThreads=*/10, @@ -370,7 +396,7 @@ public final class CommandEnvironment { } public void recordLastExecutionTime() { - runtime.recordLastExecutionTime(getCommandStartTime()); + workspace.recordLastExecutionTime(getCommandStartTime()); } public void recordCommandStartTime(long commandStartTime) { diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java index 12630051e6..5ebac2c953 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java @@ -20,7 +20,6 @@ import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.runtime.BlazeCommand; import com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.ShutdownBlazeServerException; -import com.google.devtools.build.lib.runtime.BlazeRuntime; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.shell.CommandException; @@ -125,7 +124,6 @@ public final class CleanCommand implements BlazeCommand { private void actuallyClean(CommandEnvironment env, Path outputBase, Options cleanOptions, String symlinkPrefix) throws IOException, ShutdownBlazeServerException, CommandException, ExecException, InterruptedException { - BlazeRuntime runtime = env.getRuntime(); if (env.getOutputService() != null) { env.getOutputService().clean(); } @@ -162,7 +160,7 @@ public final class CleanCommand implements BlazeCommand { .build().execute(); } else { LOG.info("Output cleaning..."); - runtime.clearCaches(); + env.getBlazeWorkspace().clearCaches(); // In order to be sure that we delete everything, delete the workspace directory both for // --deep_execroot and for --nodeep_execroot. for (String directory : new String[] { diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/DumpCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/DumpCommand.java index 6acf54658b..b18714e695 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/DumpCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/DumpCommand.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.runtime.BlazeCommandUtils; import com.google.devtools.build.lib.runtime.BlazeRuntime; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.common.options.EnumConverter; @@ -166,7 +167,9 @@ public class DumpCommand implements BlazeCommand { } if (dumpOptions.dumpSkyframe != SkyframeDumpOption.OFF) { - success &= dumpSkyframe(runtime, dumpOptions.dumpSkyframe == SkyframeDumpOption.SUMMARY, + success &= dumpSkyframe( + env.getSkyframeExecutor(), + dumpOptions.dumpSkyframe == SkyframeDumpOption.SUMMARY, out); out.println(); } @@ -188,8 +191,8 @@ public class DumpCommand implements BlazeCommand { return true; } - private boolean dumpSkyframe(BlazeRuntime runtime, boolean summarize, PrintStream out) { - runtime.getSkyframeExecutor().dump(summarize, out); + private boolean dumpSkyframe(SkyframeExecutor executor, boolean summarize, PrintStream out) { + executor.dump(summarize, out); return true; } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java index 65d72552f2..f9b217d1d5 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/InfoCommand.java @@ -284,11 +284,11 @@ public class InfoCommand implements BlazeCommand { Supplier<BuildConfiguration> configurationSupplier, OptionsProvider options) { switch (key) { // directories - case WORKSPACE : return runtime.getDirectories().getWorkspace(); - case INSTALL_BASE : return runtime.getDirectories().getInstallBase(); - case OUTPUT_BASE : return runtime.getDirectories().getOutputBase(); - case EXECUTION_ROOT : return runtime.getDirectories().getExecRoot(); - case OUTPUT_PATH : return runtime.getDirectories().getOutputPath(); + case WORKSPACE : return runtime.getWorkspace().getWorkspace(); + case INSTALL_BASE : return runtime.getWorkspace().getInstallBase(); + case OUTPUT_BASE : return runtime.getWorkspace().getOutputBase(); + case EXECUTION_ROOT : return runtime.getWorkspace().getExecRoot(); + case OUTPUT_PATH : return runtime.getWorkspace().getOutputPath(); // These are the only (non-hidden) info items that require a configuration, because the // corresponding paths contain the short name. Maybe we should recommend using the symlinks // or make them hidden by default? @@ -298,10 +298,10 @@ public class InfoCommand implements BlazeCommand { // logs case COMMAND_LOG : return BlazeCommandDispatcher.getCommandLogPath( - runtime.getDirectories().getOutputBase()); + runtime.getWorkspace().getOutputBase()); case MESSAGE_LOG : // NB: Duplicated in EventLogModule - return runtime.getDirectories().getOutputBase().getRelative("message.log"); + return runtime.getWorkspace().getOutputBase().getRelative("message.log"); // misc case RELEASE : return BlazeVersionInfo.instance().getReleaseName(); |