diff options
Diffstat (limited to 'src/main/java')
13 files changed, 573 insertions, 499 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index c6e5649a9f..b107cba24f 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -436,6 +436,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:transitive-info-provider", ], deps = [ + ":build-request-options", "//src/main/java/com/google/devtools/build/lib:base-util", "//src/main/java/com/google/devtools/build/lib:events", "//src/main/java/com/google/devtools/build/lib:exitcode-external", @@ -961,6 +962,18 @@ java_library( ) java_library( + name = "build-request-options", + srcs = ["buildtool/BuildRequestOptions.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:util", + "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/common/options", + "//third_party:guava", + ], +) + +java_library( name = "runtime", srcs = glob( [ @@ -971,8 +984,10 @@ java_library( "buildtool/*.java", "buildtool/buildevent/*.java", ], + exclude = ["buildtool/BuildRequestOptions.java"], ), deps = [ + ":build-request-options", "//src/main/java/com/google/devtools/build/docgen:docgen_javalib", "//src/main/java/com/google/devtools/build/lib:build-base", "//src/main/java/com/google/devtools/build/lib:events", diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java index 1780b1e4c5..fb9a66a8c4 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfigurationCollection.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.analysis.config; import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.buildtool.BuildRequestOptions; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import java.util.HashMap; import java.util.List; @@ -24,7 +25,7 @@ import java.util.List; * * <p>The target configuration is used for all targets specified on the command line. Multiple * target configurations are possible because of settings like {@link - * com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions#multiCpus}. + * BuildRequestOptions#multiCpus}. * * <p>The host configuration is used for tools that are executed during the build, e. g, compilers. */ diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java index c6b4c7595f..6a489ccfa2 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequest.java @@ -13,7 +13,6 @@ // limitations under the License. package com.google.devtools.build.lib.buildtool; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.cache.CacheBuilder; @@ -21,7 +20,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; -import com.google.devtools.build.lib.actions.LocalHostCapacity; import com.google.devtools.build.lib.analysis.BuildView; import com.google.devtools.build.lib.analysis.OutputGroupProvider; import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; @@ -33,23 +31,13 @@ import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler; import com.google.devtools.build.lib.util.OptionsUtils; import com.google.devtools.build.lib.util.io.OutErr; -import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.common.options.Converters; -import com.google.devtools.common.options.Converters.RangeConverter; -import com.google.devtools.common.options.Option; -import com.google.devtools.common.options.OptionDocumentationCategory; -import com.google.devtools.common.options.OptionEffectTag; -import com.google.devtools.common.options.OptionMetadataTag; import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsClassProvider; -import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsProvider; import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; -import java.util.logging.Logger; -import java.util.regex.Pattern; /** * A BuildRequest represents a single invocation of the build tool by a user. @@ -58,440 +46,6 @@ import java.util.regex.Pattern; * as --keep_going, --jobs, etc. */ public class BuildRequest implements OptionsClassProvider { - private static final Logger logger = Logger.getLogger(BuildRequest.class.getName()); - - /** - * Options interface--can be used to parse command-line arguments. - * - * <p>See also ExecutionOptions; from the user's point of view, there's no - * qualitative difference between these two sets of options. - */ - public static class BuildRequestOptions extends OptionsBase { - - /* "Execution": options related to the execution of a build: */ - - @Option( - name = "jobs", - abbrev = 'j', - defaultValue = "auto", - category = "strategy", - documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, - effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION}, - converter = JobsConverter.class, - help = - "The number of concurrent jobs to run. 0 means build sequentially." - + " \"auto\" means to use a reasonable value derived from the machine's hardware" - + " profile (e.g. the number of processors). Values above " - + MAX_JOBS - + " are not allowed, and values above " - + JOBS_TOO_HIGH_WARNING - + " may cause memory issues." - ) - public int jobs; - - @Option( - name = "progress_report_interval", - defaultValue = "0", - category = "verbosity", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - converter = ProgressReportIntervalConverter.class, - help = - "The number of seconds to wait between two reports on still running jobs. The " - + "default value 0 means to use the default 10:30:60 incremental algorithm." - ) - public int progressReportInterval; - - @Option( - name = "explain", - defaultValue = "null", - category = "verbosity", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - converter = OptionsUtils.PathFragmentConverter.class, - help = - "Causes the build system to explain each executed step of the " - + "build. The explanation is written to the specified log file." - ) - public PathFragment explanationPath; - - @Option( - name = "verbose_explanations", - defaultValue = "false", - category = "verbosity", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = - "Increases the verbosity of the explanations issued if --explain is enabled. " - + "Has no effect if --explain is not enabled." - ) - public boolean verboseExplanations; - - @Option( - name = "output_filter", - converter = Converters.RegexPatternConverter.class, - defaultValue = "null", - category = "flags", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = "Only shows warnings for rules with a name matching the provided regular expression." - ) - public Pattern outputFilter; - - @Deprecated - @Option( - name = "dump_makefile", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "this flag has no effect." - ) - public boolean dumpMakefile; - - @Deprecated - @Option( - name = "dump_action_graph", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "this flag has no effect." - ) - public boolean dumpActionGraph; - - @Deprecated - @Option( - name = "dump_action_graph_for_package", - allowMultiple = true, - defaultValue = "", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "this flag has no effect." - ) - public List<String> dumpActionGraphForPackage = new ArrayList<>(); - - @Deprecated - @Option( - name = "dump_action_graph_with_middlemen", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "this flag has no effect." - ) - public boolean dumpActionGraphWithMiddlemen; - - @Deprecated - @Option( - name = "dump_providers", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "This is a no-op." - ) - public boolean dumpProviders; - - @Deprecated - @Option( - name = "dump_targets", - defaultValue = "null", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "this flag has no effect." - ) - public String dumpTargets; - - @Deprecated - @Option( - name = "dump_host_deps", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "Deprecated" - ) - public boolean dumpHostDeps; - - @Deprecated - @Option( - name = "dump_to_stdout", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.NO_OP}, - metadataTags = {OptionMetadataTag.DEPRECATED}, - help = "Deprecated" - ) - public boolean dumpToStdout; - - @Option( - name = "experimental_post_build_query", - defaultValue = "null", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.UNKNOWN} - ) - public String queryExpression; - - @Option( - name = "analyze", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = { - OptionEffectTag.LOADING_AND_ANALYSIS, - OptionEffectTag.AFFECTS_OUTPUTS - }, - help = - "Execute the analysis phase; this is the usual behaviour. Specifying --noanalyze causes " - + "the build to stop before starting the analysis phase, returning zero iff the " - + "package loading completed successfully; this mode is useful for testing." - ) - public boolean performAnalysisPhase; - - @Option( - name = "build", - defaultValue = "true", - category = "what", - documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION, - effectTags = { - OptionEffectTag.EXECUTION, - OptionEffectTag.AFFECTS_OUTPUTS - }, - help = - "Execute the build; this is the usual behaviour. " - + "Specifying --nobuild causes the build to stop before executing the build " - + "actions, returning zero iff the package loading and analysis phases completed " - + "successfully; this mode is useful for testing those phases." - ) - public boolean performExecutionPhase; - - @Option( - name = "output_groups", - converter = Converters.CommaSeparatedOptionListConverter.class, - allowMultiple = true, - documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION, - effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.AFFECTS_OUTPUTS}, - defaultValue = "", - help = - "Specifies which output groups of the top-level targets to build. If omitted, a default " - + "set of output groups are built. When specified the default set is overridden." - + "However you may use --output_groups=+<output_group> or " - + "--output_groups=-<output_group> to instead modify the set of output groups." - ) - public List<String> outputGroups; - - @Option( - name = "show_result", - defaultValue = "1", - category = "verbosity", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = - "Show the results of the build. For each target, state whether or not it was brought " - + "up-to-date, and if so, a list of output files that were built. The printed files " - + "are convenient strings for copy+pasting to the shell, to execute them.\n" - + "This option requires an integer argument, which is the threshold number of " - + "targets above which result information is not printed. Thus zero causes " - + "suppression of the message and MAX_INT causes printing of the result to occur " - + "always. The default is one." - ) - public int maxResultTargets; - - @Option( - name = "experimental_show_artifacts", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = - "Output a list of all top level artifacts produced by this build." - + "Use output format suitable for tool consumption. " - + "This flag is temporary and intended to facilitate Android Studio integration. " - + "This output format will likely change in the future or disappear completely." - ) - public boolean showArtifacts; - - @Option( - name = "announce", - defaultValue = "false", - category = "verbosity", - documentationCategory = OptionDocumentationCategory.LOGGING, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = "Deprecated. No-op.", - deprecationWarning = "This option is now deprecated and is a no-op" - ) - public boolean announce; - - @Option( - name = "symlink_prefix", - defaultValue = "null", - category = "misc", - documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = - "The prefix that is prepended to any of the convenience symlinks that are created " - + "after a build. If '/' is passed, then no symlinks are created and no warning is " - + "emitted. If omitted, the default value is the name of the build tool." - ) - public String symlinkPrefix; - - @Option( - name = "experimental_multi_cpu", - converter = Converters.CommaSeparatedOptionListConverter.class, - allowMultiple = true, - defaultValue = "", - category = "semantics", - documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - metadataTags = {OptionMetadataTag.EXPERIMENTAL}, - help = - "This flag allows specifying multiple target CPUs. If this is specified, " - + "the --cpu option is ignored." - ) - public List<String> multiCpus; - - @Option( - name = "output_tree_tracking", - oldName = "experimental_output_tree_tracking", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, - help = - "If set, tell the output service (if any) to track when files in the output " - + "tree have been modified externally (not by the build system). " - + "This should improve incremental build speed when an appropriate output service " - + "is enabled." - ) - public boolean finalizeActions; - - @Option( - name = "aspects", - converter = Converters.CommaSeparatedOptionListConverter.class, - defaultValue = "", - documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, - effectTags = {OptionEffectTag.UNKNOWN}, - allowMultiple = true, - help = - "Comma-separated list of aspects to be applied to top-level targets. All aspects " - + "are applied to all top-level targets independently. Aspects are specified in " - + "the form <bzl-file-label>%<aspect_name>, " - + "for example '//tools:my_def.bzl%my_aspect', where 'my_aspect' is a top-level " - + "value from from a file tools/my_def.bzl" - ) - public List<String> aspects; - - public String getSymlinkPrefix(String productName) { - return symlinkPrefix == null ? productName + "-" : symlinkPrefix; - } - - // Transitional flag for safely rolling out new convenience symlink behavior. - // To be made a no-op and deleted once new symlink behavior is battle-tested. - @Option( - name = "use_top_level_targets_for_symlinks", - defaultValue = "false", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, - help = - "If enabled, the symlinks are based on the configurations of the top-level targets " - + " rather than the top-level target configuration. If this would be ambiguous, " - + " the symlinks will be deleted to avoid confusion." - ) - public boolean useTopLevelTargetsForSymlinks; - - /** - * Returns whether to use the output directories used by the top-level targets for convenience - * symlinks. - * - * <p>If true, then symlinks use the actual output directories of the top-level targets. - * The symlinks will be created iff all top-level targets share the same output directory. - * Otherwise, any stale symlinks from previous invocations will be deleted to avoid ambiguity. - * - * <p>If false, then symlinks use the output directory implied by command-line flags, regardless - * of whether top-level targets have transitions which change them (or even have any output - * directories at all, as in the case of a build with no targets or one which only builds source - * files). - */ - public boolean useTopLevelTargetsForSymlinks() { - return useTopLevelTargetsForSymlinks; - } - - @Option( - name = "use_action_cache", - defaultValue = "true", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = { - OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION, - OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS - }, - help = "Whether to use the action cache" - ) - public boolean useActionCache; - } - - /** Converter for jobs: [0, MAX_JOBS] or "auto". */ - public static class JobsConverter extends RangeConverter { - /** If not null, indicates the value to return when "auto" is selected. Useful for cases - * where the number of jobs is bound by another factor different than what we compute here. */ - private static Integer fixedAutoJobs; - - public JobsConverter() { - super(0, MAX_JOBS); - } - - @Override - public Integer convert(String input) throws OptionsParsingException { - if (input.equals("auto")) { - int jobs; - if (fixedAutoJobs == null) { - jobs = (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()); - if (jobs > MAX_JOBS) { - logger.warning( - "Detected " - + jobs - + " processors, which exceed the maximum allowed number of jobs of " - + MAX_JOBS - + "; something seems wrong"); - jobs = MAX_JOBS; - } - } else { - jobs = fixedAutoJobs; - } - logger.info("Flag \"jobs\" was set to \"auto\"; using " + jobs + " jobs"); - return jobs; - } else { - return super.convert(input); - } - } - - @Override - public String getTypeDescription() { - return "\"auto\" or " + super.getTypeDescription(); - } - - /** - * Sets the value to return by this converter when "auto" is selected. - * - * @param jobs the number of jobs to return, or null to reenable automated detection - */ - public static void setFixedAutoJobs(Integer jobs) { - Preconditions.checkArgument(jobs == null || jobs <= MAX_JOBS); - fixedAutoJobs = jobs; - } - } - - /** - * Converter for progress_report_interval: [0, 3600]. - */ - public static class ProgressReportIntervalConverter extends RangeConverter { - public ProgressReportIntervalConverter() { - super(0, 3600); - } - } - - @VisibleForTesting public static final int MAX_JOBS = 3000; - private static final int JOBS_TOO_HIGH_WARNING = 1500; - private final UUID id; private final LoadingCache<Class<? extends OptionsBase>, Optional<OptionsBase>> optionsCache; diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java new file mode 100644 index 0000000000..a771237955 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java @@ -0,0 +1,471 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.buildtool; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.actions.LocalHostCapacity; +import com.google.devtools.build.lib.util.OptionsUtils; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converters; +import com.google.devtools.common.options.Converters.RangeConverter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionMetadataTag; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +/** + * Options interface for {@link BuildRequest}: can be used to parse command-line arguments. + * + * <p>See also {@code ExecutionOptions}; from the user's point of view, there's no qualitative + * difference between these two sets of options. + */ +public class BuildRequestOptions extends OptionsBase { + private static final Logger logger = Logger.getLogger(BuildRequestOptions.class.getName()); + private static final int JOBS_TOO_HIGH_WARNING = 1500; + @VisibleForTesting public static final int MAX_JOBS = 3000; + + /* "Execution": options related to the execution of a build: */ + + @Option( + name = "jobs", + abbrev = 'j', + defaultValue = "auto", + category = "strategy", + documentationCategory = OptionDocumentationCategory.EXECUTION_STRATEGY, + effectTags = {OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS, OptionEffectTag.EXECUTION}, + converter = JobsConverter.class, + help = + "The number of concurrent jobs to run. 0 means build sequentially." + + " \"auto\" means to use a reasonable value derived from the machine's hardware" + + " profile (e.g. the number of processors). Values above " + + MAX_JOBS + + " are not allowed, and values above " + + JOBS_TOO_HIGH_WARNING + + " may cause memory issues." + ) + public int jobs; + + @Option( + name = "progress_report_interval", + defaultValue = "0", + category = "verbosity", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + converter = ProgressReportIntervalConverter.class, + help = + "The number of seconds to wait between two reports on still running jobs. The " + + "default value 0 means to use the default 10:30:60 incremental algorithm." + ) + public int progressReportInterval; + + @Option( + name = "explain", + defaultValue = "null", + category = "verbosity", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + converter = OptionsUtils.PathFragmentConverter.class, + help = + "Causes the build system to explain each executed step of the " + + "build. The explanation is written to the specified log file." + ) + public PathFragment explanationPath; + + @Option( + name = "verbose_explanations", + defaultValue = "false", + category = "verbosity", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "Increases the verbosity of the explanations issued if --explain is enabled. " + + "Has no effect if --explain is not enabled." + ) + public boolean verboseExplanations; + + @Option( + name = "output_filter", + converter = Converters.RegexPatternConverter.class, + defaultValue = "null", + category = "flags", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = "Only shows warnings for rules with a name matching the provided regular expression." + ) + public Pattern outputFilter; + + @Deprecated + @Option( + name = "dump_makefile", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "this flag has no effect." + ) + public boolean dumpMakefile; + + @Deprecated + @Option( + name = "dump_action_graph", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "this flag has no effect." + ) + public boolean dumpActionGraph; + + @Deprecated + @Option( + name = "dump_action_graph_for_package", + allowMultiple = true, + defaultValue = "", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "this flag has no effect." + ) + public List<String> dumpActionGraphForPackage = new ArrayList<>(); + + @Deprecated + @Option( + name = "dump_action_graph_with_middlemen", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "this flag has no effect." + ) + public boolean dumpActionGraphWithMiddlemen; + + @Deprecated + @Option( + name = "dump_providers", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "This is a no-op." + ) + public boolean dumpProviders; + + @Deprecated + @Option( + name = "dump_targets", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "this flag has no effect." + ) + public String dumpTargets; + + @Deprecated + @Option( + name = "dump_host_deps", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "Deprecated" + ) + public boolean dumpHostDeps; + + @Deprecated + @Option( + name = "dump_to_stdout", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.NO_OP}, + metadataTags = {OptionMetadataTag.DEPRECATED}, + help = "Deprecated" + ) + public boolean dumpToStdout; + + @Option( + name = "experimental_post_build_query", + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.UNKNOWN} + ) + public String queryExpression; + + @Option( + name = "analyze", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS, OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "Execute the analysis phase; this is the usual behaviour. Specifying --noanalyze causes " + + "the build to stop before starting the analysis phase, returning zero iff the " + + "package loading completed successfully; this mode is useful for testing." + ) + public boolean performAnalysisPhase; + + @Option( + name = "build", + defaultValue = "true", + category = "what", + documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION, + effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "Execute the build; this is the usual behaviour. " + + "Specifying --nobuild causes the build to stop before executing the build " + + "actions, returning zero iff the package loading and analysis phases completed " + + "successfully; this mode is useful for testing those phases." + ) + public boolean performExecutionPhase; + + @Option( + name = "output_groups", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + documentationCategory = OptionDocumentationCategory.OUTPUT_SELECTION, + effectTags = {OptionEffectTag.EXECUTION, OptionEffectTag.AFFECTS_OUTPUTS}, + defaultValue = "", + help = + "Specifies which output groups of the top-level targets to build. If omitted, a default " + + "set of output groups are built. When specified the default set is overridden." + + "However you may use --output_groups=+<output_group> or " + + "--output_groups=-<output_group> to instead modify the set of output groups." + ) + public List<String> outputGroups; + + @Option( + name = "show_result", + defaultValue = "1", + category = "verbosity", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "Show the results of the build. For each target, state whether or not it was brought " + + "up-to-date, and if so, a list of output files that were built. The printed files " + + "are convenient strings for copy+pasting to the shell, to execute them.\n" + + "This option requires an integer argument, which is the threshold number of " + + "targets above which result information is not printed. Thus zero causes " + + "suppression of the message and MAX_INT causes printing of the result to occur " + + "always. The default is one." + ) + public int maxResultTargets; + + @Option( + name = "experimental_show_artifacts", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "Output a list of all top level artifacts produced by this build." + + "Use output format suitable for tool consumption. " + + "This flag is temporary and intended to facilitate Android Studio integration. " + + "This output format will likely change in the future or disappear completely." + ) + public boolean showArtifacts; + + @Option( + name = "announce", + defaultValue = "false", + category = "verbosity", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = "Deprecated. No-op.", + deprecationWarning = "This option is now deprecated and is a no-op" + ) + public boolean announce; + + @Option( + name = "symlink_prefix", + defaultValue = "null", + category = "misc", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "The prefix that is prepended to any of the convenience symlinks that are created " + + "after a build. If '/' is passed, then no symlinks are created and no warning is " + + "emitted. If omitted, the default value is the name of the build tool." + ) + public String symlinkPrefix; + + @Option( + name = "experimental_multi_cpu", + converter = Converters.CommaSeparatedOptionListConverter.class, + allowMultiple = true, + defaultValue = "", + category = "semantics", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + metadataTags = {OptionMetadataTag.EXPERIMENTAL}, + help = + "This flag allows specifying multiple target CPUs. If this is specified, " + + "the --cpu option is ignored." + ) + public List<String> multiCpus; + + @Option( + name = "output_tree_tracking", + oldName = "experimental_output_tree_tracking", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION}, + help = + "If set, tell the output service (if any) to track when files in the output " + + "tree have been modified externally (not by the build system). " + + "This should improve incremental build speed when an appropriate output service " + + "is enabled." + ) + public boolean finalizeActions; + + @Option( + name = "aspects", + converter = Converters.CommaSeparatedOptionListConverter.class, + defaultValue = "", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.UNKNOWN}, + allowMultiple = true, + help = + "Comma-separated list of aspects to be applied to top-level targets. All aspects " + + "are applied to all top-level targets independently. Aspects are specified in " + + "the form <bzl-file-label>%<aspect_name>, " + + "for example '//tools:my_def.bzl%my_aspect', where 'my_aspect' is a top-level " + + "value from from a file tools/my_def.bzl" + ) + public List<String> aspects; + + public String getSymlinkPrefix(String productName) { + return symlinkPrefix == null ? productName + "-" : symlinkPrefix; + } + + // Transitional flag for safely rolling out new convenience symlink behavior. + // To be made a no-op and deleted once new symlink behavior is battle-tested. + @Option( + name = "use_top_level_targets_for_symlinks", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "If enabled, the symlinks are based on the configurations of the top-level targets " + + " rather than the top-level target configuration. If this would be ambiguous, " + + " the symlinks will be deleted to avoid confusion." + ) + public boolean useTopLevelTargetsForSymlinks; + + /** + * Returns whether to use the output directories used by the top-level targets for convenience + * symlinks. + * + * <p>If true, then symlinks use the actual output directories of the top-level targets. The + * symlinks will be created iff all top-level targets share the same output directory. Otherwise, + * any stale symlinks from previous invocations will be deleted to avoid ambiguity. + * + * <p>If false, then symlinks use the output directory implied by command-line flags, regardless + * of whether top-level targets have transitions which change them (or even have any output + * directories at all, as in the case of a build with no targets or one which only builds source + * files). + */ + public boolean useTopLevelTargetsForSymlinks() { + return useTopLevelTargetsForSymlinks; + } + + @Option( + name = "use_action_cache", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = { + OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION, + OptionEffectTag.HOST_MACHINE_RESOURCE_OPTIMIZATIONS + }, + help = "Whether to use the action cache" + ) + public boolean useActionCache; + + @Option( + name = "keep_incrementality_data", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.BUILD_TIME_OPTIMIZATION, + effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE}, + help = + "If false, discard Blaze-internal data that allows for invalidation and re-evaluation " + + "on incremental builds in order to save memory on this build. Subsequent builds " + + "will not have any incrementality with respect to this one. Usually you will want" + + "to specify the --batch startup option along with this one." + ) + public boolean keepIncrementalityData; + + /** Converter for jobs: [0, MAX_JOBS] or "auto". */ + public static class JobsConverter extends RangeConverter { + /** + * If not null, indicates the value to return when "auto" is selected. Useful for cases where + * the number of jobs is bound by another factor different than what we compute here. + */ + private static Integer fixedAutoJobs; + + public JobsConverter() { + super(0, MAX_JOBS); + } + + @Override + public Integer convert(String input) throws OptionsParsingException { + if (input.equals("auto")) { + int jobs; + if (fixedAutoJobs == null) { + jobs = (int) Math.ceil(LocalHostCapacity.getLocalHostCapacity().getCpuUsage()); + if (jobs > MAX_JOBS) { + logger.warning( + "Detected " + + jobs + + " processors, which exceed the maximum allowed number of jobs of " + + MAX_JOBS + + "; something seems wrong"); + jobs = MAX_JOBS; + } + } else { + jobs = fixedAutoJobs; + } + logger.info("Flag \"jobs\" was set to \"auto\"; using " + jobs + " jobs"); + return jobs; + } else { + return super.convert(input); + } + } + + @Override + public String getTypeDescription() { + return "\"auto\" or " + super.getTypeDescription(); + } + + /** + * Sets the value to return by this converter when "auto" is selected. + * + * @param jobs the number of jobs to return, or null to reenable automated detection + */ + public static void setFixedAutoJobs(Integer jobs) { + Preconditions.checkArgument(jobs == null || jobs <= MAX_JOBS); + fixedAutoJobs = jobs; + } + } + + /** Converter for progress_report_interval: [0, 3600]. */ + public static class ProgressReportIntervalConverter extends RangeConverter { + public ProgressReportIntervalConverter() { + super(0, 3600); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java index 72908523f0..d785f51ee6 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java @@ -40,7 +40,6 @@ import com.google.devtools.build.lib.analysis.config.InvalidConfigurationExcepti import com.google.devtools.build.lib.buildeventstream.AbortedEvent; import com.google.devtools.build.lib.buildeventstream.BuildEventId; import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.Aborted.AbortReason; -import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 24d859a915..e878b6fe2f 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -437,10 +437,9 @@ public class ExecutionTool { executor.executionPhaseStarting(); skyframeExecutor.drainChangedFiles(); - if (request.getViewOptions().discardAnalysisCache) { - // Free memory by removing cache entries that aren't going to be needed. Note that in - // skyframe full, this destroys the action graph as well, so we can only do it after the - // action graph is no longer needed. + if (request.getViewOptions().discardAnalysisCache + || !request.getBuildOptions().keepIncrementalityData) { + // Free memory by removing cache entries that aren't going to be needed. env.getSkyframeBuildView() .clearAnalysisCache(analysisResult.getTargetsToBuild(), analysisResult.getAspects()); } @@ -685,7 +684,7 @@ public class ExecutionTool { ActionCache actionCache, SkyframeExecutor skyframeExecutor, ModifiedFileSet modifiedOutputFiles) { - BuildRequest.BuildRequestOptions options = request.getBuildOptions(); + BuildRequestOptions options = request.getBuildOptions(); boolean keepGoing = request.getViewOptions().keepGoing; Path actionOutputRoot = env.getActionConsoleOutputDirectory(); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java index d8e5a51876..54552a6c5f 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java @@ -21,7 +21,6 @@ import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.PackageRootResolver; import com.google.devtools.build.lib.actions.cache.ActionCache; import com.google.devtools.build.lib.analysis.BlazeDirectories; -import com.google.devtools.build.lib.analysis.BuildView; import com.google.devtools.build.lib.analysis.SkyframePackageRootResolver; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.DefaultsPackage; @@ -607,9 +606,10 @@ public final class CommandEnvironment { // Fail fast in the case where a Blaze command forgets to install the package path correctly. skyframeExecutor.setActive(false); // Let skyframe figure out if it needs to store graph edges for this build. - skyframeExecutor.decideKeepIncrementalStateAndResetEvaluatorIfNecessary( + skyframeExecutor.decideKeepIncrementalState( runtime.getStartupOptionsProvider().getOptions(BlazeServerStartupOptions.class).batch, - options.getOptions(BuildView.Options.class)); + options, + reporter); // Start the performance and memory profilers. runtime.beforeCommand(this, commonOptions, execStartTimeNanos); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java index 5f8077f2e6..d0aa57bb81 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java @@ -15,7 +15,7 @@ package com.google.devtools.build.lib.runtime.commands; import com.google.devtools.build.lib.analysis.BuildView; import com.google.devtools.build.lib.buildtool.BuildRequest; -import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.BuildRequestOptions; import com.google.devtools.build.lib.buildtool.BuildTool; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java index 88ff555570..ba0ca599ba 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/CleanCommand.java @@ -15,7 +15,7 @@ package com.google.devtools.build.lib.runtime.commands; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.analysis.NoBuildEvent; -import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.BuildRequestOptions; import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.runtime.BlazeCommand; @@ -182,8 +182,10 @@ public final class CleanCommand implements BlazeCommand { env.getReporter().handle(Event.info(null/*location*/, cleanBanner)); try { - String symlinkPrefix = options.getOptions(BuildRequest.BuildRequestOptions.class) - .getSymlinkPrefix(env.getRuntime().getProductName()); + String symlinkPrefix = + options + .getOptions(BuildRequestOptions.class) + .getSymlinkPrefix(env.getRuntime().getProductName()); actuallyClean(env, env.getOutputBase(), expunge, expungeAsync, async, symlinkPrefix); return ExitCode.SUCCESS; } catch (IOException e) { diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java index 17f204ae8f..aee56d498a 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/RunCommand.java @@ -28,7 +28,7 @@ import com.google.devtools.build.lib.analysis.actions.CommandLine; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.RunUnder; import com.google.devtools.build.lib.buildtool.BuildRequest; -import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; +import com.google.devtools.build.lib.buildtool.BuildRequestOptions; import com.google.devtools.build.lib.buildtool.BuildResult; import com.google.devtools.build.lib.buildtool.BuildTool; import com.google.devtools.build.lib.buildtool.OutputDirectoryLinksUtils; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java index cf3c2cf7b8..5dac255451 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java @@ -27,13 +27,16 @@ import com.google.common.collect.Maps; import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.devtools.build.lib.analysis.BlazeDirectories; -import com.google.devtools.build.lib.analysis.BuildView.Options; +import com.google.devtools.build.lib.analysis.BuildView; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; +import com.google.devtools.build.lib.buildtool.BuildRequestOptions; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.concurrent.Uninterruptibles; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.AspectClass; import com.google.devtools.build.lib.packages.Package; @@ -77,6 +80,7 @@ import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsProvider; import java.io.PrintStream; import java.util.ArrayList; import java.util.Collection; @@ -107,13 +111,19 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { CLEAR_EDGES_AND_ACTIONS } - // Can only be set once over the lifetime of this object. If CLEAR_EDGES or - // CLEAR_EDGES_AND_ACTIONS, the graph will not store edges, saving memory but making incremental - // builds impossible. If CLEAR_EDGES_AND_ACTIONS, each action will be dereferenced once it is - // executed, saving memory. + /** + * If {@link IncrementalState#CLEAR_EDGES_AND_ACTIONS}, the graph will not store edges, saving + * memory but making subsequent builds not incremental. Also, each action will be dereferenced + * once it is executed, saving memory. + */ private IncrementalState incrementalState = IncrementalState.NORMAL; - private RecordingDifferencer recordingDiffer; + private boolean evaluatorNeedsReset = false; + + // This is intentionally not kept in sync with the evaluator: we may reset the evaluator without + // ever losing injected/invalidated data here. This is safe because the worst that will happen is + // that on the next build we try to inject/invalidate some nodes that aren't needed for the build. + private final RecordingDifferencer recordingDiffer = new SequencedRecordingDifferencer(); private final DiffAwarenessManager diffAwarenessManager; private final Iterable<SkyValueDirtinessChecker> customDirtinessCheckers; private Set<String> previousClientEnvironment = null; @@ -191,14 +201,6 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { } @Override - protected void init() { - // Note that we need to set recordingDiffer first since SkyframeExecutor#init calls - // SkyframeExecutor#evaluatorDiffer. - recordingDiffer = new SequencedRecordingDifferencer(); - super.init(); - } - - @Override public void resetEvaluator() { super.resetEvaluator(); diffAwarenessManager.reset(); @@ -232,6 +234,12 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { TimestampGranularityMonitor tsgm, OptionsClassProvider options) throws InterruptedException, AbruptExitException { + if (evaluatorNeedsReset) { + // Recreate MemoizingEvaluator so that graph is recreated with correct edge-clearing status, + // or if the graph doesn't have edges, so that a fresh graph can be used. + resetEvaluator(); + evaluatorNeedsReset = false; + } super.sync(eventHandler, packageCacheOptions, skylarkSemanticsOptions, outputBase, workingDirectory, defaultsPackageContents, commandId, clientEnv, tsgm, options); handleDiffs(eventHandler, packageCacheOptions.checkOutputFiles, options); @@ -490,24 +498,44 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { } @Override - public void decideKeepIncrementalStateAndResetEvaluatorIfNecessary( - boolean batch, Options viewOptions) { + public void decideKeepIncrementalState( + boolean batch, OptionsProvider options, EventHandler eventHandler) { Preconditions.checkState(!active); - if (viewOptions == null) { - // Some blaze commands don't include the view options. Don't bother with them. - return; + BuildView.Options viewOptions = options.getOptions(BuildView.Options.class); + BuildRequestOptions requestOptions = options.getOptions(BuildRequestOptions.class); + boolean explicitlyRequestedNoIncrementalData = + requestOptions != null && !requestOptions.keepIncrementalityData; + boolean implicitlyRequestedNoIncrementalData = + batch && viewOptions != null && viewOptions.discardAnalysisCache; + boolean discardingEdges = + explicitlyRequestedNoIncrementalData || implicitlyRequestedNoIncrementalData; + if (explicitlyRequestedNoIncrementalData != implicitlyRequestedNoIncrementalData) { + if (requestOptions != null && !explicitlyRequestedNoIncrementalData) { + eventHandler.handle( + Event.warn( + "--batch and --discard_analysis_cache specified, but --nokeep_incrementality_data " + + "not specified: incrementality data is implicitly discarded, but you may need" + + " to specify --nokeep_incrementality_data in the future if you want to " + + "maximize memory savings.")); + } + if (!batch) { + eventHandler.handle( + Event.warn( + "--batch not specified with --nokeep_incrementality_data: the server will " + + "remain running, but the next build will not be incremental on this one.")); + } } - if (batch && viewOptions.discardAnalysisCache) { - Preconditions.checkState( - incrementalState == IncrementalState.NORMAL, - "May only be called once if successful: %s", - incrementalState); - incrementalState = IncrementalState.CLEAR_EDGES_AND_ACTIONS; + IncrementalState oldState = incrementalState; + incrementalState = + discardingEdges ? IncrementalState.CLEAR_EDGES_AND_ACTIONS : IncrementalState.NORMAL; + if (oldState != incrementalState) { logger.info("Set incremental state to " + incrementalState); - // Recreate MemoizingEvaluator so that graph is recreated with edge-clearing enabled. - resetEvaluator(); + evaluatorNeedsReset = true; + removeActionsAfterEvaluation.set( + incrementalState == IncrementalState.CLEAR_EDGES_AND_ACTIONS); + } else if (incrementalState == IncrementalState.CLEAR_EDGES_AND_ACTIONS) { + evaluatorNeedsReset = true; } - removeActionsAfterEvaluation.set(incrementalState == IncrementalState.CLEAR_EDGES_AND_ACTIONS); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 413ea01fc1..ec365660ce 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -53,7 +53,6 @@ import com.google.devtools.build.lib.actions.ResourceManager; import com.google.devtools.build.lib.actions.Root; import com.google.devtools.build.lib.analysis.AspectCollection; import com.google.devtools.build.lib.analysis.BlazeDirectories; -import com.google.devtools.build.lib.analysis.BuildView.Options; import com.google.devtools.build.lib.analysis.ConfiguredAspect; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredTarget; @@ -81,6 +80,7 @@ import com.google.devtools.build.lib.cmdline.TargetParsingException; import com.google.devtools.build.lib.concurrent.ThreadSafety; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.events.ErrorSensingEventHandler; +import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.packages.AspectDescriptor; @@ -149,6 +149,7 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory; import com.google.devtools.common.options.OptionsClassProvider; +import com.google.devtools.common.options.OptionsProvider; import java.io.PrintStream; import java.util.ArrayList; import java.util.Collection; @@ -682,8 +683,8 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { } /** - * Decides if graph edges should be stored for this build. If not, re-creates the graph to not - * store graph edges. Necessary conditions to not store graph edges are: + * Decides if graph edges should be stored for this build. If not, notes that the next evaluation + * on the graph should reset it first. Necessary conditions to not store graph edges are: * * <ol> * <li>batch (since incremental builds are not possible); @@ -692,8 +693,8 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { * way). * </ol> */ - public void decideKeepIncrementalStateAndResetEvaluatorIfNecessary( - boolean batch, Options viewOptions) { + public void decideKeepIncrementalState( + boolean batch, OptionsProvider viewOptions, EventHandler eventHandler) { // Assume incrementality. } diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java index ec636b324e..541d340783 100644 --- a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java @@ -349,11 +349,15 @@ public final class InMemoryMemoizingEvaluator implements MemoizingEvaluator { if (entry.isDone()) { out.print(keyFormatter.apply(key)); out.print("|"); - try { - out.println( - Joiner.on('|').join(Iterables.transform(entry.getDirectDeps(), keyFormatter))); - } catch (InterruptedException e) { - throw new IllegalStateException("InMemoryGraph doesn't throw: " + entry, e); + if (((InMemoryNodeEntry) entry).keepEdges() == NodeEntry.KeepEdgesPolicy.NONE) { + out.println(" (direct deps not stored)"); + } else { + try { + out.println( + Joiner.on('|').join(Iterables.transform(entry.getDirectDeps(), keyFormatter))); + } catch (InterruptedException e) { + throw new IllegalStateException("InMemoryGraph doesn't throw: " + entry, e); + } } } } |